From 578c3e8f71c4ba259017ca5b52749cb839c20a5a Mon Sep 17 00:00:00 2001 From: Frank Date: Sun, 9 Aug 2026 11:54:40 -0700 Subject: [PATCH] Enable DFlash speculative decoding (12 tokens), increase CPU offload to 20GB --- scripts/vllm-awq-qwen36-dflash-ampere.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/vllm-awq-qwen36-dflash-ampere.sh b/scripts/vllm-awq-qwen36-dflash-ampere.sh index d7a1f72..c80dea8 100644 --- a/scripts/vllm-awq-qwen36-dflash-ampere.sh +++ b/scripts/vllm-awq-qwen36-dflash-ampere.sh @@ -22,7 +22,7 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \ -v ~/.cache/huggingface:/root/.cache/huggingface \ vllm/vllm-openai:nightly "${MODEL}" \ --served-model-name "frank-default-small" \ - # --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 4}' \ + --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}' \ --max-num-seqs 2 \ --max-model-len 32768 \ --tensor-parallel-size 1 \ @@ -33,6 +33,6 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \ --reasoning-parser qwen3 \ --trust-remote-code \ --gpu-memory-utilization 0.95 \ - --cpu-offload-gb 13 \ + --cpu-offload-gb 20 \ --cpu-offload-params "experts" \ --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'