dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Enable DFlash speculative decoding (12 tokens), increase CPU offload to 20GB

Browse Source
This commit is contained in:
Frank committed 2026-08-09 11:54:40 -07:00
1 parent 8c13c7bbaa
commit 578c3e8f71
1 file changed
+2 -2
Download Patch File Download Diff File Expand all files Collapse all files
scripts/vllm-awq-qwen36-dflash-ampere.sh
+2 -2
View File
@@ -22,7 +22,7 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
-v ~/.cache/huggingface:/root/.cache/huggingface \ -v ~/.cache/huggingface:/root/.cache/huggingface \
vllm/vllm-openai:nightly "${MODEL}" \ vllm/vllm-openai:nightly "${MODEL}" \
--served-model-name "frank-default-small" \ --served-model-name "frank-default-small" \
# --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 4}' \ --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}' \
--max-num-seqs 2 \ --max-num-seqs 2 \
--max-model-len 32768 \ --max-model-len 32768 \
--tensor-parallel-size 1 \ --tensor-parallel-size 1 \
@@ -33,6 +33,6 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
--reasoning-parser qwen3 \ --reasoning-parser qwen3 \
--trust-remote-code \ --trust-remote-code \
--gpu-memory-utilization 0.95 \ --gpu-memory-utilization 0.95 \
--cpu-offload-gb 13 \ --cpu-offload-gb 20 \
--cpu-offload-params "experts" \ --cpu-offload-params "experts" \
--default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'