Fix: move commented speculative config to end to not break bash line continuation
This commit is contained in:
1 parent
88aa18ef34
commit
83b0bea29b
1 file changed
+4
-2
@@ -23,7 +23,6 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
|
|||||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||||
vllm/vllm-openai:nightly "${MODEL}" \
|
vllm/vllm-openai:nightly "${MODEL}" \
|
||||||
--served-model-name "frank-default-small" \
|
--served-model-name "frank-default-small" \
|
||||||
# --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}' \
|
|
||||||
--max-num-seqs 2 \
|
--max-num-seqs 2 \
|
||||||
--max-model-len 32768 \
|
--max-model-len 32768 \
|
||||||
--tensor-parallel-size 1 \
|
--tensor-parallel-size 1 \
|
||||||
@@ -33,7 +32,10 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
|
|||||||
--enable-auto-tool-choice --tool-call-parser qwen3_xml \
|
--enable-auto-tool-choice --tool-call-parser qwen3_xml \
|
||||||
--reasoning-parser qwen3 \
|
--reasoning-parser qwen3 \
|
||||||
--trust-remote-code \
|
--trust-remote-code \
|
||||||
--gpu-memory-utilization 0.75 \
|
--gpu-memory-utilization 0.95 \
|
||||||
--cpu-offload-gb 20 \
|
--cpu-offload-gb 20 \
|
||||||
--cpu-offload-params "experts" \
|
--cpu-offload-params "experts" \
|
||||||
--default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'
|
--default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'
|
||||||
|
|
||||||
|
# TODO: uncomment and move to after --served-model-name to enable DFlash
|
||||||
|
# --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}' \
|
||||||
Reference in new issue
Block a user