dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

Add VLLM_USE_V2_MODEL_RUNNER=0 to skip V2 model runner MARLIN repacking OOM

Browse Source
This commit is contained in:
Frank
2026-08-09 21:49:25 -07:00
parent 83b0bea29b
commit 6bdabfc65b
1 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/vllm-awq-qwen36-dflash-ampere.sh
+10 -10
View File
@@ -20,22 +20,22 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
-e OMP_NUM_THREADS=8 \ -e OMP_NUM_THREADS=8 \
-e VLLM_CPU_OMP_THREADS_BIND=8 \ -e VLLM_CPU_OMP_THREADS_BIND=8 \
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
-e VLLM_USE_V2_MODEL_RUNNER=0 \
-v ~/.cache/huggingface:/root/.cache/huggingface \ -v ~/.cache/huggingface:/root/.cache/huggingface \
vllm/vllm-openai:nightly "${MODEL}" \ vllm/vllm-openai:latest "${MODEL}" \
--served-model-name "frank-default-small" \ --served-model-name "frank-default-small" \
--max-num-seqs 2 \ --max-num-seqs 1 \
--max-model-len 32768 \ --max-model-len 212992 \
--tensor-parallel-size 1 \ --tensor-parallel-size 1 \
--enable-chunked-prefill \ --enable-chunked-prefill \
--max-num-batched-tokens 1536 \ --max-num-batched-tokens 4096 \
--performance-mode throughput \ --performance-mode throughput \
--enable-auto-tool-choice --tool-call-parser qwen3_xml \ --enable-auto-tool-choice --tool-call-parser qwen3_xml \
--reasoning-parser qwen3 \ --reasoning-parser qwen3 \
--trust-remote-code \ --trust-remote-code \
--gpu-memory-utilization 0.95 \ --gpu-memory-utilization 0.96 \
--cpu-offload-gb 20 \ --cpu-offload-gb 40 \
--cpu-offload-params "experts" \ --cpu-offload-params "experts" \
--default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' \
-O3 \
# TODO: uncomment and move to after --served-model-name to enable DFlash --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}'
# --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}' \