Add VLLM_USE_V2_MODEL_RUNNER=0 to skip V2 model runner MARLIN repacking OOM

scripts/vllm-awq-qwen36-dflash-ampere.sh

@@ -20,22 +20,22 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
     -e OMP_NUM_THREADS=8 \
     -e VLLM_CPU_OMP_THREADS_BIND=8 \
     -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+    -e VLLM_USE_V2_MODEL_RUNNER=0 \
     -v ~/.cache/huggingface:/root/.cache/huggingface \
-    vllm/vllm-openai:nightly "${MODEL}" \
+    vllm/vllm-openai:latest "${MODEL}" \
     --served-model-name "frank-default-small" \
-    --max-num-seqs 2 \
-    --max-model-len 32768 \
+    --max-num-seqs 1 \
+    --max-model-len 212992 \
     --tensor-parallel-size 1 \
     --enable-chunked-prefill \
-    --max-num-batched-tokens 1536 \
+    --max-num-batched-tokens 4096 \
     --performance-mode throughput \
     --enable-auto-tool-choice --tool-call-parser qwen3_xml \
     --reasoning-parser qwen3 \
     --trust-remote-code \
-    --gpu-memory-utilization 0.95 \
-    --cpu-offload-gb 20 \
+    --gpu-memory-utilization 0.96 \
+    --cpu-offload-gb 40 \
     --cpu-offload-params "experts" \
-    --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'
-
-# TODO: uncomment and move to after --served-model-name to enable DFlash
-#    --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}' \
+    --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' \
+    -O3 \
+    --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}'