Lower --max-num-batched-tokens to 1536 to free VRAM for weights

2026-08-08 12:56:28 -07:00
parent 9c1f777423
commit 659fba6a5d
1 changed files with 1 additions and 1 deletions
@@ -20,7 +20,7 @@ exec docker run --rm --name vllm-qwen36 \
    --gpu-memory-utilization 0.95 \
    --max-num-seqs 4 \
    --max-model-len 32768 \
-    --max-num-batched-tokens 8192 \
+    --max-num-batched-tokens 1536 \
    --trust-remote-code \
    --cpu-offload-gb 40 \
    --cpu-offload-params "experts" \