Iteration 2: Remove --enforce-eager to enable torch.compile (+14.6% prompt speed: 5.95 -> 6.82 tok/s)

2026-08-07 23:07:51 -07:00
parent 1b90f421c6
commit 6a7025d01e
1 changed files with 1 additions and 3 deletions
@@ -5,7 +5,6 @@ set -euo pipefail
 # CPU expert offloading to fit MoE model on 24GB RTX 3090

 MODEL="Qwen/Qwen3.6-35B-A3B-FP8"
-DRAFT_MODEL="/home/dank/.cache/huggingface/models/Qwen3.6-35B-A3B-DFlash"
 PORT=8001
 HOST="0.0.0.0"

@@ -29,5 +28,4 @@ exec docker run --rm --name vllm-qwen36 \
    --cpu-offload-gb 30 \
    --cpu-offload-params "experts" \
    --enable-chunked-prefill \
-    --enable-prefix-caching \
-    --enforce-eager
+    --enable-prefix-caching