From 6a7025d01e1a00eee175fa2649d1576be31961df Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 7 Aug 2026 23:07:51 -0700 Subject: [PATCH] Iteration 2: Remove --enforce-eager to enable torch.compile (+14.6% prompt speed: 5.95 -> 6.82 tok/s) --- vllm-qwen36.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vllm-qwen36.sh b/vllm-qwen36.sh index b90e45a..ee73d63 100755 --- a/vllm-qwen36.sh +++ b/vllm-qwen36.sh @@ -5,7 +5,6 @@ set -euo pipefail # CPU expert offloading to fit MoE model on 24GB RTX 3090 MODEL="Qwen/Qwen3.6-35B-A3B-FP8" -DRAFT_MODEL="/home/dank/.cache/huggingface/models/Qwen3.6-35B-A3B-DFlash" PORT=8001 HOST="0.0.0.0" @@ -29,5 +28,4 @@ exec docker run --rm --name vllm-qwen36 \ --cpu-offload-gb 30 \ --cpu-offload-params "experts" \ --enable-chunked-prefill \ - --enable-prefix-caching \ - --enforce-eager + --enable-prefix-caching