From ced4c6f46e50b836df48b37411fe2498717eb1f0 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Mon, 10 Aug 2026 03:23:09 -0700
Subject: [PATCH] sync: update vllm-awq-qwen36-dflash-ampere.sh from dankserver

- Model: Avesed/Qwen3.6-35B-A3B-INT8-W8A8 (was AWQ 4bit)
- Image: vllm/vllm-openai:nightly (was :latest)
- Spec method: mtp with 2 tokens (was dflash with 12)
- Max model len: 200700 (was 32768)
- Max num seqs: 3 (was 1)
- Max batched tokens: 3072 (was 4096)
- GPU mem util: 0.97 (was 0.96)
- CPU offload: 30GB (was 40GB)
- Add --enable-prefix-caching
- Add --kv-cache-dtype auto
---
 scripts/vllm-awq-qwen36-dflash-ampere.sh | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)
 mode change 100644 => 100755 scripts/vllm-awq-qwen36-dflash-ampere.sh

diff --git a/scripts/vllm-awq-qwen36-dflash-ampere.sh b/scripts/vllm-awq-qwen36-dflash-ampere.sh
old mode 100644
new mode 100755
index 9835a7a..0d77df0
--- a/scripts/vllm-awq-qwen36-dflash-ampere.sh
+++ b/scripts/vllm-awq-qwen36-dflash-ampere.sh
@@ -10,7 +10,7 @@
 #
 set -euo pipefail
 
-MODEL="cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit"
+MODEL="Avesed/Qwen3.6-35B-A3B-INT8-W8A8"
 PORT=34401
 
 exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
@@ -22,20 +22,23 @@ exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
     -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
     -e VLLM_USE_V2_MODEL_RUNNER=0 \
     -v ~/.cache/huggingface:/root/.cache/huggingface \
-    vllm/vllm-openai:latest "${MODEL}" \
+    vllm/vllm-openai:nightly "${MODEL}" \
     --served-model-name "frank-default-small" \
-    --max-num-seqs 1 \
-    --max-model-len 32768 \
+    --max-num-seqs 3 \
+    --max-model-len 200700 \
     --tensor-parallel-size 1 \
     --enable-chunked-prefill \
-    --max-num-batched-tokens 4096 \
+    --enable-prefix-caching \
+    --max-num-batched-tokens 3072 \
     --performance-mode throughput \
     --enable-auto-tool-choice --tool-call-parser qwen3_xml \
     --reasoning-parser qwen3 \
     --trust-remote-code \
-    --gpu-memory-utilization 0.96 \
-    --cpu-offload-gb 40 \
+    --gpu-memory-utilization 0.97 \
+    --kv-cache-dtype auto \
+    --cpu-offload-gb 30  \
     --cpu-offload-params "experts" \
     --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' \
     -O3 \
-    --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}'
+    --speculative-config '{"method": "mtp", "num_speculative_tokens": 2}'
+