Add AWQ DFlash Ampere script and improved validation
- vllm-awq-qwen36-dflash-ampere.sh: AWQ-4bit Qwen3.6-35B-A3B with DFlash speculative decoding, optimized for Ampere hardware - Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit (AWQ Marlin) - Draft: z-lab/Qwen3.6-35B-A3B-DFlash - --max-model-len 32768 - --max-num-batched-tokens 1536 - --cpu-offload-gb 40 (experts) - --reasoning-parser qwen3 - --tool-call-parser qwen3_xml - Port: 34401 - Note: prefix caching disabled (Mamba cache align conflict) - scripts/validate-vllm.sh: Per-stream latency tests - Separate tests for 1, 2, 3, 4 streams - Different prompts per stream to avoid caching effects - Reports elapsed ms per stream count
This commit is contained in:
1 parent
659fba6a5d
commit
9227e1a8b9
3 files changed
+97
-53
No files matched your search
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
MODEL="cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit"
|
||||
DRAFT_MODEL="z-lab/Qwen3.6-35B-A3B-DFlash"
|
||||
PORT=8001
|
||||
|
||||
docker run --rm --name vllm-awq-qwen36-dflash \
|
||||
--gpus all \
|
||||
-p ${PORT}:8000 \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
vllm/vllm-openai:nightly \
|
||||
vllm serve "${MODEL}" \
|
||||
--served-model-name "frank-default-small" \
|
||||
--speculative-model "${DRAFT_MODEL}" \
|
||||
--speculative-num-steps 1 \
|
||||
--speculative-max-num-draft-tokens 4 \
|
||||
--max-num-seqs 2 \
|
||||
--max-model-len 262144 \
|
||||
--max-num-batched-tokens 1536 \
|
||||
--tensor-parallel-size 1 \
|
||||
--enable-chunked-prefill \
|
||||
--enable-prefix-caching \
|
||||
--enable-auto-tool-choice \
|
||||
--tool-call-parser qwen3_xml \
|
||||
--reasoning-parser qwen3 \
|
||||
--trust-remote-code \
|
||||
--gpu-memory-utilization 0.95 \
|
||||
--cpu-offload-gb 13 \
|
||||
--cpu-offload-params "experts" \
|
||||
--default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'
|
||||
Reference in new issue
Block a user