Frank
9227e1a8b9
- vllm-awq-qwen36-dflash-ampere.sh: AWQ-4bit Qwen3.6-35B-A3B with DFlash speculative decoding, optimized for Ampere hardware - Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit (AWQ Marlin) - Draft: z-lab/Qwen3.6-35B-A3B-DFlash - --max-model-len 32768 - --max-num-batched-tokens 1536 - --cpu-offload-gb 40 (experts) - --reasoning-parser qwen3 - --tool-call-parser qwen3_xml - Port: 34401 - Note: prefix caching disabled (Mamba cache align conflict) - scripts/validate-vllm.sh: Per-stream latency tests - Separate tests for 1, 2, 3, 4 streams - Different prompts per stream to avoid caching effects - Reports elapsed ms per stream count
32 lines
1.0 KiB
Bash
Executable File
32 lines
1.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
MODEL="cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit"
|
|
DRAFT_MODEL="z-lab/Qwen3.6-35B-A3B-DFlash"
|
|
PORT=8001
|
|
|
|
docker run --rm --name vllm-awq-qwen36-dflash \
|
|
--gpus all \
|
|
-p ${PORT}:8000 \
|
|
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
|
vllm/vllm-openai:nightly \
|
|
vllm serve "${MODEL}" \
|
|
--served-model-name "frank-default-small" \
|
|
--speculative-model "${DRAFT_MODEL}" \
|
|
--speculative-num-steps 1 \
|
|
--speculative-max-num-draft-tokens 4 \
|
|
--max-num-seqs 2 \
|
|
--max-model-len 262144 \
|
|
--max-num-batched-tokens 1536 \
|
|
--tensor-parallel-size 1 \
|
|
--enable-chunked-prefill \
|
|
--enable-prefix-caching \
|
|
--enable-auto-tool-choice \
|
|
--tool-call-parser qwen3_xml \
|
|
--reasoning-parser qwen3 \
|
|
--trust-remote-code \
|
|
--gpu-memory-utilization 0.95 \
|
|
--cpu-offload-gb 13 \
|
|
--cpu-offload-params "experts" \
|
|
--default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'
|