Frank
9227e1a8b9
- vllm-awq-qwen36-dflash-ampere.sh: AWQ-4bit Qwen3.6-35B-A3B with DFlash speculative decoding, optimized for Ampere hardware - Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit (AWQ Marlin) - Draft: z-lab/Qwen3.6-35B-A3B-DFlash - --max-model-len 32768 - --max-num-batched-tokens 1536 - --cpu-offload-gb 40 (experts) - --reasoning-parser qwen3 - --tool-call-parser qwen3_xml - Port: 34401 - Note: prefix caching disabled (Mamba cache align conflict) - scripts/validate-vllm.sh: Per-stream latency tests - Separate tests for 1, 2, 3, 4 streams - Different prompts per stream to avoid caching effects - Reports elapsed ms per stream count
40 lines
1.4 KiB
Bash
Executable File
40 lines
1.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# vllm-awq-qwen36-dflash-ampere.sh
|
|
#
|
|
# Qwen3.6-35B-A3B AWQ 4bit with DFlash speculative decoding.
|
|
# Optimized for Ampere-class GPUs.
|
|
#
|
|
# Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit
|
|
# Draft: z-lab/Qwen3.6-35B-A3B-DFlash
|
|
# Port: 34401
|
|
#
|
|
set -euo pipefail
|
|
|
|
MODEL="cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit"
|
|
PORT=34401
|
|
|
|
exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
|
|
--gpus all \
|
|
--privileged --ipc=host \
|
|
-p "${PORT}:8000" \
|
|
-e OMP_NUM_THREADS=8 \
|
|
-e VLLM_CPU_OMP_THREADS_BIND=8 \
|
|
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
|
vllm/vllm-openai:nightly "${MODEL}" \
|
|
--served-model-name "frank-default-small" \
|
|
--speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 4}' \
|
|
--max-num-seqs 2 \
|
|
--max-model-len 262144 \
|
|
--tensor-parallel-size 1 \
|
|
--enable-chunked-prefill \
|
|
--max-num-batched-tokens 1536 \
|
|
--performance-mode throughput \
|
|
# --enable-prefix-caching # Disabled: Mamba cache align mode conflicts with prefix caching \\
|
|
--enable-auto-tool-choice --tool-call-parser qwen3_xml \
|
|
--reasoning-parser qwen3 \
|
|
--trust-remote-code \
|
|
--gpu-memory-utilization 0.95 \
|
|
--cpu-offload-gb 13 \
|
|
--cpu-offload-params "experts" \
|
|
--default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'
|