42 lines
1.4 KiB
Bash
42 lines
1.4 KiB
Bash
#!/usr/bin/env bash
|
|
# vllm-awq-qwen36-dflash-ampere.sh
|
|
#
|
|
# Qwen3.6-35B-A3B AWQ 4bit with DFlash speculative decoding.
|
|
# Optimized for Ampere-class GPUs.
|
|
#
|
|
# Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit
|
|
# Draft: z-lab/Qwen3.6-35B-A3B-DFlash
|
|
# Port: 34401
|
|
#
|
|
set -euo pipefail
|
|
|
|
MODEL="cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit"
|
|
PORT=34401
|
|
|
|
exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
|
|
--gpus all \
|
|
--privileged --ipc=host \
|
|
-p "${PORT}:8000" \
|
|
-e OMP_NUM_THREADS=8 \
|
|
-e VLLM_CPU_OMP_THREADS_BIND=8 \
|
|
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
|
-e VLLM_USE_V2_MODEL_RUNNER=0 \
|
|
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
|
vllm/vllm-openai:latest "${MODEL}" \
|
|
--served-model-name "frank-default-small" \
|
|
--max-num-seqs 1 \
|
|
--max-model-len 212992 \
|
|
--tensor-parallel-size 1 \
|
|
--enable-chunked-prefill \
|
|
--max-num-batched-tokens 4096 \
|
|
--performance-mode throughput \
|
|
--enable-auto-tool-choice --tool-call-parser qwen3_xml \
|
|
--reasoning-parser qwen3 \
|
|
--trust-remote-code \
|
|
--gpu-memory-utilization 0.96 \
|
|
--cpu-offload-gb 40 \
|
|
--cpu-offload-params "experts" \
|
|
--default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' \
|
|
-O3 \
|
|
--speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}'
|