Frank
ced4c6f46e
- Model: Avesed/Qwen3.6-35B-A3B-INT8-W8A8 (was AWQ 4bit) - Image: vllm/vllm-openai:nightly (was :latest) - Spec method: mtp with 2 tokens (was dflash with 12) - Max model len: 200700 (was 32768) - Max num seqs: 3 (was 1) - Max batched tokens: 3072 (was 4096) - GPU mem util: 0.97 (was 0.96) - CPU offload: 30GB (was 40GB) - Add --enable-prefix-caching - Add --kv-cache-dtype auto
45 lines
1.4 KiB
Bash
Executable File
45 lines
1.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# vllm-awq-qwen36-dflash-ampere.sh
|
|
#
|
|
# Qwen3.6-35B-A3B AWQ 4bit with DFlash speculative decoding.
|
|
# Optimized for Ampere-class GPUs.
|
|
#
|
|
# Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit
|
|
# Draft: z-lab/Qwen3.6-35B-A3B-DFlash
|
|
# Port: 34401
|
|
#
|
|
set -euo pipefail
|
|
|
|
MODEL="Avesed/Qwen3.6-35B-A3B-INT8-W8A8"
|
|
PORT=34401
|
|
|
|
exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
|
|
--gpus all \
|
|
--privileged --ipc=host \
|
|
-p "${PORT}:8000" \
|
|
-e OMP_NUM_THREADS=8 \
|
|
-e VLLM_CPU_OMP_THREADS_BIND=8 \
|
|
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
|
-e VLLM_USE_V2_MODEL_RUNNER=0 \
|
|
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
|
vllm/vllm-openai:nightly "${MODEL}" \
|
|
--served-model-name "frank-default-small" \
|
|
--max-num-seqs 3 \
|
|
--max-model-len 200700 \
|
|
--tensor-parallel-size 1 \
|
|
--enable-chunked-prefill \
|
|
--enable-prefix-caching \
|
|
--max-num-batched-tokens 3072 \
|
|
--performance-mode throughput \
|
|
--enable-auto-tool-choice --tool-call-parser qwen3_xml \
|
|
--reasoning-parser qwen3 \
|
|
--trust-remote-code \
|
|
--gpu-memory-utilization 0.97 \
|
|
--kv-cache-dtype auto \
|
|
--cpu-offload-gb 30 \
|
|
--cpu-offload-params "experts" \
|
|
--default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}' \
|
|
-O3 \
|
|
--speculative-config '{"method": "mtp", "num_speculative_tokens": 2}'
|
|
|