#!/usr/bin/env bash
# vllm-awq-qwen36-dflash-ampere.sh
#
# Qwen3.6-35B-A3B AWQ 4bit with DFlash speculative decoding.
# Optimized for Ampere-class GPUs.
#
# Model:     cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit
# Draft:     z-lab/Qwen3.6-35B-A3B-DFlash
# Port:      34401
#
set -euo pipefail

MODEL="cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit"
PORT=34401

exec docker run --rm --name vllm-awq-qwen36-dflash-ampere \
    --gpus all \
    --privileged --ipc=host \
    -p "${PORT}:8000" \
    -e OMP_NUM_THREADS=8 \
    -e VLLM_CPU_OMP_THREADS_BIND=8 \
    -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    vllm/vllm-openai:nightly "${MODEL}" \
    --served-model-name "frank-default-small" \
    --max-num-seqs 2 \
    --max-model-len 32768 \
    --tensor-parallel-size 1 \
    --enable-chunked-prefill \
    --max-num-batched-tokens 1536 \
    --performance-mode throughput \
    --enable-auto-tool-choice --tool-call-parser qwen3_xml \
    --reasoning-parser qwen3 \
    --trust-remote-code \
    --gpu-memory-utilization 0.95 \
    --cpu-offload-gb 20 \
    --cpu-offload-params "experts" \
    --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'

# TODO: uncomment and move to after --served-model-name to enable DFlash
#    --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 12}' \