#!/usr/bin/env bash
set -euo pipefail

# Qwen3.6-35B-A3B-FP8 vLLM serving script for dankserver
# CPU expert offloading to fit MoE model on 24GB RTX 3090

MODEL="Qwen/Qwen3.6-35B-A3B-FP8"
PORT=8001
HOST="0.0.0.0"

exec docker run --rm --name vllm-qwen36 \
    --gpus all \
    --privileged \
    --ipc=host \
    -p "${PORT}:${PORT}" \
    -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
    -v /home/dank/.cache/huggingface:/root/.cache/huggingface \
    docker.io/vllm/vllm-openai:nightly \
    "${MODEL}" \
    --port "${PORT}" \
    --host "${HOST}" \
    --kv-cache-dtype fp8 \
    --gpu-memory-utilization 0.95 \
    --max-num-seqs 3 \
    --max-model-len 16384 \
    --max-num-batched-tokens 4096 \
    --trust-remote-code \
    --cpu-offload-gb 30 \
    --cpu-offload-params "experts" \
    --enable-chunked-prefill \
    --enable-prefix-caching