# Tunables for frank-default-small (vLLM, dankclaw, Strix Halo gfx1151).
#
# Runtime: vLLM (ROCm) in /home/dank/vllm-venv serving
# Avesed/Qwen3.6-35B-A3B-INT8-W8A8 (compressed-tensors INT8 W8A8, MoE 35B
# total / 3B active, Qwen3.5-MoE hybrid: 30 of 40 layers are GatedDeltaNet
# linear attention). Alias frank-default-small, port 52625.
#
# The ROCm vLLM install was created with:
#   uv venv --python /usr/bin/python3.12 /home/dank/vllm-venv
#   uv pip install --python /home/dank/vllm-venv vllm --torch-backend=auto
# The --torch-backend=auto resolver pulls the AMD ROCm wheels (torch +
# amd-torch-device-gfx1151 + vllm rocm variant) from the AMD index.

VLLM_BIN=/home/dank/vllm-venv/bin/vllm
MODEL=Avesed/Qwen3.6-35B-A3B-INT8-W8A8
ALIAS=frank-default-small
BIND_ADDR=0.0.0.0
LISTEN_PORT=52625

# Full native context. The KV cache stays unquantized (auto = model dtype,
# bf16). Worst-case KV at 262144 x 4 sequences is ~10.7 GiB; weights are
# ~35 GiB; total fits the Strix Halo unified pool with room to spare.
MAX_MODEL_LEN=262144
KV_CACHE_DTYPE=auto

# Concurrency and batching (user request).
MAX_NUM_SEQS=4
MAX_NUM_BATCHED_TOKENS=16384

# vLLM has no separate ubatch knob. The scheduler issues one batch per step
# with up to max-num-batched-tokens tokens; the KV cache is preallocated at
# max-model-len. Set gpu-memory-utilization below 1.0 so the unified pool
# keeps headroom for the host.
GPU_MEMORY_UTILIZATION=0.92

# ngram speculative decoding (vLLM ngram proposer).
# prompt_lookup_max = ngram window; num_speculative_tokens = draft depth.
NGRAM_LOOKUP_MAX=5
NGRAM_LOOKUP_MIN=1
NUM_SPEC_TOKENS=5