#!/usr/bin/env bash
# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q5_K_M) as frank-small on
# Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86).
#
# 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound
# safetensors to the llama.cpp GGUF path, because llama.cpp is the only
# runtime on this host that can serve the MiMo qwen35 hybrid with
# ASYMMETRIC KV: --cache-type-k f16 (unquantized keys) + --cache-type-v
# q8_0 (quantized values), which is Frank's explicit KV contract for this
# endpoint. SGLang's --kv-cache-dtype only moves K and V together.
#
# Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated
# DeltaNet, native 262144 window. KV lives only in the 8 attention layers;
# the DeltaNet state is per-slot fixed size. Vision comes from the repo's
# separate projector (the Q5_K_M GGUF carries no `v.*` tensors at all), so
# --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the documented
# text-only opt-out.
#
# Sampling: llama.cpp takes it per request (the fleet harness sends its
# recipe from models.yml); greedy is the harness default.
#
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides.
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
if [[ -f "$ENV_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    source "$ENV_FILE"
    set +a
fi

# --- defaults (used when the env file does not set them) -------------------
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
: "${MODEL_FILE:=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}"
: "${ALIAS:=frank-small}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7771}"
# ctx-size is the POOL llama.cpp divides across slots: 540672 / 3 gives
# 180224 per slot — three independent conversations (the native 262144 no
# longer fits three times on a 24 GiB card once the vision tower is loaded;
# frank-small.env carries the two measurements that set this number).
: "${CTX_SIZE:=540672}"
: "${PARALLEL:=3}"
: "${CACHE_TYPE_K:=f16}"
: "${CACHE_TYPE_V:=q8_0}"
: "${NGL:=99}"
# Vision projector. No colon on purpose: an explicitly empty value means
# "text-only, on purpose" and must NOT be replaced by the default path.
: "${MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}"

# Fail loudly rather than half-loading: a missing file is a download that
# has not finished, and Restart=always would turn it into a crash loop that
# reads like a runtime fault.
for path in "$LLAMA_BIN" "$MODEL_FILE"; do
    [[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
done
EXTRA_ARGS=()
if [[ -n "$MMPROJ_FILE" ]]; then
    [[ -e "$MMPROJ_FILE" ]] || { echo "frank-small: missing $MMPROJ_FILE" >&2; exit 1; }
    EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE")
fi

# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
# open after boot; make sure it exists before the server needs it.
nvidia-modprobe -u -c=0 2>/dev/null || true

# q8_0 on V REQUIRES flash attention (the quantized-value kernels only exist
# on the FA path); an FA-less build silently falls back and the KV contract
# in frank-small.env becomes a lie, so pin the flag pair here.
ARGS=(
    --model "$MODEL_FILE"
    --alias "$ALIAS"
    --host "$BIND_ADDR"
    --port "$LISTEN_PORT"
    --ctx-size "$CTX_SIZE"
    --parallel "$PARALLEL"
    --n-gpu-layers "$NGL"
    --flash-attn on
    --cache-type-k "$CACHE_TYPE_K"
    --cache-type-v "$CACHE_TYPE_V"
    --jinja
    --reasoning auto
    --reasoning-format deepseek
)
exec "$LLAMA_BIN" "${ARGS[@]}" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}