#!/usr/bin/env bash
# Serve HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M) as
# frank-small on Dankclaw's RTX 3090 via mainline llama.cpp.
#
# Topology: the whole model lives in VRAM (-ngl 999) -- 6.87 GiB of Q4_K_M
# weights plus the BF16 vision projector and q8_0 KV inside the card's
# 24 GiB. The host CPU only tokenizes and samples here; this box's CPU
# belongs to the halogen engine that serves frank-large.
#
# MTP: this model's Multi-Token Prediction head ships as a SEPARATE file
# (mtp-gemma-4-12B-it.gguf, arch gemma4-assistant, 4 nextn layers). It is
# passed via --model-draft with --spec-type draft-mtp: the draft context
# loads the assistant model. The same --spec-type value chains the
# draftless ngram-mod matcher, which attaches unconditionally; two
# context-backed drafters cannot share one process, so this chain is the
# only shape that carries both.
#
# Context: CTX_SIZE is set explicitly to PARALLEL x 262144 so every slot
# gets the model's FULL window; --fit (the CTX_SIZE-unset branch below)
# maximises the TOTAL pool instead, which would divide a smaller window
# across slots. The 4-slot layout is measured to fit the card (see the
# SIZING note in frank-small.env); more slots would fail the KV allocation.
#
# Vision + audio: --mmproj loads the BF16 projector, which also carries
# an audio encoder — input_audio content parts transcribe verbatim
# (verified 2026-09-17). The fleet's image path lives here, which is why
# halogen no longer sets HALOGEN_VISION_TOWER.
#
# Sampling: greedy by default (the fleet contract). Clients that send their own
# temperature get their own values, and the harness overrides per model from
# models.yml. --jinja is pinned even though this build defaults it on: the
# model's own template supplies reasoning_effort, preserve_thinking and the
# tool-call grammar, and a future build that flips that default must not
# silently take tools away.
#
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides it.
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
if [[ -f "$ENV_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    source "$ENV_FILE"
    set +a
fi

# --- defaults (used when the env file does not set them) -------------------
: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server}"
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp}"
: "${MODEL_FILE:=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
: "${MMPROJ_PATH:=$MODEL_DIR/mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf}"
: "${DRAFT_FILE:=}"
: "${ALIAS:=frank-small}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7771}"
: "${CTX_SIZE:=}"
: "${PARALLEL:=1}"
: "${CACHE_TYPE_K:=f16}"
: "${CACHE_TYPE_V:=f16}"
: "${CACHE_RAM:=8192}"
: "${BATCH_SIZE:=2048}"
: "${UBATCH_SIZE:=512}"
# Decode-shape knobs: the MTP drafter's per-round proposal length (build
# default 3; the Gemma4 sidecar carries 4 nextn layers) and the engine's
# busy-wait level (build default 50).
: "${SPEC_DRAFT_N_MAX:=3}"
: "${POLL_LEVEL:=50}"
: "${NUM_THREADS:=8}"
: "${THREAD_BATCH:=8}"
: "${NGL:=999}"
: "${FLASH_ATTN:=on}"
: "${SPEC_TYPE:=draft-mtp}"
: "${TEMPERATURE:=0.0}"
: "${REASONING_FORMAT:=deepseek}"
: "${REASONING_BUDGET:=-1}"

# Fail loudly rather than half-loading a model into VRAM: a missing file here
# is a download that has not finished, and the unit's restart policy would
# otherwise turn that into a crash loop that reads like a runtime fault.
for path in "$LLAMA_BIN" "$MODEL_DIR/$MODEL_FILE" "$MMPROJ_PATH"; do
    [[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
done
if [[ -n "${DRAFT_FILE:-}" ]]; then
    [[ -e "$MODEL_DIR/$DRAFT_FILE" ]] || { echo "frank-small: missing $MODEL_DIR/$DRAFT_FILE" >&2; exit 1; }
fi

# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
# open after boot; make sure it exists before the server needs it.
nvidia-modprobe -u -c=0 2>/dev/null || true

MMPROJ_ARGS=()
if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
    MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
fi

SPEC_ARGS=()
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
    SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
fi

DRAFT_ARGS=()
if [[ -n "${DRAFT_FILE:-}" ]]; then
    DRAFT_ARGS=(--model-draft "$MODEL_DIR/$DRAFT_FILE")
fi

# CTX_SIZE set -> pinned context. Unset -> --fit sizes the KV pool against
# whatever the weights, projector, draft context and compute buffers leave
# on the card; the engine prints the chosen value at startup.
if [[ -n "${CTX_SIZE:-}" ]]; then
    CTX_ARGS=(--ctx-size "$CTX_SIZE")
else
    CTX_ARGS=(--fit on)
fi

exec "$LLAMA_BIN" \
    --model "$MODEL_DIR/$MODEL_FILE" \
    "${MMPROJ_ARGS[@]}" \
    "${DRAFT_ARGS[@]}" \
    --alias "$ALIAS" \
    --host "$BIND_ADDR" \
    --port "$LISTEN_PORT" \
    --gpu-layers "$NGL" \
    --flash-attn "$FLASH_ATTN" \
    "${CTX_ARGS[@]}" \
    --parallel "$PARALLEL" \
    --batch-size "$BATCH_SIZE" \
    --ubatch-size "$UBATCH_SIZE" \
    --threads "$NUM_THREADS" \
    --threads-batch "$THREAD_BATCH" \
    --poll "$POLL_LEVEL" \
    --cache-type-k "$CACHE_TYPE_K" \
    --cache-type-v "$CACHE_TYPE_V" \
    --cache-ram "$CACHE_RAM" \
    "${SPEC_ARGS[@]}" \
    --jinja \
    --metrics \
    --temp "$TEMPERATURE" \
    --reasoning-format "$REASONING_FORMAT" \
    --reasoning-budget "$REASONING_BUDGET"