#!/usr/bin/env bash
# Serve prism-ml/Ternary-Bonsai-2-9B-gguf (PQ2_0 ternary quant WITH the MTP head
# in the file) as frank-zealot on Dankclaw's RTX 3090 via llama.cpp (PrismML
# vendor build 10777, CUDA sm_86).
#
# 2026-10-18 (Frank): Bonsai-2 27B PQ2_0 → the 9B MTP export, then the same
# evening TWO 262,144-token conversations (--ctx-size 524288 --parallel 2) over
# an UNQUANTIZED f16 KV cache. The 9B bills 8,192 elements per token per cache
# (8 full-attention layers instead of the 27B's 16), so exact K and V costs
# 32,768 B/token — 16,384.00 MiB at the 524,288 pool, which is what a 24,123 MiB
# addressable card can carry beside the trunk. Verbosity 4 ships so the VRAM-fit
# forensics (KV layout, buffer sizes, graph capture) land in the journal.
#
# Architecture (engine ledger, not the marketing page): qwen35 hybrid, 32
# layers with qwen35.full_attention_interval = 4 ⇒ 8 full-attention + 24 Gated
# DeltaNet, 4 KV heads × 256 dim, native 262,144. The KV pool scales with the 8
# full layers (16,384.00 MiB of f16 at the 524,288 pool); the DeltaNet layers
# cost a fixed recurrent state per SLOT — ~50.25 MiB a cell, so 100.50 MiB for
# two. The runtime is a hard lock, not a preference: custom GGML type 142 plus
# prism.hadamard.* metadata, which only the vendor build registers (the b10776
# gate that moved the prebuilt 10754 → 10777 was MTP beside quantized V, which
# exact KV now makes moot — the type-142 lock alone keeps the build pinned).
# Vision comes from the publisher's Q8_0 projector (the BF16 one can crash the
# server on concurrent images, and this endpoint runs two slots), so --mmproj is
# what gives this endpoint eyes; MMPROJ_FILE="" is the documented text-only
# opt-out.
#
# Speculative decoding ships OFF, measured: the head loads and accepts well
# (0.798/0.617 per position, mean length 2.41) but the draft context bills its
# own f16 KV over the WHOLE pool — 4,096 B/position × 524,288 = 2,048 MiB plus
# buffers — and the trunk already projects 21,234 MiB with exact KV. Exact KV
# and MTP cannot share this card; frank-zealot.env carries the q8_0-era arm table
# and the two doors back to the drafter (one window, or KV back to q8_0 at
# --ubatch-size 1024 for 213.5 t/s).
#
# Sampling: the checkpoint carries the publisher's thinking recipe as GGUF
# metadata (general.sampling.* = temp 1.0 / top_p 0.95 / top_k 20 / min_p 0 /
# repeat_penalty 1.0 — confirmed in the engine's own /props defaults) and
# llama.cpp reads it; --presence-penalty is the one recipe value with no
# metadata key, so this script passes it. A request may still override per call
# (the fleet harness sends the same values).
#
# Tunables come from frank-zealot.env unless FRANK_ZEALOT_ENV_FILE overrides.
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_ZEALOT_ENV_FILE:-$SCRIPT_DIR/frank-zealot.env}"
if [[ -f "$ENV_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    source "$ENV_FILE"
    set +a
fi

# --- defaults (used when the env file does not set them) -------------------
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-prism-b10777/llama-server}"
: "${MODEL_FILE:=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.gguf}"
: "${ALIAS:=frank-zealot}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7771}"
# ctx-size is the POOL llama.cpp divides across slots: 524,288 / 2 = 262,144
# per conversation — every slot still at the model's trained window. The harness
# contextWindow MUST quote the PER-SLOT number — a longer prompt is answered
# with a 400, never truncated. Under exact KV the pool cannot stay at 786,432:
# 32,768 B/token × 786,432 = 24,576 MiB, more than the card's whole addressable
# 24,123 MiB before a single weight loads.
: "${CTX_SIZE:=524288}"
: "${PARALLEL:=2}"
# Compute-buffer geometry. Ubatch is a prefill knob (decode never moves).
# 2048 is the measured CUDA optimum (MiMo era, same arch/shape: +4.6 % prefill
# at 8.9k depth, decode flat, 616 MiB). The logical batch is VRAM- and
# throughput-neutral at a fixed ubatch. It ALSO sets the compute-buffer size,
# which is the term that decides whether an MTP draft context fits beside a KV
# pool: 1024 hands back 1,306 MiB and was the arm that ran the drafter at three
# q8_0 windows (frank-zealot.env carries the table).
: "${BATCH_SIZE:=4096}"
: "${UBATCH_SIZE:=2048}"
# Exact KV: f16 K + f16 V — no quantization error anywhere in the attention
# state. 32 KiB/token at this geometry (8 full-attention layers × 4 KV heads ×
# 256 dim × 2 B × 2 caches); 16,384.00 MiB at the 524,288 pool. That is the whole
# reason the slot count is 2: the same dtype at the old 786,432 pool is 24,576
# MiB and dies at cudaMalloc. q8_0 K + q8_0 V (17,408 B/token, 13,056.00 MiB at
# 786,432) is the documented fallback rung and funds three windows; CACHE_TYPE_V
# alone is the halfway door. Explicit either way so the contract is visible in
# the argv, and --flash-attn stays pinned because quantized V — the fallback —
# is illegal without it (no true fp8 KV dtype exists in this build).
: "${CACHE_TYPE_K:=f16}"
: "${CACHE_TYPE_V:=f16}"
# Speculative decoding defaults OFF, measured on this card and shape: the head
# IS in the checkpoint (qwen35.nextn_predict_layers = 1 — MTP is a property of
# the FILE, so the plain PQ2_0 sibling could not run it at any flag value), it
# loads, and it accepts well (mean accepted length 2.41) — but the draft context
# bills its own f16 KV over the WHOLE pool (4,096 B/position, 2,048 MiB at the
# 524,288 pool) and at ubatch 2048 that squeeze halves decode: 167.2 → 109.2 t/s
# on the q8_0 3-slot shape. The draft cache dtype IS a knob (--spec-draft-type-k
# / --spec-draft-type-v, both default f16 — verified in this build's --help), so
# an armed drafter can halve its own bill; there is no --spec-draft-model because
# the draft rides the main model's weights. Card gate: quantized V alongside MTP
# needs fork build >= prism-b10776, which is why LLAMA_BIN moved to the 10777
# prebuilt; exact KV removes that gate but NOT the type-142 loader lock.
# SPEC_TYPE=draft-mtp + UBATCH_SIZE=1024 is the documented MTP arm (213.5 t/s);
# SPEC_TYPE=none strips the whole set (flags move together).
: "${SPEC_TYPE:=none}"
: "${SPEC_DRAFT_N_MAX:=2}"
: "${SPEC_DRAFT_N_MIN:=0}"
: "${SPEC_DRAFT_P_MIN:=0}"
: "${SPEC_DRAFT_FILE:=}"
# No colon on purpose, same convention as MMPROJ_FILE: an explicitly empty
# PRESENCE_PENALTY is the documented opt-out and must NOT be replaced by the
# default (a colon treats null as unset and would silently re-add the flag).
: "${PRESENCE_PENALTY=1.0}"
: "${NGL:=99}"
# Log verbosity threshold (llama.cpp -lv). 3 is the engine default (info);
# 4 adds debug detail (KV layout, buffer sizes, graph capture) which is what
# the VRAM-fit forensics need — the exact-KV size and the draft-KV bill are
# only provable from this log. Levels above 4 flood the journal.
: "${VERBOSITY:=4}"
# /metrics needs the flag: llama.cpp returns 501 unless the server starts with
# --metrics, and Dankclaw's Prometheus scrapes every fleet endpoint's /metrics —
# a 501 reads as DOWN, which is how the critical LlmEndpointDown alert ended up
# firing for this service ever since the metric has history at all — the oldest
# retained sample of up{job="frank-zealot"} (2026-10-11) is already 0, so this
# target has never been scraped successfully, across every payload era. The
# counters cost nothing on the request path;
# METRICS=0 is the documented opt-out.
: "${METRICS:=1}"
# Vision projector. No colon on purpose: an explicitly empty value means
# "text-only, on purpose" and must NOT be replaced by the default path.
: "${MMPROJ_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf}"

# Fail loudly rather than half-loading: a missing file is a download that
# has not finished, and Restart=always would turn it into a crash loop that
# reads like a runtime fault.
for path in "$LLAMA_BIN" "$MODEL_FILE"; do
    [[ -e "$path" ]] || { echo "frank-zealot: missing $path" >&2; exit 1; }
done
EXTRA_ARGS=()
if [[ -n "$MMPROJ_FILE" ]]; then
    [[ -e "$MMPROJ_FILE" ]] || { echo "frank-zealot: missing $MMPROJ_FILE" >&2; exit 1; }
    EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE")
fi

# Prometheus scraping is opt-IN in llama.cpp: without --metrics the server
# answers 501 on /metrics, and the fleet scrape then reports the endpoint DOWN.
if [[ "$METRICS" = "1" ]]; then
    EXTRA_ARGS+=(--metrics)
fi

# Speculative flags move as a SET: they reach the argv ONLY when a drafter is
# named, and all of them together — a no-spec baseline must not carry half a
# spec pair. n-max 2 / n-min 0 / p-min 0 is the publisher's recipe ("Draft depth
# 2 works best"); the MTP head is IN the checkpoint (blk.32.nextn.*,
# qwen35.nextn_predict_layers = 1), so there is no --spec-draft-model here and
# the draft context rides the MAIN model's weights.
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
    EXTRA_ARGS+=(--spec-type "$SPEC_TYPE"
                 --spec-draft-n-max "$SPEC_DRAFT_N_MAX"
                 --spec-draft-n-min "$SPEC_DRAFT_N_MIN"
                 --spec-draft-p-min "$SPEC_DRAFT_P_MIN")
    if [[ -n "$SPEC_DRAFT_FILE" ]]; then
        [[ -e "$SPEC_DRAFT_FILE" ]] || { echo "frank-zealot: missing $SPEC_DRAFT_FILE" >&2; exit 1; }
        EXTRA_ARGS+=(--spec-draft-model "$SPEC_DRAFT_FILE")
    fi
fi

# The card's sampler recipe lives partly in the checkpoint: general.sampling.*
# (temp 1.0 / top_p 0.95 / top_k 20 / min_p 0 / repeat_penalty 1.0) is read by
# llama.cpp itself, and --repeat-last-n already defaults to 64. Presence penalty
# is the ONE value with no GGUF metadata key — the card says so explicitly — so
# it is the one knob that must reach the argv or the recipe is silently
# incomplete. PRESENCE_PENALTY="" is the documented opt-out.
if [[ -n "$PRESENCE_PENALTY" ]]; then
    EXTRA_ARGS+=(--presence-penalty "$PRESENCE_PENALTY")
fi

# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
# open after boot; make sure it exists before the server needs it.
nvidia-modprobe -u -c=0 2>/dev/null || true

# Flash attention stays pinned on. Exact f16 KV runs fine with or without it,
# but the documented fallback rung (quantized V) is ILLEGAL without it, and the
# build carries GGML_CUDA_FA_ALL_QUANTS=ON (since 2026-09-17) so that rung is a
# real path — dropping the flag here would leave the retreat route silently
# falling back to something the env file does not describe.
ARGS=(
    --model "$MODEL_FILE"
    --alias "$ALIAS"
    --host "$BIND_ADDR"
    --port "$LISTEN_PORT"
    --ctx-size "$CTX_SIZE"
    --parallel "$PARALLEL"
    --batch-size "$BATCH_SIZE"
    --ubatch-size "$UBATCH_SIZE"
    --n-gpu-layers "$NGL"
    --flash-attn on
    --cache-type-k "$CACHE_TYPE_K"
    --cache-type-v "$CACHE_TYPE_V"
    --log-verbosity "$VERBOSITY"
    --jinja
    --reasoning auto
    --reasoning-format deepseek
)
exec "$LLAMA_BIN" "${ARGS[@]}" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}