#!/usr/bin/env bash
# Serve HauhauCS/Gemma4-26B-A4B-QAT-Uncensored (QAT **Q4_K_M**) as
# frank-secondary on Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86).
#
# 2026-10-07 (Frank): Gemma4-26B-A4B swap with MTP OFF and ONE 200,704-token
# conversation (--parallel 1) — the 24 GiB card cannot fund a second slot at
# this weight, and the MTP assistant's KV (built against the target's full
# window, ~1.9 GiB measured) plus the f16-K raise do not coexist on it.
# Verbosity 4 ships so the VRAM-fit forensics land in the journal.
#
# Architecture: gemma4_26b_a4b — 30-layer MoE (active ~4B), hybrid attention:
# 25 sliding-window layers (fixed 1024-token ring each) + 5 full-attention
# layers (5, 11, 17, 23, 29), 8 KV heads x 256 dim, n_ctx_train 262,144. KV
# therefore scales ONLY with the 5 full layers: ~30.7 KiB/token at f16 K +
# q8_0 V (~6.2 GiB at 200,704). attention_k_eq_v is declared by the file and
# honoured by the build (bill K-side accounting; the measured boot is the
# referee: 21,890 used / 2,235 free at q8_0/q8_0, MTP off).
# Vision comes from the projector file (the GGUF carries no vision
# tensors; the publisher ships its own BF16 mmproj in the same repo), so
# --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the
# documented text-only opt-out.
#
# Sampling: llama.cpp takes it per request (the fleet harness sends its
# recipe from models.yml); greedy is the harness default.
#
# Tunables come from frank-secondary.env unless FRANK_SECONDARY_ENV_FILE overrides.
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_SECONDARY_ENV_FILE:-$SCRIPT_DIR/frank-secondary.env}"
if [[ -f "$ENV_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    source "$ENV_FILE"
    set +a
fi

# --- defaults (used when the env file does not set them) -------------------
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
: "${MODEL_FILE:=/home/dank/models/Gemma4-26B-A4B/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}"
: "${ALIAS:=frank-secondary}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7771}"
# ctx-size is the POOL llama.cpp divides across slots; with --parallel 1 the
# pool IS the conversation window. 200,704 is inside the model's trained
# 262,144; the harness contextWindow MUST quote this number — a longer prompt
# is answered with a 400, never truncated.
: "${CTX_SIZE:=200704}"
: "${PARALLEL:=1}"
# Compute-buffer geometry. Ubatch is a prefill knob (decode never moves).
# 2026-10-07 (Frank): 1024 — cut from 2048 to fund the f16-K raise. The
# logical batch is VRAM- and throughput-neutral at a fixed ubatch.
: "${BATCH_SIZE:=4096}"
: "${UBATCH_SIZE:=1024}"
# KV at FP8 class = q8_0 on llama.cpp (1.0625 B/elem => 8.5 effective bits;
# there is no true fp8 KV dtype). Explicit either way so the contract is
# visible in the argv.
: "${CACHE_TYPE_K:=q8_0}"
: "${CACHE_TYPE_V:=q8_0}"
# Speculative decoding is OFF by default since the Gemma4 swap (Frank,
# 2026-10-07): Gemma4's MTP is a SEPARATE assistant file (arch
# gemma4-assistant), not an in-file head, and its KV is built against the
# target's full window — ~1.9 GiB measured, which the f16-K raise cannot
# afford. SPEC_TYPE=none strips BOTH speculative flags (flags move as a
# set); naming a drafter requires SPEC_DRAFT_FILE alongside SPEC_TYPE.
: "${SPEC_TYPE:=none}"
: "${SPEC_DRAFT_N_MAX:=3}"
: "${SPEC_DRAFT_FILE:=}"
: "${NGL:=99}"
# Log verbosity threshold (llama.cpp -lv). 3 is the engine default (info);
# 4 adds debug detail (KV layout, buffer sizes, graph capture) which is what
# the VRAM-fit forensics need. Levels above 4 flood the journal.
: "${VERBOSITY:=3}"
# Vision projector. No colon on purpose: an explicitly empty value means
# "text-only, on purpose" and must NOT be replaced by the default path.
: "${MMPROJ_FILE=/home/dank/models/Gemma4-26B-A4B/mmproj-Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf}"

# Fail loudly rather than half-loading: a missing file is a download that
# has not finished, and Restart=always would turn it into a crash loop that
# reads like a runtime fault.
for path in "$LLAMA_BIN" "$MODEL_FILE"; do
    [[ -e "$path" ]] || { echo "frank-secondary: missing $path" >&2; exit 1; }
done
EXTRA_ARGS=()
if [[ -n "$MMPROJ_FILE" ]]; then
    [[ -e "$MMPROJ_FILE" ]] || { echo "frank-secondary: missing $MMPROJ_FILE" >&2; exit 1; }
    EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE")
fi

# Speculative flags move as a set: they reach the argv ONLY when a drafter
# is named. SPEC_TYPE=none (the shipped contract) strips both — a no-spec
# baseline must not carry half a spec pair.
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
    EXTRA_ARGS+=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
if [[ -n "$SPEC_DRAFT_FILE" ]]; then
    [[ -e "$SPEC_DRAFT_FILE" ]] || { echo "frank-secondary: missing $SPEC_DRAFT_FILE" >&2; exit 1; }
    EXTRA_ARGS+=(--spec-draft-model "$SPEC_DRAFT_FILE")
fi
fi

# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
# open after boot; make sure it exists before the server needs it.
nvidia-modprobe -u -c=0 2>/dev/null || true

# Flash attention stays pinned on: it is REQUIRED for quantized V (q8_0),
# and the build carries GGML_CUDA_FA_ALL_QUANTS=ON (since 2026-09-17), so
# the q8_0 KV contract in frank-secondary.env is a real path — without this
# flag or that build option the engine silently falls back and the
# contract becomes a lie.
ARGS=(
    --model "$MODEL_FILE"
    --alias "$ALIAS"
    --host "$BIND_ADDR"
    --port "$LISTEN_PORT"
    --ctx-size "$CTX_SIZE"
    --parallel "$PARALLEL"
    --batch-size "$BATCH_SIZE"
    --ubatch-size "$UBATCH_SIZE"
    --n-gpu-layers "$NGL"
    --flash-attn on
    --cache-type-k "$CACHE_TYPE_K"
    --cache-type-v "$CACHE_TYPE_V"
    --log-verbosity "$VERBOSITY"
    --jinja
    --reasoning auto
    --reasoning-format deepseek
)
exec "$LLAMA_BIN" "${ARGS[@]}" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}