#!/usr/bin/env bash
# Serve ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp as frank-small on Dankclaw's
# RTX 3090 via mainline llama.cpp.
#
# Topology: the whole model lives in VRAM (-ngl 999) -- 11.29 GiB of IQ3_S-mtp
# weights, 0.87 GiB of BF16 vision projector and ~8 GiB of f16 KV inside the
# card's 24 GiB. The host CPU only tokenizes and samples here; this box's CPU
# belongs to the halogen engine that serves frank-large.
#
# MTP: the `-mtp` build carries the NextN block (blk.64.nextn.*), and
# `--spec-type draft-mtp` makes llama.cpp load it and run the draft context on
# the SAME weights. There is no --model-draft here on purpose: a separate draft
# path would load a second 12 GB copy of a 12 GB model. Draft depth is
# llama.cpp's default (--spec-draft-n-max 3).
#
# Vision: --mmproj loads the BF16 projector, so image content parts work. The
# fleet's image path moved here from frank-large on 2026-09-16, which is why
# halogen no longer sets HALOGEN_VISION_TOWER.
#
# Sampling: greedy by default (the fleet contract). Clients that send their own
# temperature get their own values, and the harness overrides per model from
# models.yml. --jinja is pinned even though this build defaults it on: the
# model's own template supplies reasoning_effort, preserve_thinking and the
# tool-call grammar, and a future build that flips that default must not
# silently take tools away.
#
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides it.
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
if [[ -f "$ENV_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    source "$ENV_FILE"
    set +a
fi

# --- defaults (used when the env file does not set them) -------------------
: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server}"
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp}"
: "${MODEL_FILE:=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
: "${MMPROJ_PATH:=$MODEL_DIR/mmproj-Qwen3.8-27B-BF16.gguf}"
: "${ALIAS:=frank-small}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7771}"
: "${CTX_SIZE:=131072}"
: "${PARALLEL:=1}"
: "${CACHE_TYPE_K:=f16}"
: "${CACHE_TYPE_V:=f16}"
: "${BATCH_SIZE:=2048}"
: "${UBATCH_SIZE:=512}"
: "${NUM_THREADS:=8}"
: "${THREAD_BATCH:=8}"
: "${NGL:=999}"
: "${FLASH_ATTN:=on}"
: "${SPEC_TYPE:=draft-mtp}"
: "${TEMPERATURE:=0.0}"
: "${REASONING_FORMAT:=deepseek}"
: "${REASONING_BUDGET:=-1}"

# Fail loudly rather than half-loading a model into VRAM: a missing file here
# is a download that has not finished, and the unit's restart policy would
# otherwise turn that into a crash loop that reads like a runtime fault.
for path in "$LLAMA_BIN" "$MODEL_DIR/$MODEL_FILE" "$MMPROJ_PATH"; do
    [[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
done

# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
# open after boot; make sure it exists before the server needs it.
nvidia-modprobe -u -c=0 2>/dev/null || true

MMPROJ_ARGS=()
if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
    MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
fi

SPEC_ARGS=()
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
    SPEC_ARGS=(--spec-type "$SPEC_TYPE")
fi

exec "$LLAMA_BIN" \
    --model "$MODEL_DIR/$MODEL_FILE" \
    "${MMPROJ_ARGS[@]}" \
    --alias "$ALIAS" \
    --host "$BIND_ADDR" \
    --port "$LISTEN_PORT" \
    --gpu-layers "$NGL" \
    --flash-attn "$FLASH_ATTN" \
    --ctx-size "$CTX_SIZE" \
    --parallel "$PARALLEL" \
    --batch-size "$BATCH_SIZE" \
    --ubatch-size "$UBATCH_SIZE" \
    --threads "$NUM_THREADS" \
    --threads-batch "$THREAD_BATCH" \
    --cache-type-k "$CACHE_TYPE_K" \
    --cache-type-v "$CACHE_TYPE_V" \
    "${SPEC_ARGS[@]}" \
    --jinja \
    --metrics \
    --temp "$TEMPERATURE" \
    --reasoning-format "$REASONING_FORMAT" \
    --reasoning-budget "$REASONING_BUDGET"