#!/usr/bin/env bash
# serve-frank-dragoon.sh — frank-dragoon endpoint: halogen-flash-server 0.16.2
# (closed-source engine, image ghcr.io/peonist-ai/halogen-flash-server,
# digest-pinned in frank-dragoon.env) serving the Quat3rnion abliterated
# Swift1.5 finetune of peonist's Qwen3.8-Flash-Next v2 checkpoint on
# Dankclaw's Strix Halo iGPU, host :7770. 2026-10-06 cutover from the
# halogen 27B-dense shape — revert the swap commit's trio to roll back
# (the 27B payload stays on disk).
#
# The engine is driven by `podman run` DIRECTLY (the fleet's frank-archon
# pattern): the image starts engine + OpenAI frontend in one container.
# The frontend listens on 8731 inside (HALOGEN_API_PORT default) and is
# published as host 7770; the engine's own token port (8730, no auth)
# stays unpublished on the container's loopback. HALOGEN_CHECKPOINT
# selects the file inside the /models mount; the n-gram lookup table
# (HALOGEN_NGRAM_TABLE, the card sets it explicitly though the documented
# fallback is "beside the checkpoint") and the vision tower
# (HALOGEN_VISION_TOWER) ride in the same directory.
#
# Gates mirror the deployer EXACTLY (the 2026-10-01 crash-loop lesson —
# serve gate and deploy gate disagreeing = green deploy, red unit):
# checkpoint/n-gram/vision byte-exact, tokenizer directory present with
# every named file a FLAT REGULAR FILE (a HF cache snapshot's symlinks
# into blobs/ dangle inside the mount), GPU device nodes present.
#
# Why containers instead of a bare binary: the payload is .hgn and only
# the flash-server line reads it; it ships as a container; the root unit
# keeps LimitMEMLOCK=infinity for parity and to outlive the login session,
# exactly like frank-archon's.
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
if [ ! -f "$ENV_FILE" ]; then
    echo "frank-dragoon: env file not found: $ENV_FILE" >&2
    exit 1
fi
set -a
# shellcheck disable=SC1090
source "$ENV_FILE"
set +a

: "${FRANK_DRAGOON_IMAGE:?FRANK_DRAGOON_IMAGE must name the digest-pinned image}"
: "${FRANK_DRAGOON_CONTAINER:?FRANK_DRAGOON_CONTAINER must name the container}"
: "${FRANK_DRAGOON_PUBLISH:?FRANK_DRAGOON_PUBLISH must be host:container}"
: "${FRANK_DRAGOON_MODEL_DIR:?FRANK_DRAGOON_MODEL_DIR must name the payload dir}"
: "${FRANK_DRAGOON_CHECKPOINT:?FRANK_DRAGOON_CHECKPOINT must name the .hgn inside it}"
: "${FRANK_DRAGOON_CHECKPOINT_BYTES:?FRANK_DRAGOON_CHECKPOINT_BYTES must pin the size}"
: "${FRANK_DRAGOON_NGRAM_TABLE:?FRANK_DRAGOON_NGRAM_TABLE must name the lookup table}"
: "${FRANK_DRAGOON_NGRAM_BYTES:?FRANK_DRAGOON_NGRAM_BYTES must pin the table size}"
: "${FRANK_DRAGOON_VISION_TOWER:?FRANK_DRAGOON_VISION_TOWER must name the tower}"
: "${FRANK_DRAGOON_VISION_BYTES:?FRANK_DRAGOON_VISION_BYTES must pin the tower size}"
: "${FRANK_DRAGOON_TOKENIZER_FILES:?FRANK_DRAGOON_TOKENIZER_FILES must list the tokenizer dir contents}"
: "${HALOGEN_MODEL_ID:?HALOGEN_MODEL_ID must be the served alias (frank-dragoon)}"
: "${HALOGEN_CTX:?HALOGEN_CTX must be set (per-slot window)}"
: "${HALOGEN_KV_POOL_POSITIONS:?HALOGEN_KV_POOL_POSITIONS must be set}"
: "${HALOGEN_KV_SLOTS:?HALOGEN_KV_SLOTS must be set}"
: "${HALOGEN_SPEC_ADAPT:?HALOGEN_SPEC_ADAPT must be set (0 = head always on)}"

# Capture podman's output BEFORE grepping: under `set -o pipefail` the
# `podman images | grep -q` pattern races (grep exits on first match,
# podman dies SIGPIPE 141, pipefail turns the 141 into "no match" — the
# 2026-09-30 deploy refusal).
DIGESTS="$(podman images --format '{{.RepoDigests}}')"
if ! grep -qF "${FRANK_DRAGOON_IMAGE#*@}" <<<"$DIGESTS"; then
    echo "frank-dragoon: image $FRANK_DRAGOON_IMAGE not in podman storage;" >&2
    echo "  pull it: podman pull ${FRANK_DRAGOON_IMAGE%@*}" >&2
    exit 1
fi

# Size-exact gates on all three payload files: a truncated curl exits 0 on
# some failures and a partial .hgn loads far enough to look alive, so the
# byte counts from the Hub tree API are the contract (fleet fetch law).
ck() {
    # ck <env-name> <path> <expected-bytes>
    if [ ! -f "$2" ]; then
        echo "frank-dragoon: $1 $2 missing;" >&2
        echo "  fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
        exit 1
    fi
    local sz
    sz="$(stat -c %s "$2")"
    if [ "$sz" != "$3" ]; then
        echo "frank-dragoon: $1 is $sz bytes, expected $3 (partial download?)" >&2
        exit 1
    fi
}
ck checkpoint "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_CHECKPOINT" "$FRANK_DRAGOON_CHECKPOINT_BYTES"
ck n-gram-table "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_NGRAM_TABLE" "$FRANK_DRAGOON_NGRAM_BYTES"
ck vision-tower "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_VISION_TOWER" "$FRANK_DRAGOON_VISION_BYTES"

# The payload ships the tokenizer as a FLAT DIRECTORY. A HuggingFace cache
# snapshot symlinks each file into a sibling blobs/, and those symlinks
# dangle inside the container mount — so the directory must be real and
# every named file must be a regular file, not a link.
TOK_DIR="$FRANK_DRAGOON_MODEL_DIR/tokenizer"
if [ -L "$TOK_DIR" ] || [ ! -d "$TOK_DIR" ]; then
    echo "frank-dragoon: tokenizer dir $TOK_DIR missing or a symlink (HF cache snapshot?);" >&2
    echo "  fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
    exit 1
fi
for TF in $FRANK_DRAGOON_TOKENIZER_FILES; do
    if [ ! -f "$TOK_DIR/$TF" ] || [ -L "$TOK_DIR/$TF" ]; then
        echo "frank-dragoon: tokenizer file $TOK_DIR/$TF missing or a symlink;" >&2
        echo "  fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
        exit 1
    fi
done

# GPU access is mandatory but the node paths are variables so the seam can
# drive the gate on a box without an AMD GPU.
DEV_KFD="${FRANK_DRAGOON_DEV_KFD:-/dev/kfd}"
DEV_DRI="${FRANK_DRAGOON_DEV_DRI:-/dev/dri}"
if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then
    echo "frank-dragoon: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
    exit 1
fi
# Host-RAM preflight — the engine's pin gate, checked BEFORE podman run.
# The engine refuses to pin when MemAvailable is below the checkpoint plus
# a 16 GiB floor: pinned pages cannot be swapped, and under the floor the
# n-gram lookup pages in from disk on every long prompt. A RAM-short host
# (a co-tenant spike, a sibling engine dying) crash-loops the unit — each
# boot dies in ~5 s at the pin gate — until the RAM clears. The 2026-10-06
# cutover hit this: 8 boots, MemAvailable 74.77 GiB vs the 78.11 GiB needed,
# self-resolved in ~2 min when the dying 27B engine freed RAM. Check here and
# WAIT for the RAM to clear, so a transient dip boots once instead of
# crash-looping; if it never clears, refuse loudly.
MEMINFO="${FRANK_DRAGOON_MEMINFO:-/proc/meminfo}"
PIN_KB=$(( FRANK_DRAGOON_CHECKPOINT_BYTES / 1024 ))
NEED_KB=$(( PIN_KB + 16 * 1048576 ))
ram_avail_kb() { awk '/^MemAvailable:/{print $2; exit}' "$MEMINFO"; }
MEMAVAIL_KB="$(ram_avail_kb)" || MEMAVAIL_KB=""
if [ -n "$MEMAVAIL_KB" ] && [ "$MEMAVAIL_KB" -lt "$NEED_KB" ]; then
    WAIT_S="${FRANK_DRAGOON_RAM_WAIT:-600}"
    GIB() { awk -v kb="$1" 'BEGIN{printf "%.2f", kb/1048576}'; }
    echo "frank-dragoon: host RAM below the pin gate — MemAvailable $(GIB "$MEMAVAIL_KB") GiB < needed $(GIB "$NEED_KB") GiB ($(GIB "$PIN_KB") GiB checkpoint + 16.00 GiB floor); waiting up to ${WAIT_S}s for host RAM to clear" >&2
    deadline=$(( $(date +%s) + WAIT_S ))
    while [ "$(date +%s)" -lt "$deadline" ]; do
        sleep 5
        MEMAVAIL_KB="$(ram_avail_kb)" || MEMAVAIL_KB=""
        if [ -n "$MEMAVAIL_KB" ] && [ "$MEMAVAIL_KB" -ge "$NEED_KB" ]; then
            break
        fi
    done
    if [ -z "$MEMAVAIL_KB" ] || [ "$MEMAVAIL_KB" -lt "$NEED_KB" ]; then
        echo "frank-dragoon: MemAvailable $(GIB "${MEMAVAIL_KB:-0}") GiB after ${WAIT_S}s; refusing to start (the engine would refuse the pin)" >&2
        exit 1
    fi
    echo "frank-dragoon: MemAvailable cleared to $(GIB "$MEMAVAIL_KB") GiB; starting" >&2
fi
for CTR in ${FRANK_DRAGOON_LEGACY_CONTAINERS:-} "$FRANK_DRAGOON_CONTAINER"; do
    podman rm -f "$CTR" >/dev/null 2>&1 || true
done

LOG="${FRANK_DRAGOON_LOG:-/home/dank/logs/frank_dragoon.log}"
mkdir -p "$(dirname "$LOG")"
# --ipc=host for the udev-granted supplementary group, seccomp unconfined
# for the ROCm ioctl surface, and unbounded memlock (the unit's
# LimitMEMLOCK=infinity is inherited, so runc never has to RAISE it). NO
# --ulimit nofile: Dankclaw's hard limit is 524288, rootless runc cannot
# raise a hard limit and dies with "error setting rlimit type 7:
# operation not permitted" (measured 2026-10-05 cutover); the model card
# and both sibling serve scripts pass no nofile either. No comment lines
# inside the backslash chain — a mid-chain # terminates the continuation
# (the 2026-10-05 exit-127 crash loop on the sibling serve script).
podman run --name "$FRANK_DRAGOON_CONTAINER" \
  --publish "$FRANK_DRAGOON_PUBLISH" \
  --ipc=host \
  --ulimit memlock=-1:-1 \
  --device "$DEV_KFD" --device "$DEV_DRI" \
  --group-add keep-groups \
  --security-opt seccomp=unconfined \
  -e HALOGEN_MODEL_ID="$HALOGEN_MODEL_ID" \
  -e HALOGEN_CHECKPOINT="/models/$FRANK_DRAGOON_CHECKPOINT" \
  -e HALOGEN_NGRAM_TABLE="/models/$FRANK_DRAGOON_NGRAM_TABLE" \
  -e HALOGEN_VISION_TOWER="/models/$FRANK_DRAGOON_VISION_TOWER" \
  -e HALOGEN_TOKENIZER=/tokenizer \
  -e HALOGEN_CTX="$HALOGEN_CTX" \
  -e HALOGEN_KV_POOL_POSITIONS="$HALOGEN_KV_POOL_POSITIONS" \
  -e HALOGEN_KV_SLOTS="$HALOGEN_KV_SLOTS" \
  -e HALOGEN_HOST_RESERVE_GIB="${HALOGEN_HOST_RESERVE_GIB:-20}" \
  -e HALOGEN_SPEC_ADAPT="$HALOGEN_SPEC_ADAPT" \
  -e HALOGEN_MAX_TOK="${HALOGEN_MAX_TOK:-8192}" \
  -e HALOGEN_MAX_TOKENS_CAP="${HALOGEN_MAX_TOKENS_CAP:-65536}" \
  -e HALOGEN_QUEUE_TIMEOUT="${HALOGEN_QUEUE_TIMEOUT:-7200}" \
  -e HALOGEN_TEMPERATURE="${HALOGEN_TEMPERATURE:-1.0}" \
  -e HALOGEN_TOP_P="${HALOGEN_TOP_P:-0.95}" \
  -e HALOGEN_TOP_K="${HALOGEN_TOP_K:-20}" \
  ${HALOGEN_WEIGHTS_LOCK:+-e HALOGEN_WEIGHTS_LOCK="$HALOGEN_WEIGHTS_LOCK"} \
  -v "$FRANK_DRAGOON_MODEL_DIR:/models:ro" \
  -v "$FRANK_DRAGOON_MODEL_DIR/tokenizer:/tokenizer:ro" \
  "$FRANK_DRAGOON_IMAGE" \
  > "$LOG" 2>&1

# FOREGROUND `podman run`: the engine's stdout/stderr are this script's
# output, so the boot ledger (kv pool, prompt cache, MTP head, any clamp)
# lands in $LOG as documented. podman run returns the container's exit
# code, so a crash is a nonzero ExecStart and the unit's Restart=always
# stays real; systemd stop goes through the unit's ExecStop and the run
# client forwards SIGTERM to the engine, which drains on it.