#!/usr/bin/env bash
# serve-frank-large.sh — frank-large endpoint: halogen (closed-source
# C++/HIP engine, image ghcr.io/peonist-ai/halogen, digest-pinned in
# frank-large.env) serving peonist-ai/halogen-qwen3.8-27b on Dankclaw's
# Strix Halo iGPU, host :7770. 2026-10-05 cutover from the ROCmFPX
# llama.cpp shape — revert the swap commit's trio to roll back (payload
# and fork build stay on disk).
#
# The engine is driven by `podman run` DIRECTLY (the fleet's frank-primary
# pattern): the image entrypoint `halogen-docker` in default "all" mode
# starts engine + OpenAI frontend in one container. The frontend listens on
# 8731 inside (HALOGEN_API_PORT default) and is published as host 7770; the
# engine's own token port (8730, no auth) stays unpublished on the
# container's loopback. The image ships NO default checkpoint:
# HALOGEN_CHECKPOINT selects the file inside the /models mount.
#
# Gates mirror the deployer EXACTLY (the 2026-10-01 crash-loop lesson —
# serve gate and deploy gate disagreeing = green deploy, red unit):
# tokenizer directory present with every named file a FLAT REGULAR FILE —
# the model card ships `tokenizer/` as a flat directory and the engine
# defaults HALOGEN_TOKENIZER to /tokenizer, so a HF cache snapshot's
# symlinks into blobs/ dangle inside the mount; GPU device nodes present.
#
# Why containers instead of the bare binary (the old frank-large shape):
# the payload is .hgn and only halogen reads it; halogen ships as a
# container; the root unit keeps LimitMEMLOCK=infinity for parity and to
# outlive the login session, exactly like frank-primary's.
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
if [ ! -f "$ENV_FILE" ]; then
    echo "frank-large: env file not found: $ENV_FILE" >&2
    exit 1
fi
set -a
# shellcheck disable=SC1090
source "$ENV_FILE"
set +a

: "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE must name the digest-pinned image}"
: "${FRANK_LARGE_CONTAINER:?FRANK_LARGE_CONTAINER must name the container}"
: "${FRANK_LARGE_PUBLISH:?FRANK_LARGE_PUBLISH must be host:container}"
: "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR must name the payload dir}"
: "${FRANK_LARGE_CHECKPOINT:?FRANK_LARGE_CHECKPOINT must name the .hgn inside it}"
: "${FRANK_LARGE_CHECKPOINT_BYTES:?FRANK_LARGE_CHECKPOINT_BYTES must pin the size}"
: "${FRANK_LARGE_TOKENIZER_FILES:?FRANK_LARGE_TOKENIZER_FILES must list the tokenizer dir contents}"
: "${HALOGEN_MODEL_ID:?HALOGEN_MODEL_ID must be the served alias (frank-large)}"
: "${HALOGEN_SLOT_CTX:?HALOGEN_SLOT_CTX must be set}"
: "${HALOGEN_KV_SLOTS:?HALOGEN_KV_SLOTS must be set (1 keeps the drafter on)}"
: "${HALOGEN_DRAFTER:?HALOGEN_DRAFTER must be set (2 = DFlash2)}"

# Capture podman's output BEFORE grepping: under `set -o pipefail` the
# `podman images | grep -q` pattern races (grep exits on first match,
# podman dies SIGPIPE 141, pipefail turns the 141 into "no match" — the
# 2026-09-30 deploy refusal).
DIGESTS="$(podman images --format '{{.RepoDigests}}')"
if ! grep -qF "${FRANK_LARGE_IMAGE#*@}" <<<"$DIGESTS"; then
    echo "frank-large: image $FRANK_LARGE_IMAGE not in podman storage;" >&2
    echo "  pull it: podman pull ${FRANK_LARGE_IMAGE%@*}" >&2
    exit 1
fi

# The checkpoint gate is size-exact: a truncated curl exits 0 on some
# failures and a partial .hgn loads far enough to look alive, so the byte
# count from the model card is the contract (the fleet fetch law).
CKPT="$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_CHECKPOINT"
if [ ! -f "$CKPT" ]; then
    echo "frank-large: checkpoint $CKPT missing;" >&2
    echo "  fetch it: bash scripts/download-frank-large-halogen.sh" >&2
    exit 1
fi
CKPT_SIZE="$(stat -c %s "$CKPT")"
if [ "$CKPT_SIZE" != "$FRANK_LARGE_CHECKPOINT_BYTES" ]; then
    echo "frank-large: checkpoint is $CKPT_SIZE bytes, expected $FRANK_LARGE_CHECKPOINT_BYTES (partial download?)" >&2
    exit 1
fi

# The payload ships the tokenizer as a FLAT DIRECTORY (model card:
# "tokenizer/ flat tokenizer directory"). A HuggingFace cache snapshot
# symlinks each file into a sibling blobs/, and those symlinks dangle
# inside the container mount — so the directory must be real and every
# named file must be a regular file, not a link.
TOK_DIR="$FRANK_LARGE_MODEL_DIR/tokenizer"
if [ -L "$TOK_DIR" ] || [ ! -d "$TOK_DIR" ]; then
    echo "frank-large: tokenizer dir $TOK_DIR missing or a symlink (HF cache snapshot?);" >&2
    echo "  fetch it: bash scripts/download-frank-large-halogen.sh" >&2
    exit 1
fi
for TF in $FRANK_LARGE_TOKENIZER_FILES; do
    if [ ! -f "$TOK_DIR/$TF" ] || [ -L "$TOK_DIR/$TF" ]; then
        echo "frank-large: tokenizer file $TOK_DIR/$TF missing or a symlink;" >&2
        echo "  fetch it: bash scripts/download-frank-large-halogen.sh" >&2
        exit 1
    fi
done

# GPU access is mandatory but the node paths are variables so the seam can
# drive the gate on a box without an AMD GPU.
DEV_KFD="${FRANK_LARGE_DEV_KFD:-/dev/kfd}"
DEV_DRI="${FRANK_LARGE_DEV_DRI:-/dev/dri}"
if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then
    echo "frank-large: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
    exit 1
fi
for CTR in ${FRANK_LARGE_LEGACY_CONTAINERS:-} "$FRANK_LARGE_CONTAINER"; do
    podman rm -f "$CTR" >/dev/null 2>&1 || true
done

LOG="${FRANK_LARGE_LOG:-/home/dank/logs/frank_large.log}"
mkdir -p "$(dirname "$LOG")"
# for the udev-granted supplementary group, seccomp unconfined for the ROCm
# ioctl surface, and unbounded memlock (the unit's LimitMEMLOCK=infinity is
# inherited, so runc never has to RAISE it). NO --ulimit nofile: Dankclaw's
# hard limit is 524288, rootless runc cannot raise a hard limit and dies
# with "error setting rlimit type 7: operation not permitted" (measured
# 2026-10-05 cutover), and 524288 fds is already ~100x what the engine
# wants; the model card and both sibling serve scripts pass no nofile
# either. No comment lines inside the backslash
# chain — a mid-chain # terminates the continuation (the 2026-10-05
# exit-127 crash loop on the sibling serve script).
podman run -d --name "$FRANK_LARGE_CONTAINER" \
  --publish "$FRANK_LARGE_PUBLISH" \
  --ipc=host \
  --ulimit memlock=-1:-1 \
  --device "$DEV_KFD" --device "$DEV_DRI" \
  --group-add keep-groups \
  --security-opt seccomp=unconfined \
  -e HALOGEN_MODEL_ID="$HALOGEN_MODEL_ID" \
  -e HALOGEN_CHECKPOINT="/models/$FRANK_LARGE_CHECKPOINT" \
  -e HALOGEN_SLOT_CTX="$HALOGEN_SLOT_CTX" \
  -e HALOGEN_KV_SLOTS="${HALOGEN_KV_SLOTS:-1}" \
  -e HALOGEN_DRAFTER="$HALOGEN_DRAFTER" \
  -e HALOGEN_MAX_TOKENS_CAP="${HALOGEN_MAX_TOKENS_CAP:-65536}" \
  -e HALOGEN_QUEUE_TIMEOUT="${HALOGEN_QUEUE_TIMEOUT:-7200}" \
  ${HALOGEN_CACHE_MB:+-e HALOGEN_CACHE_MB="$HALOGEN_CACHE_MB"} \
  -v "$FRANK_LARGE_MODEL_DIR:/models:ro" \
  -v "$FRANK_LARGE_MODEL_DIR/tokenizer:/tokenizer:ro" \
  "$FRANK_LARGE_IMAGE" \
  > "$LOG" 2>&1

# This script's lifetime IS the container's: podman wait returns the
# container's exit code, so a crash is a nonzero ExecStart and the unit's
# Restart=always is real. systemd stop goes through the unit's ExecStop
# (podman wait alone cannot stop the child).
exec podman wait "$FRANK_LARGE_CONTAINER"