#!/usr/bin/env bash
# serve-frank-archon.sh — frank-archon endpoint: radiance (StillDeadcode,
# Apache-2.0) serving the PUBLISHER's 4-bit-codebook container of the
# official Qwen/Qwen3.8-Flash-Next (the engine author's own rad-convert of
# the bf16 checkpoint: qwen3.8-next-flash-fp8-iq4r-moe.rad, fetched by
# scripts/download-frank-archon-iq4r.sh) on BOTH R9700s (TP=2, wht6 wire),
# dankserver :7770. 2026-10-15 payload swap from the in-house w4dn8 of the
# Swift1.5 finetune (payload on disk = rollback; the swap commit is the
# revert); 2026-10-05 radiance cutover from FreeToken (git tag
# frank-archon-freetoken-pr587-20261005 is the pre-radiance state).
#
# The run line below IS the engine argv: the image's entrypoint is the
# engine and its tools sit beside it on PATH. Unlike every predecessor
# there is no per-boot VRAM planner to second-guess: the boot ledger in the
# log states every budget before anything is allocated (weights vs KV
# split, the expert host pool's coverage of the worst case, the PLE pin,
# the P2P answer per pair) — read it after every resize.
# Deployed shape (2026-10-19, radiance 1.8.1): THREE decode slots x 200,000
# tokens, fp8 KV, 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache in
# pinned host RAM, PLE pinned in RAM. The per-card split between the expert slab
# and the KV pool is NO LONGER an operator number — 1.3.0 removed the VRAM pair
# and the budget resolver derives it from headroom, the cache bound and the host
# pool, so the slab size and the pool's token count are LEDGER lines: after every
# boot read them out of the log and check the pool still backs 3 x 200,000.
#
# The root unit (scripts/frank-archon.service, User=dank,
# LimitMEMLOCK=infinity) grants the pin budget: 2 x 44 GiB expert pools +
# the 47.7 GiB PLE table + 8 GiB prefix host tier — ~147 GiB of pinned
# pages, which is ALSO why this script refuses to start when ttm's
# pages_limit/pages_max cap is under FRANK_ARCHON_TTM_MIN_MIB (see below).
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_ARCHON_ENV_FILE:-$SCRIPT_DIR/frank-archon.env}"

if [ ! -f "$ENV_FILE" ]; then
    echo "ERROR: frank-archon env file absent: $ENV_FILE" >&2
    exit 1
fi
set -a
# shellcheck disable=SC1090
source "$ENV_FILE"
set +a

: "${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}"
: "${FRANK_ARCHON_MODEL_DIR:?FRANK_ARCHON_MODEL_DIR must name the .rad dir}"
: "${FRANK_ARCHON_MODEL_FILE:?FRANK_ARCHON_MODEL_FILE must name the .rad file}"
: "${FRANK_ARCHON_STATE_DIR:?FRANK_ARCHON_STATE_DIR must name the host dir holding the rad-tune cache}"
: "${FRANK_ARCHON_CONTAINER:?FRANK_ARCHON_CONTAINER must name the container}"
: "${FRANK_ARCHON_SERVED_NAME:?FRANK_ARCHON_SERVED_NAME must name the alias}"
: "${FRANK_ARCHON_PORT:?FRANK_ARCHON_PORT must be the port}"
: "${FRANK_ARCHON_TEMPLATE:?FRANK_ARCHON_TEMPLATE must name the chat template}"
: "${FRANK_ARCHON_TP:?FRANK_ARCHON_TP must be set}"
: "${FRANK_ARCHON_TP_WIRE:?FRANK_ARCHON_TP_WIRE must be set}"
: "${FRANK_ARCHON_MAX_SEQS:?FRANK_ARCHON_MAX_SEQS must be set}"
: "${FRANK_ARCHON_MAX_MODEL_LEN:?FRANK_ARCHON_MAX_MODEL_LEN must be set}"
: "${FRANK_ARCHON_PLACEMENT:?FRANK_ARCHON_PLACEMENT must be set}"
: "${FRANK_ARCHON_HOST_POOL_MIB:?FRANK_ARCHON_HOST_POOL_MIB must be set}"
: "${FRANK_ARCHON_GPU_HEADROOM_MIB:?FRANK_ARCHON_GPU_HEADROOM_MIB must be set}"
: "${FRANK_ARCHON_BUSY_PREFILL_CHUNK:?FRANK_ARCHON_BUSY_PREFILL_CHUNK must be set (auto|off|N)}"
: "${FRANK_ARCHON_KV_DTYPE:?FRANK_ARCHON_KV_DTYPE must be set}"
: "${FRANK_ARCHON_SPEC_TOKENS:?FRANK_ARCHON_SPEC_TOKENS must be set}"
: "${FRANK_ARCHON_MAX_BATCHED_TOKENS:?FRANK_ARCHON_MAX_BATCHED_TOKENS must be set}"
: "${FRANK_ARCHON_PREFIX_CACHE_MIB:?FRANK_ARCHON_PREFIX_CACHE_MIB must be set}"
: "${FRANK_ARCHON_NGRAM_PLACEMENT:?FRANK_ARCHON_NGRAM_PLACEMENT must be set}"
: "${FRANK_ARCHON_TTM_MIN_MIB:?FRANK_ARCHON_TTM_MIN_MIB must be set}"
: "${FRANK_ARCHON_TTM_TARGET_MIB:?FRANK_ARCHON_TTM_TARGET_MIB must be set}"
: "${FRANK_ARCHON_RAD_MIN_MIB:?FRANK_ARCHON_RAD_MIN_MIB must be set}"

# The image gate is the digest-pinned ref itself: a name@sha256 ref exists
# in storage only if THAT manifest was pulled — a retagged look-alike or a
# newer latest does not satisfy it. `podman image exists` takes a
# name@digest ref in one command — no pipefail/grep race (the 2026-09-30
# lesson). Runs as dank: the unit's process uses dank's ROOTLESS podman
# storage (root unit, User=dank — the fleet's frank-archon shape).
if ! podman image exists "$FRANK_ARCHON_IMAGE"; then
    echo "ERROR: radiance image absent: $FRANK_ARCHON_IMAGE" >&2
    echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest (as dank)" >&2
    exit 1
fi

# The payload gate: the .rad container file, sized. A truncated convert
# would otherwise buy a guaranteed long-timeout boot instead of this fast
# failure. The staging dir / raw checkpoint are NOT this script's concern
# (scripts/build-frank-archon-rad.sh built the file; its rad-info output
# is the record).
RAD="$FRANK_ARCHON_MODEL_DIR/$FRANK_ARCHON_MODEL_FILE"
if [ ! -f "$RAD" ]; then
    echo "ERROR: .rad container missing: $RAD" >&2
    echo "frank-archon: run scripts/build-frank-archon-rad.sh" >&2
    exit 1
fi
RAD_MIB=$(( $(stat -c %s "$RAD") / 1048576 ))
if [ "$RAD_MIB" -lt "$FRANK_ARCHON_RAD_MIN_MIB" ]; then
    echo "ERROR: .rad container too small: $RAD is ${RAD_MIB} MiB, want >= $FRANK_ARCHON_RAD_MIN_MIB" >&2
    echo "frank-archon: the convert was truncated — rebuild it" >&2
    exit 1
fi

if [ ! -f "$FRANK_ARCHON_TEMPLATE" ]; then
    echo "ERROR: chat template missing: $FRANK_ARCHON_TEMPLATE" >&2
    exit 1
fi

mkdir -p "$FRANK_ARCHON_STATE_DIR/tune"
# rad-tune's kernel-selection cache (see the runbook's "Tuning levers"):
# the engine's selector reads $RADIANCE_HOME/tune/<machine>.tune at boot;
# $RADIANCE_HOME defaults to /opt/radiance, which lives in the image, so
# the cache must ride a host mount or it dies with the --rm container
# that wrote it. The engine's create is not recursive, so the dir is pre-created.

if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then
    echo "ERROR: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2
    exit 1
fi

# TTM preflight: the pinned ledger (2 x FRANK_ARCHON_HOST_POOL_MIB + the
# 47.7 GiB PLE table + the prefix host tier, ~147 GiB) rides TTM's shared
# pages, capped by ttm's pages_limit (the current kernel exposes NO
# pages_max; the cap function takes the min over the knobs that exist).
# Below FRANK_ARCHON_TTM_MIN_MIB the load dies mid-pin with an allocator
# error, so the gate refuses first — and RAISES the knob before refusing:
# the cap is a runtime knob whose default is half of RAM, so a power failure
# resets it (measured 2026-10-12 23:17 on this host: 200 GiB -> 125,988 MiB,
# and Restart=always re-ran the refusal 6,919 times over ~20 h).
# FRANK_ARCHON_TTM_TARGET_MIB is the recorded host ceiling, not the floor.
# shellcheck source=/dev/null
source "$SCRIPT_DIR/ttm-preflight.sh"
ttm_ensure_cap || exit 1

# A crashed/killed unit can leave the container name registered (systemd
# SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy
# identities (the FreeToken-era frank-archon-qwen, the retired R9V
# container and the ad hoc experiment containers all hold this GPU); all
# idempotent.
for CTR in $FRANK_ARCHON_LEGACY_CONTAINERS "$FRANK_ARCHON_CONTAINER"; do
    podman rm -f "$CTR" >/dev/null 2>&1 || true
done

LOG=/home/dank/logs/frank_archon.log
mkdir -p "$(dirname "$LOG")"

# Interleave is opt-in: 0 is the engine's default and passing it anyway would put
# a flag in argv that changes nothing but the record. The sharing cap is always
# explicit because `auto` vs `off` is the difference between decode waiting one
# chunk and waiting the whole step.
PREFILL_ARGS=()
if [ "${FRANK_ARCHON_PREFILL_INTERLEAVE:-0}" -gt 0 ]; then
    PREFILL_ARGS=(--prefill-interleave "$FRANK_ARCHON_PREFILL_INTERLEAVE")
fi

# Publisher-vetted container contract (docs/DOCKER.md, compose/common.yaml):
# --network host (the server binds the host's addresses directly, no NAT),
# --init (the engine releases the cards on SIGTERM), seccomp=unconfined
# (Docker/podman's default profile refuses io_uring, which the n-gram
# table's reads go through, and the NUMA binding calls the ROCm runtime
# makes for host memory), memlock unbounded (the pinned ledger — legal
# because the root unit grants LimitMEMLOCK=infinity to this dank process),
# /dev/kfd + /dev/dri (render/video groups are moot for a root container).
# The template rides a single-file mount and the .rad dir is ro. The prefix cache
# has no disk tier any more, so the only host state this container writes is the
# rad-tune cache under /opt/radiance/share/radiance/tune.
podman run -d --name "$FRANK_ARCHON_CONTAINER" \
  --network host \
  --init \
  --security-opt seccomp=unconfined \
  --ulimit memlock=-1:-1 \
  --device /dev/kfd --device /dev/dri \
  -v "$FRANK_ARCHON_MODEL_DIR":/models:ro \
  -v "$FRANK_ARCHON_STATE_DIR/tune":/opt/radiance/share/radiance/tune \
  -v "$FRANK_ARCHON_TEMPLATE":/templates/template.jinja:ro \
  "$FRANK_ARCHON_IMAGE" \
  --model "/models/$FRANK_ARCHON_MODEL_FILE" \
  --host :: --port "$FRANK_ARCHON_PORT" \
  --served-model-name "$FRANK_ARCHON_SERVED_NAME" \
  --override-chat-template /templates/template.jinja \
  --tp "$FRANK_ARCHON_TP" --tp-wire "$FRANK_ARCHON_TP_WIRE" \
  --max-num-seqs "$FRANK_ARCHON_MAX_SEQS" \
  --max-model-len "$FRANK_ARCHON_MAX_MODEL_LEN" \
  --placement "$FRANK_ARCHON_PLACEMENT" \
  --host-pool-mib "$FRANK_ARCHON_HOST_POOL_MIB" \
  --gpu-headroom-mib "$FRANK_ARCHON_GPU_HEADROOM_MIB" \
  --kv-cache-dtype "$FRANK_ARCHON_KV_DTYPE" \
  --prefix-cache-mib "$FRANK_ARCHON_PREFIX_CACHE_MIB" \
  --ngram-placement "$FRANK_ARCHON_NGRAM_PLACEMENT" \
  --num-speculative-tokens "$FRANK_ARCHON_SPEC_TOKENS" \
  --max-num-batched-tokens "$FRANK_ARCHON_MAX_BATCHED_TOKENS" \
  --busy-prefill-chunk "$FRANK_ARCHON_BUSY_PREFILL_CHUNK" \
  "${PREFILL_ARGS[@]}" \
  ${FRANK_ARCHON_EXTRA_ARGS:+$FRANK_ARCHON_EXTRA_ARGS} \
  > "$LOG" 2>&1

# From here on, this script's lifetime IS the container's: wait returns the
# container's exit code, so a crash is a nonzero ExecStart and the unit's
# Restart=always is real. (systemd stop goes through ExecStop in the unit —
# podman wait alone cannot stop the container child.)
exec podman wait "$FRANK_ARCHON_CONTAINER"