#!/usr/bin/env bash
# run-qwen38-bf16.sh — RUNS ON DANKSERVER. Boot a container over the campaign's
# ONE measuring shape for the phases that need both R9700s. The engine owns the
# cards, so frank-archon is stopped first and restored on every exit path — the
# autoresearch pattern: stop, never mask (the unit file in /etc/systemd/system
# is a symlink into the checkout and mask fails on a relinked unit; the deploy
# restores it).
#
#   kldref            --kld-record the reference log-probabilities: the engine
#                     runs /home/dank/corpora/out/kld.jsonl (197 prompts, 50,432
#                     scored positions, score tail 256) through prefill, writes
#                     /home/dank/campaign/qwen38-w5nl64/kld-ref, and EXITS. The
#                     model behind it is the BF16 container from
#                     scripts/build-qwen38-bf16-rad.sh — the vendor checkpoint's
#                     own numbers, which is what makes it the zero point.
#   score RAD OUT     --kld-ref score ONE container against that recording and
#                     EXIT: the FP8-native anchor, the served iq4r, and every
#                     arm. Deliberately the SAME engine pin, the same TP, the
#                     same step size, the same fp8 KV, --deterministic and no
#                     speculation as kldref — KL and top-1 agreement are only
#                     comparable across boots when everything except the weights
#                     is identical, so this mode is one argv with --model and
#                     the two --kld-* flags changed, never a second shape.
#                     RAD must sit under the models dir (the container reads it
#                     through the ro /models mount); OUT lands on a writable
#                     mount because --kld-out F also writes F.rows beside it.
#   calib             serve on :7770 with the calibration tap open
#                     (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS=12000000). A
#                     separate driver — scripts/calib-push.py --port 7770
#                     --container qwen38-bf16 --corpus
#                     /home/dank/corpora/out/calib.jsonl — pushes the corpus; at
#                     12M pooled positions the engine writes the Grams and the
#                     tap switches itself off (calib-push prints TAP_DONE). Stop
#                     this script after that; the trap brings frank-archon back.
#
# Shape: the vendor's bf16 calibration lines — expert_tiered placement with the
# weights disk tier, deterministic (the resident set pinned to the loader's plan,
# so a recorded reference and a scored candidate see the same one), no
# speculation, fp8 KV, a 32,768-token window, the PLE n-gram table pinned to RAM,
# and the production chat template rather than the one baked into the container —
# every scored container is asked the same tokenised questions the reference
# answered. The window the flag spells is 32,768; the pool this payload's boot
# actually gets is 7,436 tokens, so kld.jsonl is re-chunked by
# scripts/rechunk-kld-corpus.py into documents of at most 7,168 tokens — the
# widest one the reference can hold. The kldref gate below re-checks the file
# against the declared window, and the re-chunker is what keeps that honest.
#
# NO --vram-kv-mib: it was REMOVED at 1.3.0 and naming it is a boot refusal, not
# a no-op — verified on the pinned image 2026-10-27: "E --vram-kv-mib was
# removed: the card is split between the weights and the cache from what the
# model needs." The resolver derives the split from --host-pool-mib,
# --gpu-headroom-mib and the activation arena; the rig inherits the first two from
# frank-archon.env, the file that owns them. On THIS payload the host pool is not
# the cache's supplier: the boot lands in vram_budget.cpp's `no lender` branch,
# where the experts keep their plan-fixed 25.29 GiB and the cache gets the residue.
# Measured twice — 12,288 and 49,152 MiB of host pool both printed the same
# "kv cache 271.25 MiB ... 7,436"; only --gpu-headroom-mib moved anything
# (768 -> 96 grew vram_weights 29.41 -> 30.06 GiB). The supplier the source names
# is the arena: vram_budget.cpp:117, "lower --max-num-batched-tokens (the arena
# scales with it)" — which is what the step size below now does.
#
# The pool, not the window, decides what the recorder accepts: it refuses any
# document longer than the pool, so the 32,768-token window was decorative while the
# cache held 7,436 — "E kld: document 196 is 8507 tokens and the KV pool holds
# 7436, so it can never be scored" on all 197 records, exit 1. The longest record
# is 10,554 tokens by the engine's own count (chars/3.6 for this file, not the
# chars/4.97 this repo measured on the K3 corpus).
#
# The step size is ONE value for the recording and every scoring pass, and it is
# the value frank-archon.env owns (FRANK_ARCHON_MAX_BATCHED_TOKENS, 2,048): the
# quantised expert FILE tiers (the served iq4r, the arms) stream only at
# MNBT<=2048 and refuse the widened form above it, and the arena a step costs the
# card comes straight out of the cache — 1.08 GiB at a 2,048-token step against
# 504.39 MiB at 1,024. The corpora are cut to the window the boot ledger reports,
# which is what the pool raise is for (frank-archon.env: HOST_POOL_MIB); a narrow
# step was never the fix, it was the symptom's disguise.
#
# usage: scripts/run-qwen38-bf16.sh kldref|calib|score RAD OUT [--dry-run] [--image REF]
#   (RAD and OUT are score's two positionals: RAD is a container under the models
#    dir, OUT the report path whose directory is created if missing)
#   --dry-run   print the exact `podman run` argv, one shell-quoted argument a
#               line, and exit 0: the service is NOT stopped, nothing is
#               mounted, nothing is written — the offline test seam. It still
#               validates the argv's own shape (paths through the /models mount,
#               a step size that is a number), because an argv that could not
#               boot is not worth printing.
#   --image REF override the digest pin (a probe boot, or the test seam)
#
# knobs: FRANK_QWEN38_MODELS (the dir mounted at /models), FRANK_QWEN38_RAD (the
#        bf16 container, kldref only), FRANK_QWEN38_CAMPAIGN (holds kld-ref/ and
#        calib/), FRANK_QWEN38_KLD_CORPUS, FRANK_QWEN38_TPL (the template copy
#        the container reads through /models), FRANK_QWEN38_LOG_DIR, and the step
#        and slot counts inherited as FRANK_ARCHON_MAX_BATCHED_TOKENS /
#        FRANK_ARCHON_MAX_SEQS — no per-mode literals (the audit law: a mode that
#        carries its own step measures a shape nothing serves; calib is exempt, see
#        Q38_CALIB_MNBT / Q38_CALIB_SEQS below — its output is never compared) —
#        and FRANK_QWEN38_MAX_MODEL_LEN — the window every mode boots at (32,768;
#        see the corpus census above, and raise it together with the corpus).
# kldref refuses (exit 2) when the longest record of the corpus it is about to
# read is longer than MAX_MODEL_LEN x 3 characters — chars/3 being the
# pessimistic token:char ratio, so the script says no before the engine finds out
# inside a drained window, after the checkpoint work is already done. score
# refuses when the reference dir is absent, when RAD is not on disk or not under
# the models mount, or when OUT already exists: the trash law means a second
# score over the same name is a deliberate act, and two numbers in one file is
# how a campaign loses its own history.
# Engine stdout goes to $LOG_DIR/qwen38_bf16_<mode>.log — the boot ledger is the
# record of what the resolver actually gave the cards; the journal never sees it.
# exit: the engine's code; 2 = refused before the service was touched
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "$SCRIPT_DIR/frank-archon.env"
IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}"

MODE=""
POS=()
DRY=0
while [ $# -gt 0 ]; do
    case "$1" in
        kldref|calib|score) MODE="$1" ;;
        --dry-run)          DRY=1 ;;
        --image)            [ "$#" -ge 2 ] || { echo "--image needs a reference" >&2; exit 2; }
                            IMAGE="$2"; shift ;;
        *)
            # score's two positional arguments, and nothing else may be bare.
            if [ "$MODE" = score ] && [ "${#POS[@]}" -lt 2 ]; then
                POS+=("$1")
            else
                echo "usage: ${0##*/} kldref|calib|score RAD OUT [--dry-run] [--image REF]  (unknown argument: $1)" >&2
                exit 2
            fi ;;
    esac
    shift
done
if [ -z "$MODE" ]; then
    echo "usage: ${0##*/} kldref|calib|score RAD OUT [--dry-run] [--image REF]  (no mode given)" >&2
    exit 2
fi
if [ "$MODE" = score ] && [ "${#POS[@]}" -ne 2 ]; then
    echo "usage: ${0##*/} score RAD OUT  — RAD is the container to score, OUT the report to write (got ${#POS[@]} argument(s))" >&2
    exit 2
fi

MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
KLD_CORPUS="${FRANK_QWEN38_KLD_CORPUS:-/home/dank/corpora/out/kld.jsonl}"
RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}"
TPL="${FRANK_QWEN38_TPL:-$MODELS/tpl.jinja}"
TEMPLATE="$SCRIPT_DIR/qwen38-froggeric-v22.5.jinja"
LOG_DIR="${FRANK_QWEN38_LOG_DIR:-/home/dank/logs}"
LOG="$LOG_DIR/qwen38_bf16_${MODE}.log"
KLD_REF_DIR="$CAMPAIGN/kld-ref"
CALIB_DIR="$CAMPAIGN/calib"
C=qwen38-bf16
SERVICE=frank-archon
# The card split, taken from the file that owns it instead of a private pair:
# --host-pool-mib also funds the KV cache (see the header), so hardcoding it
# here once starved the recorder. The `:?` guards keep an empty env fatal.
HOST_POOL_MIB="${FRANK_ARCHON_HOST_POOL_MIB:?FRANK_ARCHON_HOST_POOL_MIB must be set}"
GPU_HEADROOM_MIB="${FRANK_ARCHON_GPU_HEADROOM_MIB:?FRANK_ARCHON_GPU_HEADROOM_MIB must be set}"
# The slot count, from the same file that owns the card split: production runs
# four slots and every mode measures at production's geometry. A per-mode
# literal here is what let calib ask for eight slots and then starve its own
# cache. The `:?` guard keeps an empty env fatal, as for the card split.
MAX_SEQS="${FRANK_ARCHON_MAX_SEQS:?FRANK_ARCHON_MAX_SEQS must be set}"
if ! [[ "$MAX_SEQS" =~ ^[0-9]+$ ]]; then
    echo "REFUSED: FRANK_ARCHON_MAX_SEQS must be a slot count (got: $MAX_SEQS)" >&2
    exit 2
fi

# ONE step size for every mode, quoted from the file that owns it — the audit
# law for a radiance rig is to inherit the proven number, not to carry a
# campaign-private value: production's FRANK_ARCHON_MAX_BATCHED_TOKENS. The
# 1,024 default this script carried was PR #80's answer to a starved cache;
# production moved to its widest legal step (2,048) on 2026-10-29 and the rig
# follows. The pool raise that shipped with it (66,560 a rank) bought this
# container 1.26 MiB of cache (`vram_kv 269.99 -> 271.25 MiB`, window still
# 7,436 tokens — the 95.37 GiB n-gram table's claim decides the rig's cache,
# not the pool) and was folded back the same day. The corpora stay cut to the
# window the boot ledger reports, never the other way round.
MNBT="${FRANK_ARCHON_MAX_BATCHED_TOKENS:?FRANK_ARCHON_MAX_BATCHED_TOKENS must be set}"
if ! [[ "$MNBT" =~ ^[0-9]+$ ]]; then
    echo "REFUSED: FRANK_ARCHON_MAX_BATCHED_TOKENS must be a token count (got: $MNBT)" >&2
    exit 2
fi

# Calib's shape is NOT a measuring shape. kldref and score produce KL numbers that are only
# comparable across boots at an identical shape, which is why they inherit production's step and
# slots to the letter. The tap's output is layer Hessians, never a number any report compares, so
# its step and slot count are its own — and they are the tap's dominant cost: a 7,168-token
# document cut into four 2,048-token steps re-streams each layer's routed expert union four
# times from the container, so one wide step is worth up to ~4x the read volume. Measured
# 2026-10-29 at the inherited 2,048/4: prefill 71 tok/s and a 47-hour tap; the probe exists to
# find out what the wide step is worth before the campaign pays for two days of drained serving.
# Both default to the inherited value, so nothing moves unless a probe asks it to.
CALIB_MNBT="${Q38_CALIB_MNBT:-$MNBT}"
CALIB_SEQS="${Q38_CALIB_SEQS:-$MAX_SEQS}"
for pair in "Q38_CALIB_MNBT=$CALIB_MNBT" "Q38_CALIB_SEQS=$CALIB_SEQS"; do
    if ! [[ "${pair#*=}" =~ ^[0-9]+$ ]]; then
        echo "REFUSED: ${pair%%=*} must be a number (got: ${pair#*=})" >&2
        exit 2
    fi
done

# The window every mode boots at. Over-length is a REFUSAL, not a truncation (the
# env file says so for production), so this is the number the corpus must fit in.
MAX_MODEL_LEN="${FRANK_QWEN38_MAX_MODEL_LEN:-32768}"
if ! [[ "$MAX_MODEL_LEN" =~ ^[0-9]+$ ]]; then
    echo "REFUSED: FRANK_QWEN38_MAX_MODEL_LEN must be a token count (got: $MAX_MODEL_LEN)" >&2
    exit 2
fi

# Where the n-gram table lives. On the bf16 payload it is 95.37 GiB and `ram`
# PINS it: the calibration boot died at "startup stopped: out of memory" with
# 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available (measured 2026-10-29),
# while the reference recording and every score boot had already succeeded with
# the same pool and slots. The difference that matters is the job: calib only
# needs the routed weights prefilled, so it reads the table from the container's
# own file tier and the pin never happens; kldref and score keep production's
# placement, because their numbers are only comparable when every other term of
# the shape is identical. FRANK_QWEN38_NGRAM_PLACEMENT overrides either default.
case "$MODE" in
    calib) NGRAM_PLACEMENT="${FRANK_QWEN38_NGRAM_PLACEMENT:-disk}" ;;
    *)     NGRAM_PLACEMENT="${FRANK_QWEN38_NGRAM_PLACEMENT:-ram}" ;;
esac
case "$NGRAM_PLACEMENT" in
    auto|vram|ram|disk) ;;
    *) echo "REFUSED: FRANK_QWEN38_NGRAM_PLACEMENT must be auto|vram|ram|disk (got: $NGRAM_PLACEMENT)" >&2
       exit 2 ;;
esac

# Every container this campaign boots is read through the ro /models mount, so
# the host path must live under it; the container-side spelling is what goes
# into the argv. Same rule for the container a score pass opens.
under_models() {
    case "$1" in
        "$MODELS"/*) printf '/models/%s' "${1#"$MODELS"/}" ;;
        *) return 1 ;;
    esac
}

# same_dir A B — the two name one directory, judged only when BOTH exist: a dry
# run has no fixtures on disk, and two absent paths are not the same place.
same_dir() {
    [ -d "$1" ] && [ -d "$2" ] && [ "$(cd "$1" && pwd)" = "$(cd "$2" && pwd)" ]
}

if [ "$MODE" = score ]; then
    SCORE_RAD="${POS[0]}"
    SCORE_OUT="${POS[1]}"
    OUT_DIR="$(dirname "$SCORE_OUT")"
    MODEL_IN="$(under_models "$SCORE_RAD")" || {
        echo "REFUSED: container to score $SCORE_RAD is not under the mounted models dir $MODELS — the engine cannot read it" >&2
        exit 2
    }
else
    MODEL_IN="$(under_models "$RAD")" || {
        echo "REFUSED: reference container $RAD is not under the mounted models dir $MODELS — the engine cannot read it" >&2
        exit 2
    }
fi

# The measuring shape. Identical for kldref and score by construction — this
# array is the only place the campaign's KL numbers are defined.
ARGS=(--tp 2 --placement expert_tiered --weights-disk-tier
      --host-pool-mib "$HOST_POOL_MIB" --gpu-headroom-mib "$GPU_HEADROOM_MIB"
      --deterministic --num-speculative-tokens 0
      --kv-cache-dtype fp8 --max-model-len "$MAX_MODEL_LEN"
      --ngram-placement "$NGRAM_PLACEMENT"
      --override-chat-template /models/tpl.jinja)
MOUNTS=(-v "$MODELS:/models:ro")
TAPENV=()
case "$MODE" in
  kldref)
    # Recording is prefill-only over four slots at the campaign's single step
    # size, which is frank-archon.env's FRANK_ARCHON_MAX_BATCHED_TOKENS (2,048) —
    # see the header: the file tier's ceiling and the slot count both come from
    # the file that owns them. Corpus read-only, record dir rw.
    ARGS+=(--model "$MODEL_IN"
           --kld-record /data/kld-ref --kld-corpus /data/kld.jsonl
           --max-num-seqs "$MAX_SEQS" --max-num-batched-tokens "$MNBT")
    MOUNTS+=(-v "$KLD_CORPUS:/data/kld.jsonl:ro"
             -v "$KLD_REF_DIR:/data/kld-ref")
    ;;
  score)
    # The same argv with three words changed: which container, and that this
    # time the engine reads a recording and writes a report instead of the
    # reverse. The reference dir mounts READ-ONLY — a scoring pass must not be
    # able to touch the zero point — and the report gets its own writable bind.
    if [ "$OUT_DIR" = "$KLD_REF_DIR" ] || same_dir "$OUT_DIR" "$KLD_REF_DIR" \
       || [ "$(realpath -m "$OUT_DIR")" = "$(realpath -m "$KLD_REF_DIR")" ]; then
        echo "REFUSED: score output $SCORE_OUT is inside the reference dir $KLD_REF_DIR, which this mode mounts read-only" >&2
        exit 2
    fi
    ARGS+=(--model "$MODEL_IN"
           --kld-ref /data/kld-ref --kld-out "/data/out/$(basename "$SCORE_OUT")"
           --max-num-seqs "$MAX_SEQS" --max-num-batched-tokens "$MNBT")
    MOUNTS+=(-v "$KLD_REF_DIR:/data/kld-ref:ro"
             -v "$OUT_DIR:/data/out")
    ;;
  calib)
    # Calibration rides the shared step like every other mode. The old
    # exemption — eight slots at MNBT 65,536 for "free throughput", because
    # "a disk-tier step costs the same at 1 token as at 64k" — is false: the
    # step's activation arena is billed on the card, not on the disk. The
    # engine's own ledger for this container: at MNBT 1,024 the arena is
    # 504 MiB and the card claims 30.87 GiB (weights 28.90 GiB of it); at
    # 65,536 the arena is 10.65 GiB (fixed region 1.25 GiB), the claim falls
    # to 16.85 GiB, the experts shrink to 10.98 GiB and the KV to 271 MiB —
    # one slot's worth of cache for eight slots, and the boot dies at
    # "startup stopped: out of memory" (measured 2026-10-29). A wide step
    # buys throughput with the expert slab, so price it like any other
    # VRAM knob. The slots follow frank-archon.env's FRANK_ARCHON_MAX_SEQS
    # through MAX_SEQS, exactly as kldref and score do.
    ARGS+=(--model "$MODEL_IN" --port 7770
           --max-num-seqs "$CALIB_SEQS" --max-num-batched-tokens "$CALIB_MNBT")
    # The tap is only real if this directory exists and the engine can write it:
    # without the check a typo'd CALIB_DIR auto-creates a root-owned empty dir in
    # whatever path was given, and the run burns hours serving prompts into a tap
    # nobody reads. RADIANCE_CALIB_TOKENS is the whole corpus' floor, not a round's.
    [ "$DRY" = 1 ] || [ -w "$CALIB_DIR" ] || {
        echo "REFUSED: calib tap dir $CALIB_DIR is not writable (mkdir it, chown it, then re-run)" >&2
        exit 2
    }
    MOUNTS+=(-v "$CALIB_DIR:/data/calib")
    TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib
            -e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-12000000}")
    # The tap's own authorization. A pusher must see this marker before it sends
    # a prompt (calib-push.py --tap-marker), it names the container so a push
    # aimed anywhere else refuses, and a boot that fails removes it again below.
    # Measured 2026-10-29: with no marker, the chain pushed 4 rounds x 10M tokens
    # into RESTORED PRODUCTION after the rig's boot died, and nothing recorded.
    if [ "$DRY" = 1 ]; then
        :
    else
        printf 'container=%s\nimage=%s\nmode=calib\nstarted=%s\npid=%s\n' \
            "$C" "$IMAGE" "$(date -Is)" "$$" > "$CALIB_DIR/.rig-boot"
    fi
    ;;
esac

CMD=(podman run --rm --name "$C"
     --network host --init
     --security-opt seccomp=unconfined
     --ulimit memlock=-1:-1
     --device /dev/kfd --device /dev/dri
     "${MOUNTS[@]}" "${TAPENV[@]}"
     "$IMAGE" "${ARGS[@]}")

# ---- gate: the window must hold the corpus (kldref alone can meet it) --------
# score reads a recording instead of a corpus, and every arm declares at the
# checkpoint's training context, so neither can meet a sequence too long to boot.
# Measured on the raw JSON line — the pessimistic direction, and the same figure
# the census in the header quotes. It runs on a dry run too, because a refusal
# this cheap should never have to wait for the drained window to be discovered.
if [ "$MODE" = kldref ] && [ -f "$KLD_CORPUS" ]; then
    read -r REC_NO REC_CHARS < <(awk '{ if (length($0) > best) { best = length($0); where = NR } } END { print where + 0, best + 0 }' "$KLD_CORPUS")
    LIMIT_CHARS=$(( MAX_MODEL_LEN * 3 ))
    if [ "$REC_CHARS" -gt "$LIMIT_CHARS" ]; then
        echo "REFUSED: record $REC_NO of $KLD_CORPUS is $REC_CHARS chars, past the $LIMIT_CHARS a $MAX_MODEL_LEN-token window promises at the pessimistic chars/3 — raise FRANK_QWEN38_MAX_MODEL_LEN, and the container's declared --max-ctx with it, before booting the reference" >&2
        exit 2
    fi
    echo "corpus fits: longest record $REC_CHARS chars (number $REC_NO) inside the $MAX_MODEL_LEN-token window (limit $LIMIT_CHARS chars)"
fi

if [ "$DRY" = 1 ]; then
    echo "dry run: $MODE — the production unit is left running, nothing is mounted, nothing is written"
    printf '%q\n' "${CMD[@]}"
    exit 0
fi

# ---- gates that only a real run needs ---------------------------------------
if [ "$MODE" = score ]; then
    if [ ! -d "$KLD_REF_DIR" ]; then
        echo "REFUSED: no KLD reference at $KLD_REF_DIR — record it first in a drained window: scripts/run-qwen38-bf16.sh kldref" >&2
        exit 2
    fi
    if [ ! -f "$SCORE_RAD" ]; then
        echo "REFUSED: no container to score at $SCORE_RAD" >&2
        exit 2
    fi
    if [ -e "$SCORE_OUT" ]; then
        echo "REFUSED: $SCORE_OUT already exists — one report per name; move the old one to trash before re-scoring" >&2
        exit 2
    fi
    mkdir -p "$OUT_DIR"
else
    if [ ! -f "$RAD" ]; then
        echo "REFUSED: no reference container at $RAD — build it first: scripts/build-qwen38-bf16-rad.sh" >&2
        exit 2
    fi
    [ "$MODE" = kldref ] && [ ! -f "$KLD_CORPUS" ] && {
        echo "REFUSED: no KLD corpus at $KLD_CORPUS — the record would be empty" >&2
        exit 2
    }
fi
if [ ! -f "$TEMPLATE" ]; then
    echo "REFUSED: no chat template at $TEMPLATE" >&2
    exit 2
fi
mkdir -p "$LOG_DIR"
# The prod template, staged where the ro /models mount reaches it: a reference
# recorded with the container's baked template scores a different tokenisation
# than the arms served with froggeric's, and the KL would be measuring that.
cp -f "$TEMPLATE" "$TPL" || {
    echo "REFUSED: cannot stage the chat template at $TPL (the container reads it through /models)" >&2
    exit 2
}
[ "$MODE" = kldref ] && mkdir -p "$KLD_REF_DIR"
[ "$MODE" = calib ]  && mkdir -p "$CALIB_DIR"

# Stop the campaign container BEFORE handing the cards back: a killed run leaves
# its conmon alive (--rm only fires when podman itself exits), and production
# then boots beside a container still holding the GPUs — measured 2026-10-29,
# where the service reported `active` while its engine starved on the rig's VRAM.
restore() {
    podman rm -f "$C" >/dev/null 2>&1 || true
    sudo systemctl start "$SERVICE" >/dev/null 2>&1 || true
}
trap restore EXIT
echo "run-qwen38-bf16 $MODE: stopping $SERVICE ($(date -Is)); log: $LOG"
sudo systemctl stop "$SERVICE" >/dev/null 2>&1 || true
podman rm -f "$C" >/dev/null 2>&1 || true

"${CMD[@]}" 2>&1 | tee "$LOG"
rc=${PIPESTATUS[0]}
# A boot that failed must not leave the tap marked as rig-authorized: the marker
# is what calib-push.py trusts, and the point of it is that a dead rig can never
# be pushed into again.
if [ "$MODE" = calib ] && [ "$rc" -ne 0 ] && [ -f "$CALIB_DIR/.rig-boot" ]; then
    mv -f "$CALIB_DIR/.rig-boot" "$CALIB_DIR/.rig-boot.failed-$(date +%Y%m%d-%H%M%S)" 2>/dev/null || true
    echo "the rig boot failed (rc=$rc): the tap marker is retired, so no pusher can take this tap for a live rig"
fi
echo "run-qwen38-bf16 $MODE exited rc=$rc; $SERVICE restored; log: $LOG"
exit "$rc"