#!/usr/bin/env bash
# qwen38-calib-window.sh — RUNS ON DANKSERVER. The calibration window, and nothing else:
# boot the BF16 rig, wait until it is actually serving, push the corpus until the engine says
# the tap is full, stop the rig, restore frank-archon, and bank the reference container.
#
# The reference recording is already taken (kld-ref/kld.json exists), so this window is the
# calibration tap alone — the one job that needs both cards after it. The kldref phase is NOT
# re-run here: the chained launcher that used to do that moved the banked kld-ref into trash
# first, which is how a 16-hour phase gets paid for twice.
#
# Every phase is gated on the previous one, because the last time this ran unguarded
# (2026-10-29) the rig's boot died of an OOM, the health loop quietly fell through, and the
# pusher fed four rounds of 10M tokens into RESTORED PRODUCTION with nothing recorded:
#
#   1. the runner must still be alive and its container must answer /health, or this script
#      kills it, restores the service and exits non-zero — no push, ever;
#   2. the pusher refuses on its own unless the rig wrote its marker into the tap dir, the
#      marker names this container, and the container is running (scripts/calib-push.py
#      --tap-marker), so a second lock survives a mistake in this file;
#   3. the reference container is banked into trash only when BOTH the serve and the push
#      exited 0 and the tap dir holds grams. The checkpoint that reproduces it stays.
#
# knobs: Q38_WINDOW_CALIB_CORPUS (calib-7168.jsonl — the re-chunked file the rig's pool can
#        hold), Q38_WINDOW_HEALTH_TIMEOUT (seconds to wait for /health), Q38_WINDOW_ROUNDS
#        (pusher rounds; one is 10M tokens), Q38_WINDOW_KEEP_RAD=1 to leave the .rad in place.
# Usage: scripts/qwen38-calib-window.sh [--dry-run]
set -uo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 9

DRY=0
[ "${1:-}" = "--dry-run" ] && DRY=1
say() { printf '%s %s\n' "$(date -Is)" "$*"; }

MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}"
CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}"
KLDREF="$CAMPAIGN/kld-ref/kld.json"
CALIB="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}"
HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}"
ROUNDS="${Q38_WINDOW_ROUNDS:-6}"
RIG=qwen38-bf16
SERVICE=frank-archon
PY="${PY:-python3}"

# ---- preflight: everything this window needs, before a card is touched -------
[ -f "$KLDREF" ] || { echo "REFUSED: no reference recording at $KLDREF — record it first (scripts/run-qwen38-bf16.sh kldref); this window only fills the calibration tap" >&2; exit 2; }
[ -f "$RAD" ]    || { echo "REFUSED: no BF16 reference container at $RAD — nothing to serve; scripts/build-qwen38-bf16-rad.sh" >&2; exit 2; }
[ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; }
[ -d "$CALIB" ]  || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; }
[ -w "$CALIB" ]  || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; }
# One tap seal writes ~40 GiB (measured 2026-10-30). The runner refuses too; this
# names the same number before a card is touched, so a full volume never becomes
# a 17-hour dead run.
avail_mib=$(df --output=avail -B 1M "$CALIB" 2>/dev/null | tail -1 | tr -d ' ')
[[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0
need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}"
[ "$avail_mib" -ge "$need_mib" ] || {
    echo "REFUSED: tap dir $CALIB has ${avail_mib} MiB free; one seal writes ~40 GiB and this window needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2
    exit 2
}
! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; }
! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; }

if [ "$DRY" = 1 ]; then
    echo "dry run: calib window — the service is left running, nothing is mounted or written"
    echo "would run: bash scripts/run-qwen38-bf16.sh calib          (rig boots, writes $CALIB/.rig-boot)"
    echo "would wait: /health up within ${HEALTH_TIMEOUT}s, else kill the rig, restore $SERVICE, exit 1"
    echo "would run: $PY scripts/calib-push.py --container $RIG --corpus $CORPUS --max-rounds $ROUNDS"
    echo "would bank: $RAD -> ~/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad/  (only if both phases exit 0)"
    exit 0
fi

say "calib window start pid=$$ corpus=$CORPUS rounds=$ROUNDS"
[ -n "$(ls -A "$CALIB" 2>/dev/null)" ] && say "tap dir $CALIB is not empty; the engine's token floor is pooled, so an earlier partial tap still counts"

# ---- phase 1: the rig -------------------------------------------------------
say "phase=calib serve begin"
bash scripts/run-qwen38-bf16.sh calib & spid=$!

rig_up() {
    podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true \
        && curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health
}

waited=0
until rig_up; do
    if ! kill -0 "$spid" 2>/dev/null; then
        wait "$spid"; crc=$?
        say "phase=calib serve end rc=$crc (the rig exited before it ever answered /health)"
        echo "FAIL: the rig boot died (rc=$crc) — nothing was pushed; read /home/dank/logs/qwen38_bf16_calib.log" >&2
        exit 1
    fi
    if [ "$waited" -ge "$HEALTH_TIMEOUT" ]; then
        say "phase=calib health never came after ${HEALTH_TIMEOUT}s; killing the rig"
        podman stop "$RIG" >/dev/null 2>&1 || true
        wait "$spid"
        echo "FAIL: the rig never answered /health within ${HEALTH_TIMEOUT}s — nothing was pushed" >&2
        exit 1
    fi
    sleep 15
    waited=$((waited + 15))
done
say "phase=calib health ok waited=${waited}s"

# ---- phase 2: the tap -------------------------------------------------------
say "phase=calib push begin"
"$PY" scripts/calib-push.py --port 7770 --parallel 8 --container "$RIG" \
    --tap-dir "$CALIB" --corpus "$CORPUS" --max-rounds "$ROUNDS"
prc=$?
say "phase=calib push end rc=$prc"

podman stop "$RIG" >/dev/null 2>&1 || true
wait "$spid"; crc=$?
say "phase=calib serve end rc=$crc"

if [ "$prc" -ne 0 ] || [ "$crc" -ne 0 ]; then
    echo "FAIL: serve rc=$crc push rc=$prc — the tap is NOT banked and $RAD stays: $(basename "$RAD")" >&2
    exit 1
fi
[ -n "$(ls -A "$CALIB" 2>/dev/null)" ] || { echo "FAIL: both phases exited 0 but the tap dir $CALIB is empty" >&2; exit 1; }
say "TAP BANKED: $(ls -1 "$CALIB" | wc -l) file(s) in $CALIB"

# ---- phase 3: bank the reference container ----------------------------------
if [ "${Q38_WINDOW_KEEP_RAD:-0}" = "1" ]; then
    say "reference .rad kept (Q38_WINDOW_KEEP_RAD=1)"
else
    T="${Q38_WINDOW_TRASH_ROOT:-$HOME/.AGENTTRASH}/$(date +%Y%m%d)-qwen38-bf16-rad"
    mkdir -p "$T" || { echo "FAIL: cannot create trash dir $T" >&2; exit 1; }
    mv "$RAD" "$T/" && say "reference .rad banked in $T (the checkpoint that rebuilds it stays)"
fi
say "calib window end"