#!/usr/bin/env bash
# build-qwen38-bf16-rad.sh — RUNS ON DANKSERVER. rad-convert the vendor BF16
# checkpoint /home/dank/models/Qwen3.8-Flash-Next into
# /home/dank/models/rad/qwen38-bf16.rad with NO recipe and NO --reuse: a weight
# no rule matches keeps the checkpoint's encoding, so every tensor stays the
# bf16 it is in the safetensors and the container is the checkpoint's own
# numbers behind a .rad header.
#
# This file is the campaign's measuring stick, twice over: scripts/
# run-qwen38-bf16.sh kldref records the KLD reference logits from it (every
# arm's --kld-ref is scored against that record) and the same container serves
# with the calibration tap to write the Grams that derive the protected-expert
# list and feed GPTQ. A reference that is not the vendor checkpoint, byte for
# byte in bf16, silently moves every gate in the campaign — hence no recipe, no
# --reuse, and the four refusal gates below.
#
# Why a container at all: a checkpoint dir has no mapped/file tiers, and a
# weight only a host kernel reads (the 47.7 GiB PLE n-gram table) cannot be
# served from one; rad-convert also bakes the tokenizer. The convert runs on
# the HOST backend — no --device /dev/kfd, no /dev/dri — so declare resolves
# kernels on CPU and production serving is untouched for the whole run (the
# precedent that proved it: scripts/convert-swift15-w4dn8.sh).
#
# Gates, every one a refusal with exit 2 and a named reason, BEFORE anything is
# written, cheapest first — the verifier runs last of them because its Hub gate
# hashes 343 GiB and must not be paid for by a run that was going to refuse on
# the output file anyway.
#   checkpoint  the tree must be on disk; an absent dir names the command that
#               makes one (scripts/download-qwen38-bf16.sh --fetch) and stops.
#   free space  the convert writes ~343,323 MiB next to the ~343,323 MiB
#               checkpoint; QWEN38_MIN_FREE_MIB (default 350000 = that figure
#               plus 6.5 GiB of slack) is the floor. The volume that cannot
#               hold both is discovered at minute 2, not hour 5.
#   output      the campaign converts the reference ONCE. An existing
#               qwen38-bf16.rad is refused; --force retires it to
#               ~/.AGENTTRASH/<utc>-qwen38-bf16.rad (trash law: nothing is
#               deleted, and the move is a rename on the same volume).
#   verified    scripts/download-qwen38-bf16.sh's verify mode must pass. A
#               failed verify exits 1, not 2: the tree being the wrong payload
#               is a result, a missing environment is a refusal.
#   plan        rad-convert --plan-only -v first — the oracle for "does this
#               shape boot". A refused plan costs seconds and saves a convert.
#
# usage: scripts/build-qwen38-bf16-rad.sh [--dry-run] [--plan-only] [--force]
#   --dry-run    print the argv it would exec (podman wrapping rad-convert),
#               one shell-quoted argument a line, and exit 0. No gate runs, no
#               container starts, nothing is written — the offline test seam.
#               It does resolve the checkpoint and output paths through the
#               /models mount, because an argv that carries host paths would
#               not run either.
#   --plan-only  run the plan and every gate except the disk and output ones —
#               those two protect bytes the plan never writes, and this is the
#               probe to run when the disk gate itself is what refuses
#   --force      retire an existing output container to the trash dir first
#
# knobs: FRANK_QWEN38_MODELS   the host dir mounted at /models (default
#        /home/dank/models); FRANK_QWEN38_CKPT (default $MODELS/
#        Qwen3.8-Flash-Next, the same knob the verifier reads);
#        FRANK_QWEN38_BF16_OUT (default $MODELS/rad/qwen38-bf16.rad);
#        QWEN38_MIN_FREE_MIB; QWEN38_TRASH (default $HOME/.AGENTTRASH). The
#        verifier's own census knobs (FRANK_QWEN38_SHARDS / _BYTES / _REPO)
#        reach it through the environment untouched.
# exit: 0 converted (or planned), 1 the checkpoint failed verification,
#       2 refused before anything was written
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# ONE owner of the engine pin: the digest frank-archon serves on. The reference
# must be built by the SAME engine build that scores it, or a KLD delta is
# partly a version delta.
# shellcheck disable=SC1091
source "$SCRIPT_DIR/frank-archon.env"
IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}"

MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
CKPT="${FRANK_QWEN38_CKPT:-$MODELS/Qwen3.8-Flash-Next}"
OUT="${FRANK_QWEN38_BF16_OUT:-$MODELS/rad/qwen38-bf16.rad}"
TRASH="${QWEN38_TRASH:-$HOME/.AGENTTRASH}"
MIN_FREE_MIB="${QWEN38_MIN_FREE_MIB:-350000}"
LOG="/home/dank/logs/qwen38_bf16_convert.log"

DRY=0
PLAN_ONLY=0
FORCE=0
while [ $# -gt 0 ]; do
    case "$1" in
        --dry-run)   DRY=1 ;;
        --plan-only) PLAN_ONLY=1 ;;
        --force)     FORCE=1 ;;
        *) echo "usage: ${0##*/} [--dry-run] [--plan-only] [--force]" >&2
           exit 2 ;;
    esac
    shift
done

# The convert sees the tree only through the /models mount, so both paths must
# be under it; the container-side spelling is what goes into the argv.
case "$CKPT" in
    "$MODELS"/*) CKPT_IN="/models/${CKPT#"$MODELS"/}" ;;
    *) echo "REFUSED: checkpoint $CKPT is not under the mounted models dir $MODELS — the convert cannot see it" >&2
       exit 2 ;;
esac
case "$OUT" in
    "$MODELS"/*) OUT_IN="/models/${OUT#"$MODELS"/}" ;;
    *) echo "REFUSED: output $OUT is not under the mounted models dir $MODELS — rad-convert could not write it" >&2
       exit 2 ;;
esac

# Declare with NO recipe line — that is what keeps every tensor bf16 — and at the
# bound this container will actually be served at. Both flags come off the pinned
# image's own help: --kv-cache-dtype at convert "only picks the kernels it
# checks" and the container is identical either way, so naming fp8 is free
# insurance — if the fp8 KV kernels the fleet serves would not resolve, the
# convert refuses here instead of the boot failing on a drained window.
# --max-ctx is the bound declare is run at, whose default is the checkpoint's
# training context (262,144 here, whose fp8 cache is 258.8 GiB against a 192 GiB
# machine, so the default is not an option). 32,768 is set by the corpus, not by
# taste: kld.jsonl's longest record is 38,263 characters — ~9,600 tokens at the
# pessimistic chars/4, ~7,700 at the chars/4.97 this repo measured on the K3
# corpus — so the window the rig boots at clears every record 2.6x over and the
# reference boot cannot die on a sequence that will not fit. The rig's window is
# one knob (FRANK_QWEN38_MAX_MODEL_LEN in scripts/run-qwen38-bf16.sh, default
# 32,768) and that script refuses a corpus its window cannot hold, so raising one
# without the other is caught before the drained window, not inside it. The
# payload containers the gate scores declare at the training context because
# production must hold the 200k shape; a declared bound is a declare-time check,
# not a weight change, so the reference and the payloads carry the same numbers
# whatever each declared at — which is why one can be measured against the other.
CONVERT_ARGS=("$CKPT_IN" -o "$OUT_IN" --max-ctx 32768 --kv-cache-dtype fp8)
PODMAN=(podman run --rm --entrypoint rad-convert -v "$MODELS:/models")
PLAN_CMD=("${PODMAN[@]}" --name qwen38-bf16-plan "$IMAGE" "${CONVERT_ARGS[@]}" --plan-only -v)
CONV_CMD=("${PODMAN[@]}" --name qwen38-bf16-convert "$IMAGE" "${CONVERT_ARGS[@]}")

if [ "$DRY" = "1" ]; then
    echo "dry run: no gate checked, no container started, nothing written"
    echo "# $CKPT -> $OUT (the convert sees them as $CKPT_IN and $OUT_IN)"
    echo "# plan argv:"
    printf '%q\n' "${PLAN_CMD[@]}"
    echo "# convert argv:"
    printf '%q\n' "${CONV_CMD[@]}"
    exit 0
fi

# ---- gate: the checkpoint is on disk ---------------------------------------
if [ ! -d "$CKPT" ]; then
    echo "REFUSED: no checkpoint at $CKPT — fetch and verify it first: scripts/download-qwen38-bf16.sh --fetch" >&2
    exit 2
fi

# ---- gate: peak disk --------------------------------------------------------
# The deepest existing ancestor is the right thing to ask: a fresh
# /home/dank/models/rad has no free space of its own to report.
OUT_DIR="$(dirname "$OUT")"
PROBE="$OUT_DIR"
while [ ! -d "$PROBE" ] && [ "$PROBE" != "/" ]; do PROBE="$(dirname "$PROBE")"; done
FREE_MIB="$(df --output=avail -m "$PROBE" | tail -1 | tr -d ' ')"
if ! [[ "$FREE_MIB" =~ ^[0-9]+$ ]]; then
    echo "REFUSED: cannot read the free space under $PROBE (df said: $FREE_MIB)" >&2
    exit 2
fi
if [ "$PLAN_ONLY" != "1" ] && [ "$FREE_MIB" -lt "$MIN_FREE_MIB" ]; then
    echo "REFUSED peak-disk: $FREE_MIB MiB free under $PROBE, $MIN_FREE_MIB MiB wanted (QWEN38_MIN_FREE_MIB) — the bf16 container rewrites the checkpoint's own 343,323 MiB of tensors beside it" >&2
    exit 2
fi

# ---- gate: one conversion ---------------------------------------------------
if [ -e "$OUT" ]; then
    if [ "$PLAN_ONLY" = "1" ]; then
        echo "note: $OUT already exists; --plan-only writes nothing, so it stays"
    elif [ "$FORCE" = "1" ]; then
        mkdir -p "$TRASH"
        DEST="$TRASH/$(date -u +%Y%m%dT%H%M%SZ)-$(basename "$OUT")"
        mv "$OUT" "$DEST"
        echo "TRASHED: retired the existing reference container $OUT -> $DEST"
    else
        echo "REFUSED: $OUT already exists — the campaign converts the reference once; re-run with --force to retire it to $TRASH first" >&2
        exit 2
    fi
fi

# ---- gate: the tree really is the checkpoint (the full verify) -------------
echo "== verifying $CKPT (scripts/download-qwen38-bf16.sh, full gates)"
rc=0
bash "$SCRIPT_DIR/download-qwen38-bf16.sh" --dir "$CKPT" || rc=$?
if [ "$rc" -ne 0 ]; then
    if [ "$rc" -eq 2 ]; then
        echo "REFUSED: the checkpoint verifier refused the environment (rc=2); nothing was converted" >&2
        exit 2
    fi
    echo "REFUSED: $CKPT failed verification (rc=$rc) — a half tree converts to a half reference" >&2
    exit 1
fi

# ---- plan, then convert ------------------------------------------------------
mkdir -p "$(dirname "$LOG")"
# rad-convert opens -o during declare as well, so the output directory exists
# before the plan; that empty directory is all this creates ahead of it.
mkdir -p "$OUT_DIR"
echo "== rad-convert --plan-only -v (declare + plan; writes nothing)"
"${PLAN_CMD[@]}" 2>&1 | tee -a "$LOG"
rc=${PIPESTATUS[0]}
if [ "$rc" -ne 0 ]; then
    echo "REFUSED plan: rad-convert --plan-only exited rc=$rc — the engine refuses this encoding/shape, no container was written (log: $LOG)" >&2
    exit 2
fi
if [ "$PLAN_ONLY" = "1" ]; then
    echo "PLAN_OK: $CKPT_IN declares for $OUT_IN; no container written (log: $LOG)"
    exit 0
fi

echo "== convert: $CKPT -> $OUT ($(date -Is))"
"${CONV_CMD[@]}" 2>&1 | tee -a "$LOG"
rc=${PIPESTATUS[0]}
if [ "$rc" -ne 0 ]; then
    echo "rad-convert rc=$rc; log: $LOG" >&2
    exit "$rc"
fi
MIB=$(( $(stat -c %s "$OUT") / 1048576 ))
echo "BUILD_DONE: $OUT is ${MIB} MiB (log: $LOG)"
echo "NEXT: scripts/run-qwen38-bf16.sh kldref records the reference logits from it"