serverAppConfigs/scripts/run-qwen38-bf16.sh

#!/usr/bin/env bash
# run-qwen38-bf16.sh — RUNS ON DANKSERVER. Boot a container over the campaign's
# ONE measuring shape for the phases that need both R9700s. The engine owns the
# cards, so frank-archon is stopped first and restored on every exit path — the
# autoresearch pattern: stop, never mask (the unit file in /etc/systemd/system
# is a symlink into the checkout and mask fails on a relinked unit; the deploy
# restores it).
#
#   kldref            --kld-record the reference log-probabilities: the engine
#                     runs /home/dank/corpora/out/kld.jsonl (197 prompts, 50,432
#                     scored positions, score tail 256) through prefill, writes
#                     /home/dank/campaign/qwen38-w5nl64/kld-ref, and EXITS. The
#                     model behind it is the BF16 container from
#                     scripts/build-qwen38-bf16-rad.sh — the vendor checkpoint's
#                     own numbers, which is what makes it the zero point.
#   score RAD OUT     --kld-ref score ONE container against that recording and
#                     EXIT: the FP8-native anchor, the served iq4r, and every
#                     arm. Deliberately the SAME engine pin, the same TP, the
#                     same step size, the same fp8 KV, --deterministic and no
#                     speculation as kldref — KL and top-1 agreement are only
#                     comparable across boots when everything except the weights
#                     is identical, so this mode is one argv with --model and
#                     the two --kld-* flags changed, never a second shape.
#                     RAD must sit under the models dir (the container reads it
#                     through the ro /models mount); OUT lands on a writable
#                     mount because --kld-out F also writes F.rows beside it.
#   calib             serve on :7770 with the calibration tap open
#                     (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS=12000000). A
#                     separate driver — scripts/calib-push.py --port 7770
#                     --container qwen38-bf16 --corpus
#                     /home/dank/corpora/out/calib.jsonl — pushes the corpus; at
#                     12M pooled positions the engine writes the Grams and the
#                     tap switches itself off (calib-push prints TAP_DONE). Stop
#                     this script after that; the trap brings frank-archon back.
#
# Shape: the vendor's bf16 calibration lines — expert_tiered placement with the
# weights disk tier, deterministic (the resident set pinned to the loader's plan,
# so a recorded reference and a scored candidate see the same one), no
# speculation, fp8 KV, a 32,768-token window, the PLE n-gram table pinned to RAM,
# and the production chat template rather than the one baked into the container —
# every scored container is asked the same tokenised questions the reference
# answered. The window is 32,768 because this corpus is long: measured on
# Dankserver 2026-10-27, kld.jsonl is 197 records with a longest of 38,263
# characters (longest prompt FIELD 36,855, median record 27,228), which is ~9,600
# tokens at the pessimistic chars/4 and ~7,700 at the ratio this repo actually
# measured on the K3 corpus (380,143 chars = 76,568 tokens, chars/4.97). A 16,384
# window cleared that by 1.7x; 32,768 clears every record of both corpora by at
# least 2.6x, and the kldref gate below re-checks it against the file on disk.
#
# NO --vram-kv-mib: it was REMOVED at 1.3.0 and naming it is a boot refusal, not
# a no-op — verified on the pinned image 2026-10-27: "E --vram-kv-mib was
# removed: the card is split between the weights and the cache from what the
# model needs." The budget resolver derives the per-card split from
# --host-pool-mib and --gpu-headroom-mib, and those two are deliberately NOT
# production's 49,152 / 1,536 (frank-archon.env:77): the pool decides WHICH
# experts stay resident, so it buys speed and never logits — the shape the gate
# has to hold identical is the kv dtype, the slot count, the step size, the spec
# depth and the window. It is also the only figure that fits: a 335 GiB bf16
# container cannot be booted against a 49 GiB resident set on a 192 GiB machine,
# and a prefill-only pass does not care that the disk tier paces at ~3 GiB/s.
#
# The step size is ONE value for the recording and every scoring pass:
# FRANK_QWEN38_MNBT, default 2,048. The gate's whole claim is that the reference,
# the served iq4r and every arm rode one shape, so the ceiling is set by the
# TIGHTEST payload the gate boots rather than by the bf16 reference: the quantised
# expert FILE tiers (the served iq4r, the arms) stream only at MNBT<=2048 and
# refuse the widened form above it, and 2,048 is what production holds anyway.
# Recording the reference at 32,768 would therefore measure a shape the payloads
# cannot reproduce; at 2,048 the 50,432 positions cost more steps, which is still
# one drained window. Calibration keeps the wide 65,536 — grams are not logits,
# nothing downstream is compared against a step size, and that is where a big
# step pays (a disk-tier step costs the same at 1 token as at 64k).
#
# usage: scripts/run-qwen38-bf16.sh kldref|calib|score RAD OUT [--dry-run] [--image REF]
#   (RAD and OUT are score's two positionals: RAD is a container under the models
#    dir, OUT the report path whose directory is created if missing)
#   --dry-run   print the exact `podman run` argv, one shell-quoted argument a
#               line, and exit 0: the service is NOT stopped, nothing is
#               mounted, nothing is written — the offline test seam. It still
#               validates the argv's own shape (paths through the /models mount,
#               a step size that is a number), because an argv that could not
#               boot is not worth printing.
#   --image REF override the digest pin (a probe boot, or the test seam)
#
# knobs: FRANK_QWEN38_MODELS (the dir mounted at /models), FRANK_QWEN38_RAD (the
#        bf16 container, kldref only), FRANK_QWEN38_CAMPAIGN (holds kld-ref/ and
#        calib/), FRANK_QWEN38_KLD_CORPUS, FRANK_QWEN38_TPL (the template copy
#        the container reads through /models), FRANK_QWEN38_LOG_DIR, and
#        FRANK_QWEN38_MNBT — the step size shared by kldref and score (2,048 is
#        the file tier's ceiling and production's own value, four slots and all),
#        and FRANK_QWEN38_MAX_MODEL_LEN — the window every mode boots at (32,768;
#        see the corpus census above, and raise it together with the corpus).
# kldref refuses (exit 2) when the longest record of the corpus it is about to
# read is longer than MAX_MODEL_LEN x 3 characters — chars/3 being the
# pessimistic token:char ratio, so the script says no before the engine finds out
# inside a drained window, after the checkpoint work is already done. score
# refuses when the reference dir is absent, when RAD is not on disk or not under
# the models mount, or when OUT already exists: the trash law means a second
# score over the same name is a deliberate act, and two numbers in one file is
# how a campaign loses its own history.
# Engine stdout goes to $LOG_DIR/qwen38_bf16_<mode>.log — the boot ledger is the
# record of what the resolver actually gave the cards; the journal never sees it.
# exit: the engine's code; 2 = refused before the service was touched
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "$SCRIPT_DIR/frank-archon.env"
IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}"

MODE=""
POS=()
DRY=0
while [ $# -gt 0 ]; do
    case "$1" in
        kldref|calib|score) MODE="$1" ;;
        --dry-run)          DRY=1 ;;
        --image)            IMAGE="$2"; shift ;;
        *)
            # score's two positional arguments, and nothing else may be bare.
            if [ "$MODE" = score ] && [ "${#POS[@]}" -lt 2 ]; then
                POS+=("$1")
            else
                echo "usage: ${0##*/} kldref|calib|score RAD OUT [--dry-run] [--image REF]  (unknown argument: $1)" >&2
                exit 2
            fi ;;
    esac
    shift
done
if [ -z "$MODE" ]; then
    echo "usage: ${0##*/} kldref|calib|score RAD OUT [--dry-run] [--image REF]  (no mode given)" >&2
    exit 2
fi
if [ "$MODE" = score ] && [ "${#POS[@]}" -ne 2 ]; then
    echo "usage: ${0##*/} score RAD OUT  — RAD is the container to score, OUT the report to write (got ${#POS[@]} argument(s))" >&2
    exit 2
fi

MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
KLD_CORPUS="${FRANK_QWEN38_KLD_CORPUS:-/home/dank/corpora/out/kld.jsonl}"
RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}"
TPL="${FRANK_QWEN38_TPL:-$MODELS/tpl.jinja}"
TEMPLATE="$SCRIPT_DIR/qwen38-froggeric-v22.5.jinja"
LOG_DIR="${FRANK_QWEN38_LOG_DIR:-/home/dank/logs}"
LOG="$LOG_DIR/qwen38_bf16_${MODE}.log"
KLD_REF_DIR="$CAMPAIGN/kld-ref"
CALIB_DIR="$CAMPAIGN/calib"
C=qwen38-bf16
SERVICE=frank-archon

# ONE step size for both logits modes. The file tier streams only at <=2,048 and
# the gate scores file-tier payloads (the served iq4r, the arms) against the
# reference, so the reference is recorded at the ceiling the payloads can meet —
# never two numbers that a KL between them would have to forgive.
MNBT="${FRANK_QWEN38_MNBT:-2048}"
if ! [[ "$MNBT" =~ ^[0-9]+$ ]]; then
    echo "REFUSED: FRANK_QWEN38_MNBT must be a token count (got: $MNBT)" >&2
    exit 2
fi

# The window every mode boots at. Over-length is a REFUSAL, not a truncation (the
# env file says so for production), so this is the number the corpus must fit in.
MAX_MODEL_LEN="${FRANK_QWEN38_MAX_MODEL_LEN:-32768}"
if ! [[ "$MAX_MODEL_LEN" =~ ^[0-9]+$ ]]; then
    echo "REFUSED: FRANK_QWEN38_MAX_MODEL_LEN must be a token count (got: $MAX_MODEL_LEN)" >&2
    exit 2
fi

# Every container this campaign boots is read through the ro /models mount, so
# the host path must live under it; the container-side spelling is what goes
# into the argv. Same rule for the container a score pass opens.
under_models() {
    case "$1" in
        "$MODELS"/*) printf '/models/%s' "${1#"$MODELS"/}" ;;
        *) return 1 ;;
    esac
}

# same_dir A B — the two name one directory, judged only when BOTH exist: a dry
# run has no fixtures on disk, and two absent paths are not the same place.
same_dir() {
    [ -d "$1" ] && [ -d "$2" ] && [ "$(cd "$1" && pwd)" = "$(cd "$2" && pwd)" ]
}

if [ "$MODE" = score ]; then
    SCORE_RAD="${POS[0]}"
    SCORE_OUT="${POS[1]}"
    OUT_DIR="$(dirname "$SCORE_OUT")"
    MODEL_IN="$(under_models "$SCORE_RAD")" || {
        echo "REFUSED: container to score $SCORE_RAD is not under the mounted models dir $MODELS — the engine cannot read it" >&2
        exit 2
    }
else
    MODEL_IN="$(under_models "$RAD")" || {
        echo "REFUSED: reference container $RAD is not under the mounted models dir $MODELS — the engine cannot read it" >&2
        exit 2
    }
fi

# The measuring shape. Identical for kldref and score by construction — this
# array is the only place the campaign's KL numbers are defined.
ARGS=(--tp 2 --placement expert_tiered --weights-disk-tier
      --host-pool-mib 12288 --gpu-headroom-mib 768
      --deterministic --num-speculative-tokens 0
      --kv-cache-dtype fp8 --max-model-len "$MAX_MODEL_LEN"
      --ngram-placement ram
      --override-chat-template /models/tpl.jinja)
MOUNTS=(-v "$MODELS:/models:ro")
TAPENV=()
case "$MODE" in
  kldref)
    # Recording is prefill-only over four slots and the shared step, which is
    # production's own operating point (frank-archon.env: FRANK_ARCHON_MAX_SEQS=4
    # with FRANK_ARCHON_MAX_BATCHED_TOKENS=2048). Corpus read-only, record dir rw.
    ARGS+=(--model "$MODEL_IN"
           --kld-record /data/kld-ref --kld-corpus /data/kld.jsonl
           --max-num-seqs 4 --max-num-batched-tokens "$MNBT")
    MOUNTS+=(-v "$KLD_CORPUS:/data/kld.jsonl:ro"
             -v "$KLD_REF_DIR:/data/kld-ref")
    ;;
  score)
    # The same argv with three words changed: which container, and that this
    # time the engine reads a recording and writes a report instead of the
    # reverse. The reference dir mounts READ-ONLY — a scoring pass must not be
    # able to touch the zero point — and the report gets its own writable bind.
    if [ "$OUT_DIR" = "$KLD_REF_DIR" ] || same_dir "$OUT_DIR" "$KLD_REF_DIR"; then
        echo "REFUSED: score output $SCORE_OUT is inside the reference dir $KLD_REF_DIR, which this mode mounts read-only" >&2
        exit 2
    fi
    ARGS+=(--model "$MODEL_IN"
           --kld-ref /data/kld-ref --kld-out "/data/out/$(basename "$SCORE_OUT")"
           --max-num-seqs 4 --max-num-batched-tokens "$MNBT")
    MOUNTS+=(-v "$KLD_REF_DIR:/data/kld-ref:ro"
             -v "$OUT_DIR:/data/out")
    ;;
  calib)
    # Calibration is exempt from the shared step: the tap counts pooled
    # positions and nothing downstream is compared against a step size, so the
    # wide step is free throughput (a disk-tier step costs the same at 1 token
    # as at 64k).
    ARGS+=(--model "$MODEL_IN" --port 7770
           --max-num-seqs 8 --max-num-batched-tokens 65536)
    MOUNTS+=(-v "$CALIB_DIR:/data/calib")
    TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib
            -e RADIANCE_CALIB_TOKENS=12000000)
    ;;
esac

CMD=(podman run --rm --name "$C"
     --network host --init
     --security-opt seccomp=unconfined
     --ulimit memlock=-1:-1
     --device /dev/kfd --device /dev/dri
     "${MOUNTS[@]}" "${TAPENV[@]}"
     "$IMAGE" "${ARGS[@]}")

# ---- gate: the window must hold the corpus (kldref alone can meet it) --------
# score reads a recording instead of a corpus, and every arm declares at the
# checkpoint's training context, so neither can meet a sequence too long to boot.
# Measured on the raw JSON line — the pessimistic direction, and the same figure
# the census in the header quotes. It runs on a dry run too, because a refusal
# this cheap should never have to wait for the drained window to be discovered.
if [ "$MODE" = kldref ] && [ -f "$KLD_CORPUS" ]; then
    read -r REC_NO REC_CHARS < <(awk '{ if (length($0) > best) { best = length($0); where = NR } } END { print where + 0, best + 0 }' "$KLD_CORPUS")
    LIMIT_CHARS=$(( MAX_MODEL_LEN * 3 ))
    if [ "$REC_CHARS" -gt "$LIMIT_CHARS" ]; then
        echo "REFUSED: record $REC_NO of $KLD_CORPUS is $REC_CHARS chars, past the $LIMIT_CHARS a $MAX_MODEL_LEN-token window promises at the pessimistic chars/3 — raise FRANK_QWEN38_MAX_MODEL_LEN, and the container's declared --max-ctx with it, before booting the reference" >&2
        exit 2
    fi
    echo "corpus fits: longest record $REC_CHARS chars (number $REC_NO) inside the $MAX_MODEL_LEN-token window (limit $LIMIT_CHARS chars)"
fi

if [ "$DRY" = 1 ]; then
    echo "dry run: $MODE — the production unit is left running, nothing is mounted, nothing is written"
    printf '%q\n' "${CMD[@]}"
    exit 0
fi

# ---- gates that only a real run needs ---------------------------------------
if [ "$MODE" = score ]; then
    if [ ! -d "$KLD_REF_DIR" ]; then
        echo "REFUSED: no KLD reference at $KLD_REF_DIR — record it first in a drained window: scripts/run-qwen38-bf16.sh kldref" >&2
        exit 2
    fi
    if [ ! -f "$SCORE_RAD" ]; then
        echo "REFUSED: no container to score at $SCORE_RAD" >&2
        exit 2
    fi
    if [ -e "$SCORE_OUT" ]; then
        echo "REFUSED: $SCORE_OUT already exists — one report per name; move the old one to trash before re-scoring" >&2
        exit 2
    fi
    mkdir -p "$OUT_DIR"
else
    if [ ! -f "$RAD" ]; then
        echo "REFUSED: no reference container at $RAD — build it first: scripts/build-qwen38-bf16-rad.sh" >&2
        exit 2
    fi
    [ "$MODE" = kldref ] && [ ! -f "$KLD_CORPUS" ] && {
        echo "REFUSED: no KLD corpus at $KLD_CORPUS — the record would be empty" >&2
        exit 2
    }
fi
if [ ! -f "$TEMPLATE" ]; then
    echo "REFUSED: no chat template at $TEMPLATE" >&2
    exit 2
fi
mkdir -p "$LOG_DIR"
# The prod template, staged where the ro /models mount reaches it: a reference
# recorded with the container's baked template scores a different tokenisation
# than the arms served with froggeric's, and the KL would be measuring that.
cp -f "$TEMPLATE" "$TPL" || {
    echo "REFUSED: cannot stage the chat template at $TPL (the container reads it through /models)" >&2
    exit 2
}
[ "$MODE" = kldref ] && mkdir -p "$KLD_REF_DIR"
[ "$MODE" = calib ]  && mkdir -p "$CALIB_DIR"

restore() { sudo systemctl start "$SERVICE" >/dev/null 2>&1 || true; }
trap restore EXIT
echo "run-qwen38-bf16 $MODE: stopping $SERVICE ($(date -Is)); log: $LOG"
sudo systemctl stop "$SERVICE" >/dev/null 2>&1 || true
podman rm -f "$C" >/dev/null 2>&1 || true

"${CMD[@]}" 2>&1 | tee "$LOG"
rc=${PIPESTATUS[0]}
echo "run-qwen38-bf16 $MODE exited rc=$rc; $SERVICE restored; log: $LOG"
exit "$rc"