diff --git a/scripts/build-qwen38-bf16-rad.sh b/scripts/build-qwen38-bf16-rad.sh new file mode 100755 index 0000000..2defea7 --- /dev/null +++ b/scripts/build-qwen38-bf16-rad.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# build-qwen38-bf16-rad.sh — RUNS ON DANKSERVER. rad-convert the vendor BF16 +# checkpoint /home/dank/models/Qwen3.8-Flash-Next into +# /home/dank/models/rad/qwen38-bf16.rad with NO recipe and NO --reuse: a weight +# no rule matches keeps the checkpoint's encoding, so every tensor stays the +# bf16 it is in the safetensors and the container is the checkpoint's own +# numbers behind a .rad header. +# +# This file is the campaign's measuring stick, twice over: scripts/ +# run-qwen38-bf16.sh kldref records the KLD reference logits from it (every +# arm's --kld-ref is scored against that record) and the same container serves +# with the calibration tap to write the Grams that derive the protected-expert +# list and feed GPTQ. A reference that is not the vendor checkpoint, byte for +# byte in bf16, silently moves every gate in the campaign — hence no recipe, no +# --reuse, and the four refusal gates below. +# +# Why a container at all: a checkpoint dir has no mapped/file tiers, and a +# weight only a host kernel reads (the 47.7 GiB PLE n-gram table) cannot be +# served from one; rad-convert also bakes the tokenizer. The convert runs on +# the HOST backend — no --device /dev/kfd, no /dev/dri — so declare resolves +# kernels on CPU and production serving is untouched for the whole run (the +# precedent that proved it: scripts/convert-swift15-w4dn8.sh). +# +# Gates, every one a refusal with exit 2 and a named reason, BEFORE anything is +# written, cheapest first — the verifier runs last of them because its Hub gate +# hashes 343 GiB and must not be paid for by a run that was going to refuse on +# the output file anyway. +# checkpoint the tree must be on disk; an absent dir names the command that +# makes one (scripts/download-qwen38-bf16.sh --fetch) and stops. +# free space the convert writes ~343,323 MiB next to the ~343,323 MiB +# checkpoint; QWEN38_MIN_FREE_MIB (default 350000 = that figure +# plus 6.5 GiB of slack) is the floor. The volume that cannot +# hold both is discovered at minute 2, not hour 5. +# output the campaign converts the reference ONCE. An existing +# qwen38-bf16.rad is refused; --force retires it to +# ~/.AGENTTRASH/-qwen38-bf16.rad (trash law: nothing is +# deleted, and the move is a rename on the same volume). +# verified scripts/download-qwen38-bf16.sh's verify mode must pass. A +# failed verify exits 1, not 2: the tree being the wrong payload +# is a result, a missing environment is a refusal. +# plan rad-convert --plan-only -v first — the oracle for "does this +# shape boot". A refused plan costs seconds and saves a convert. +# +# usage: scripts/build-qwen38-bf16-rad.sh [--dry-run] [--plan-only] [--force] +# --dry-run print the argv it would exec (podman wrapping rad-convert), +# one shell-quoted argument a line, and exit 0. No gate runs, no +# container starts, nothing is written — the offline test seam. +# It does resolve the checkpoint and output paths through the +# /models mount, because an argv that carries host paths would +# not run either. +# --plan-only run the plan and every gate except the disk and output ones — +# those two protect bytes the plan never writes, and this is the +# probe to run when the disk gate itself is what refuses +# --force retire an existing output container to the trash dir first +# +# knobs: FRANK_QWEN38_MODELS the host dir mounted at /models (default +# /home/dank/models); FRANK_QWEN38_CKPT (default $MODELS/ +# Qwen3.8-Flash-Next, the same knob the verifier reads); +# FRANK_QWEN38_BF16_OUT (default $MODELS/rad/qwen38-bf16.rad); +# QWEN38_MIN_FREE_MIB; QWEN38_TRASH (default $HOME/.AGENTTRASH). The +# verifier's own census knobs (FRANK_QWEN38_SHARDS / _BYTES / _REPO) +# reach it through the environment untouched. +# exit: 0 converted (or planned), 1 the checkpoint failed verification, +# 2 refused before anything was written +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# ONE owner of the engine pin: the digest frank-archon serves on. The reference +# must be built by the SAME engine build that scores it, or a KLD delta is +# partly a version delta. +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/frank-archon.env" +IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}" + +MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}" +CKPT="${FRANK_QWEN38_CKPT:-$MODELS/Qwen3.8-Flash-Next}" +OUT="${FRANK_QWEN38_BF16_OUT:-$MODELS/rad/qwen38-bf16.rad}" +TRASH="${QWEN38_TRASH:-$HOME/.AGENTTRASH}" +MIN_FREE_MIB="${QWEN38_MIN_FREE_MIB:-350000}" +LOG="/home/dank/logs/qwen38_bf16_convert.log" + +DRY=0 +PLAN_ONLY=0 +FORCE=0 +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) DRY=1 ;; + --plan-only) PLAN_ONLY=1 ;; + --force) FORCE=1 ;; + *) echo "usage: ${0##*/} [--dry-run] [--plan-only] [--force]" >&2 + exit 2 ;; + esac + shift +done + +# The convert sees the tree only through the /models mount, so both paths must +# be under it; the container-side spelling is what goes into the argv. +case "$CKPT" in + "$MODELS"/*) CKPT_IN="/models/${CKPT#"$MODELS"/}" ;; + *) echo "REFUSED: checkpoint $CKPT is not under the mounted models dir $MODELS — the convert cannot see it" >&2 + exit 2 ;; +esac +case "$OUT" in + "$MODELS"/*) OUT_IN="/models/${OUT#"$MODELS"/}" ;; + *) echo "REFUSED: output $OUT is not under the mounted models dir $MODELS — rad-convert could not write it" >&2 + exit 2 ;; +esac + +# Declare with NO recipe line — that is what keeps every tensor bf16 — and at the +# bound this container will actually be served at. Both flags come off the pinned +# image's own help: --kv-cache-dtype at convert "only picks the kernels it +# checks" and the container is identical either way, so naming fp8 is free +# insurance — if the fp8 KV kernels the fleet serves would not resolve, the +# convert refuses here instead of the boot failing on a drained window. +# --max-ctx is the bound declare is run at, whose default is the checkpoint's +# training context (262,144 here, whose fp8 cache is 258.8 GiB against a 192 GiB +# machine, so the default is not an option). 32,768 is set by the corpus, not by +# taste: kld.jsonl's longest record is 38,263 characters — ~9,600 tokens at the +# pessimistic chars/4, ~7,700 at the chars/4.97 this repo measured on the K3 +# corpus — so the window the rig boots at clears every record 2.6x over and the +# reference boot cannot die on a sequence that will not fit. The rig's window is +# one knob (FRANK_QWEN38_MAX_MODEL_LEN in scripts/run-qwen38-bf16.sh, default +# 32,768) and that script refuses a corpus its window cannot hold, so raising one +# without the other is caught before the drained window, not inside it. The +# payload containers the gate scores declare at the training context because +# production must hold the 200k shape; a declared bound is a declare-time check, +# not a weight change, so the reference and the payloads carry the same numbers +# whatever each declared at — which is why one can be measured against the other. +CONVERT_ARGS=("$CKPT_IN" -o "$OUT_IN" --max-ctx 32768 --kv-cache-dtype fp8) +PODMAN=(podman run --rm --entrypoint rad-convert -v "$MODELS:/models") +PLAN_CMD=("${PODMAN[@]}" --name qwen38-bf16-plan "$IMAGE" "${CONVERT_ARGS[@]}" --plan-only -v) +CONV_CMD=("${PODMAN[@]}" --name qwen38-bf16-convert "$IMAGE" "${CONVERT_ARGS[@]}") + +if [ "$DRY" = "1" ]; then + echo "dry run: no gate checked, no container started, nothing written" + echo "# $CKPT -> $OUT (the convert sees them as $CKPT_IN and $OUT_IN)" + echo "# plan argv:" + printf '%q\n' "${PLAN_CMD[@]}" + echo "# convert argv:" + printf '%q\n' "${CONV_CMD[@]}" + exit 0 +fi + +# ---- gate: the checkpoint is on disk --------------------------------------- +if [ ! -d "$CKPT" ]; then + echo "REFUSED: no checkpoint at $CKPT — fetch and verify it first: scripts/download-qwen38-bf16.sh --fetch" >&2 + exit 2 +fi + +# ---- gate: peak disk -------------------------------------------------------- +# The deepest existing ancestor is the right thing to ask: a fresh +# /home/dank/models/rad has no free space of its own to report. +OUT_DIR="$(dirname "$OUT")" +PROBE="$OUT_DIR" +while [ ! -d "$PROBE" ] && [ "$PROBE" != "/" ]; do PROBE="$(dirname "$PROBE")"; done +FREE_MIB="$(df --output=avail -m "$PROBE" | tail -1 | tr -d ' ')" +if ! [[ "$FREE_MIB" =~ ^[0-9]+$ ]]; then + echo "REFUSED: cannot read the free space under $PROBE (df said: $FREE_MIB)" >&2 + exit 2 +fi +if [ "$PLAN_ONLY" != "1" ] && [ "$FREE_MIB" -lt "$MIN_FREE_MIB" ]; then + echo "REFUSED peak-disk: $FREE_MIB MiB free under $PROBE, $MIN_FREE_MIB MiB wanted (QWEN38_MIN_FREE_MIB) — the bf16 container rewrites the checkpoint's own 343,323 MiB of tensors beside it" >&2 + exit 2 +fi + +# ---- gate: one conversion --------------------------------------------------- +if [ -e "$OUT" ]; then + if [ "$PLAN_ONLY" = "1" ]; then + echo "note: $OUT already exists; --plan-only writes nothing, so it stays" + elif [ "$FORCE" = "1" ]; then + mkdir -p "$TRASH" + DEST="$TRASH/$(date -u +%Y%m%dT%H%M%SZ)-$(basename "$OUT")" + mv "$OUT" "$DEST" + echo "TRASHED: retired the existing reference container $OUT -> $DEST" + else + echo "REFUSED: $OUT already exists — the campaign converts the reference once; re-run with --force to retire it to $TRASH first" >&2 + exit 2 + fi +fi + +# ---- gate: the tree really is the checkpoint (the full verify) ------------- +echo "== verifying $CKPT (scripts/download-qwen38-bf16.sh, full gates)" +rc=0 +bash "$SCRIPT_DIR/download-qwen38-bf16.sh" --dir "$CKPT" || rc=$? +if [ "$rc" -ne 0 ]; then + if [ "$rc" -eq 2 ]; then + echo "REFUSED: the checkpoint verifier refused the environment (rc=2); nothing was converted" >&2 + exit 2 + fi + echo "REFUSED: $CKPT failed verification (rc=$rc) — a half tree converts to a half reference" >&2 + exit 1 +fi + +# ---- plan, then convert ------------------------------------------------------ +mkdir -p "$(dirname "$LOG")" +# rad-convert opens -o during declare as well, so the output directory exists +# before the plan; that empty directory is all this creates ahead of it. +mkdir -p "$OUT_DIR" +echo "== rad-convert --plan-only -v (declare + plan; writes nothing)" +"${PLAN_CMD[@]}" 2>&1 | tee -a "$LOG" +rc=${PIPESTATUS[0]} +if [ "$rc" -ne 0 ]; then + echo "REFUSED plan: rad-convert --plan-only exited rc=$rc — the engine refuses this encoding/shape, no container was written (log: $LOG)" >&2 + exit 2 +fi +if [ "$PLAN_ONLY" = "1" ]; then + echo "PLAN_OK: $CKPT_IN declares for $OUT_IN; no container written (log: $LOG)" + exit 0 +fi + +echo "== convert: $CKPT -> $OUT ($(date -Is))" +"${CONV_CMD[@]}" 2>&1 | tee -a "$LOG" +rc=${PIPESTATUS[0]} +if [ "$rc" -ne 0 ]; then + echo "rad-convert rc=$rc; log: $LOG" >&2 + exit "$rc" +fi +MIB=$(( $(stat -c %s "$OUT") / 1048576 )) +echo "BUILD_DONE: $OUT is ${MIB} MiB (log: $LOG)" +echo "NEXT: scripts/run-qwen38-bf16.sh kldref records the reference logits from it" diff --git a/scripts/run-qwen38-bf16.sh b/scripts/run-qwen38-bf16.sh new file mode 100755 index 0000000..5568ef4 --- /dev/null +++ b/scripts/run-qwen38-bf16.sh @@ -0,0 +1,329 @@ +#!/usr/bin/env bash +# run-qwen38-bf16.sh — RUNS ON DANKSERVER. Boot a container over the campaign's +# ONE measuring shape for the phases that need both R9700s. The engine owns the +# cards, so frank-archon is stopped first and restored on every exit path — the +# autoresearch pattern: stop, never mask (the unit file in /etc/systemd/system +# is a symlink into the checkout and mask fails on a relinked unit; the deploy +# restores it). +# +# kldref --kld-record the reference log-probabilities: the engine +# runs /home/dank/corpora/out/kld.jsonl (197 prompts, 50,432 +# scored positions, score tail 256) through prefill, writes +# /home/dank/campaign/qwen38-w5nl64/kld-ref, and EXITS. The +# model behind it is the BF16 container from +# scripts/build-qwen38-bf16-rad.sh — the vendor checkpoint's +# own numbers, which is what makes it the zero point. +# score RAD OUT --kld-ref score ONE container against that recording and +# EXIT: the FP8-native anchor, the served iq4r, and every +# arm. Deliberately the SAME engine pin, the same TP, the +# same step size, the same fp8 KV, --deterministic and no +# speculation as kldref — KL and top-1 agreement are only +# comparable across boots when everything except the weights +# is identical, so this mode is one argv with --model and +# the two --kld-* flags changed, never a second shape. +# RAD must sit under the models dir (the container reads it +# through the ro /models mount); OUT lands on a writable +# mount because --kld-out F also writes F.rows beside it. +# calib serve on :7770 with the calibration tap open +# (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS=12000000). A +# separate driver — scripts/calib-push.py --port 7770 +# --container qwen38-bf16 --corpus +# /home/dank/corpora/out/calib.jsonl — pushes the corpus; at +# 12M pooled positions the engine writes the Grams and the +# tap switches itself off (calib-push prints TAP_DONE). Stop +# this script after that; the trap brings frank-archon back. +# +# Shape: the vendor's bf16 calibration lines — expert_tiered placement with the +# weights disk tier, deterministic (the resident set pinned to the loader's plan, +# so a recorded reference and a scored candidate see the same one), no +# speculation, fp8 KV, a 32,768-token window, the PLE n-gram table pinned to RAM, +# and the production chat template rather than the one baked into the container — +# every scored container is asked the same tokenised questions the reference +# answered. The window is 32,768 because this corpus is long: measured on +# Dankserver 2026-10-27, kld.jsonl is 197 records with a longest of 38,263 +# characters (longest prompt FIELD 36,855, median record 27,228), which is ~9,600 +# tokens at the pessimistic chars/4 and ~7,700 at the ratio this repo actually +# measured on the K3 corpus (380,143 chars = 76,568 tokens, chars/4.97). A 16,384 +# window cleared that by 1.7x; 32,768 clears every record of both corpora by at +# least 2.6x, and the kldref gate below re-checks it against the file on disk. +# +# NO --vram-kv-mib: it was REMOVED at 1.3.0 and naming it is a boot refusal, not +# a no-op — verified on the pinned image 2026-10-27: "E --vram-kv-mib was +# removed: the card is split between the weights and the cache from what the +# model needs." The budget resolver derives the per-card split from +# --host-pool-mib and --gpu-headroom-mib, and those two are deliberately NOT +# production's 49,152 / 1,536 (frank-archon.env:77): the pool decides WHICH +# experts stay resident, so it buys speed and never logits — the shape the gate +# has to hold identical is the kv dtype, the slot count, the step size, the spec +# depth and the window. It is also the only figure that fits: a 335 GiB bf16 +# container cannot be booted against a 49 GiB resident set on a 192 GiB machine, +# and a prefill-only pass does not care that the disk tier paces at ~3 GiB/s. +# +# The step size is ONE value for the recording and every scoring pass: +# FRANK_QWEN38_MNBT, default 2,048. The gate's whole claim is that the reference, +# the served iq4r and every arm rode one shape, so the ceiling is set by the +# TIGHTEST payload the gate boots rather than by the bf16 reference: the quantised +# expert FILE tiers (the served iq4r, the arms) stream only at MNBT<=2048 and +# refuse the widened form above it, and 2,048 is what production holds anyway. +# Recording the reference at 32,768 would therefore measure a shape the payloads +# cannot reproduce; at 2,048 the 50,432 positions cost more steps, which is still +# one drained window. Calibration keeps the wide 65,536 — grams are not logits, +# nothing downstream is compared against a step size, and that is where a big +# step pays (a disk-tier step costs the same at 1 token as at 64k). +# +# usage: scripts/run-qwen38-bf16.sh kldref|calib|score RAD OUT [--dry-run] [--image REF] +# (RAD and OUT are score's two positionals: RAD is a container under the models +# dir, OUT the report path whose directory is created if missing) +# --dry-run print the exact `podman run` argv, one shell-quoted argument a +# line, and exit 0: the service is NOT stopped, nothing is +# mounted, nothing is written — the offline test seam. It still +# validates the argv's own shape (paths through the /models mount, +# a step size that is a number), because an argv that could not +# boot is not worth printing. +# --image REF override the digest pin (a probe boot, or the test seam) +# +# knobs: FRANK_QWEN38_MODELS (the dir mounted at /models), FRANK_QWEN38_RAD (the +# bf16 container, kldref only), FRANK_QWEN38_CAMPAIGN (holds kld-ref/ and +# calib/), FRANK_QWEN38_KLD_CORPUS, FRANK_QWEN38_TPL (the template copy +# the container reads through /models), FRANK_QWEN38_LOG_DIR, and +# FRANK_QWEN38_MNBT — the step size shared by kldref and score (2,048 is +# the file tier's ceiling and production's own value, four slots and all), +# and FRANK_QWEN38_MAX_MODEL_LEN — the window every mode boots at (32,768; +# see the corpus census above, and raise it together with the corpus). +# kldref refuses (exit 2) when the longest record of the corpus it is about to +# read is longer than MAX_MODEL_LEN x 3 characters — chars/3 being the +# pessimistic token:char ratio, so the script says no before the engine finds out +# inside a drained window, after the checkpoint work is already done. score +# refuses when the reference dir is absent, when RAD is not on disk or not under +# the models mount, or when OUT already exists: the trash law means a second +# score over the same name is a deliberate act, and two numbers in one file is +# how a campaign loses its own history. +# Engine stdout goes to $LOG_DIR/qwen38_bf16_.log — the boot ledger is the +# record of what the resolver actually gave the cards; the journal never sees it. +# exit: the engine's code; 2 = refused before the service was touched +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/frank-archon.env" +IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}" + +MODE="" +POS=() +DRY=0 +while [ $# -gt 0 ]; do + case "$1" in + kldref|calib|score) MODE="$1" ;; + --dry-run) DRY=1 ;; + --image) IMAGE="$2"; shift ;; + *) + # score's two positional arguments, and nothing else may be bare. + if [ "$MODE" = score ] && [ "${#POS[@]}" -lt 2 ]; then + POS+=("$1") + else + echo "usage: ${0##*/} kldref|calib|score RAD OUT [--dry-run] [--image REF] (unknown argument: $1)" >&2 + exit 2 + fi ;; + esac + shift +done +if [ -z "$MODE" ]; then + echo "usage: ${0##*/} kldref|calib|score RAD OUT [--dry-run] [--image REF] (no mode given)" >&2 + exit 2 +fi +if [ "$MODE" = score ] && [ "${#POS[@]}" -ne 2 ]; then + echo "usage: ${0##*/} score RAD OUT — RAD is the container to score, OUT the report to write (got ${#POS[@]} argument(s))" >&2 + exit 2 +fi + +MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}" +CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}" +KLD_CORPUS="${FRANK_QWEN38_KLD_CORPUS:-/home/dank/corpora/out/kld.jsonl}" +RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}" +TPL="${FRANK_QWEN38_TPL:-$MODELS/tpl.jinja}" +TEMPLATE="$SCRIPT_DIR/qwen38-froggeric-v22.5.jinja" +LOG_DIR="${FRANK_QWEN38_LOG_DIR:-/home/dank/logs}" +LOG="$LOG_DIR/qwen38_bf16_${MODE}.log" +KLD_REF_DIR="$CAMPAIGN/kld-ref" +CALIB_DIR="$CAMPAIGN/calib" +C=qwen38-bf16 +SERVICE=frank-archon + +# ONE step size for both logits modes. The file tier streams only at <=2,048 and +# the gate scores file-tier payloads (the served iq4r, the arms) against the +# reference, so the reference is recorded at the ceiling the payloads can meet — +# never two numbers that a KL between them would have to forgive. +MNBT="${FRANK_QWEN38_MNBT:-2048}" +if ! [[ "$MNBT" =~ ^[0-9]+$ ]]; then + echo "REFUSED: FRANK_QWEN38_MNBT must be a token count (got: $MNBT)" >&2 + exit 2 +fi + +# The window every mode boots at. Over-length is a REFUSAL, not a truncation (the +# env file says so for production), so this is the number the corpus must fit in. +MAX_MODEL_LEN="${FRANK_QWEN38_MAX_MODEL_LEN:-32768}" +if ! [[ "$MAX_MODEL_LEN" =~ ^[0-9]+$ ]]; then + echo "REFUSED: FRANK_QWEN38_MAX_MODEL_LEN must be a token count (got: $MAX_MODEL_LEN)" >&2 + exit 2 +fi + +# Every container this campaign boots is read through the ro /models mount, so +# the host path must live under it; the container-side spelling is what goes +# into the argv. Same rule for the container a score pass opens. +under_models() { + case "$1" in + "$MODELS"/*) printf '/models/%s' "${1#"$MODELS"/}" ;; + *) return 1 ;; + esac +} + +# same_dir A B — the two name one directory, judged only when BOTH exist: a dry +# run has no fixtures on disk, and two absent paths are not the same place. +same_dir() { + [ -d "$1" ] && [ -d "$2" ] && [ "$(cd "$1" && pwd)" = "$(cd "$2" && pwd)" ] +} + +if [ "$MODE" = score ]; then + SCORE_RAD="${POS[0]}" + SCORE_OUT="${POS[1]}" + OUT_DIR="$(dirname "$SCORE_OUT")" + MODEL_IN="$(under_models "$SCORE_RAD")" || { + echo "REFUSED: container to score $SCORE_RAD is not under the mounted models dir $MODELS — the engine cannot read it" >&2 + exit 2 + } +else + MODEL_IN="$(under_models "$RAD")" || { + echo "REFUSED: reference container $RAD is not under the mounted models dir $MODELS — the engine cannot read it" >&2 + exit 2 + } +fi + +# The measuring shape. Identical for kldref and score by construction — this +# array is the only place the campaign's KL numbers are defined. +ARGS=(--tp 2 --placement expert_tiered --weights-disk-tier + --host-pool-mib 12288 --gpu-headroom-mib 768 + --deterministic --num-speculative-tokens 0 + --kv-cache-dtype fp8 --max-model-len "$MAX_MODEL_LEN" + --ngram-placement ram + --override-chat-template /models/tpl.jinja) +MOUNTS=(-v "$MODELS:/models:ro") +TAPENV=() +case "$MODE" in + kldref) + # Recording is prefill-only over four slots and the shared step, which is + # production's own operating point (frank-archon.env: FRANK_ARCHON_MAX_SEQS=4 + # with FRANK_ARCHON_MAX_BATCHED_TOKENS=2048). Corpus read-only, record dir rw. + ARGS+=(--model "$MODEL_IN" + --kld-record /data/kld-ref --kld-corpus /data/kld.jsonl + --max-num-seqs 4 --max-num-batched-tokens "$MNBT") + MOUNTS+=(-v "$KLD_CORPUS:/data/kld.jsonl:ro" + -v "$KLD_REF_DIR:/data/kld-ref") + ;; + score) + # The same argv with three words changed: which container, and that this + # time the engine reads a recording and writes a report instead of the + # reverse. The reference dir mounts READ-ONLY — a scoring pass must not be + # able to touch the zero point — and the report gets its own writable bind. + if [ "$OUT_DIR" = "$KLD_REF_DIR" ] || same_dir "$OUT_DIR" "$KLD_REF_DIR"; then + echo "REFUSED: score output $SCORE_OUT is inside the reference dir $KLD_REF_DIR, which this mode mounts read-only" >&2 + exit 2 + fi + ARGS+=(--model "$MODEL_IN" + --kld-ref /data/kld-ref --kld-out "/data/out/$(basename "$SCORE_OUT")" + --max-num-seqs 4 --max-num-batched-tokens "$MNBT") + MOUNTS+=(-v "$KLD_REF_DIR:/data/kld-ref:ro" + -v "$OUT_DIR:/data/out") + ;; + calib) + # Calibration is exempt from the shared step: the tap counts pooled + # positions and nothing downstream is compared against a step size, so the + # wide step is free throughput (a disk-tier step costs the same at 1 token + # as at 64k). + ARGS+=(--model "$MODEL_IN" --port 7770 + --max-num-seqs 8 --max-num-batched-tokens 65536) + MOUNTS+=(-v "$CALIB_DIR:/data/calib") + TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib + -e RADIANCE_CALIB_TOKENS=12000000) + ;; +esac + +CMD=(podman run --rm --name "$C" + --network host --init + --security-opt seccomp=unconfined + --ulimit memlock=-1:-1 + --device /dev/kfd --device /dev/dri + "${MOUNTS[@]}" "${TAPENV[@]}" + "$IMAGE" "${ARGS[@]}") + +# ---- gate: the window must hold the corpus (kldref alone can meet it) -------- +# score reads a recording instead of a corpus, and every arm declares at the +# checkpoint's training context, so neither can meet a sequence too long to boot. +# Measured on the raw JSON line — the pessimistic direction, and the same figure +# the census in the header quotes. It runs on a dry run too, because a refusal +# this cheap should never have to wait for the drained window to be discovered. +if [ "$MODE" = kldref ] && [ -f "$KLD_CORPUS" ]; then + read -r REC_NO REC_CHARS < <(awk '{ if (length($0) > best) { best = length($0); where = NR } } END { print where + 0, best + 0 }' "$KLD_CORPUS") + LIMIT_CHARS=$(( MAX_MODEL_LEN * 3 )) + if [ "$REC_CHARS" -gt "$LIMIT_CHARS" ]; then + echo "REFUSED: record $REC_NO of $KLD_CORPUS is $REC_CHARS chars, past the $LIMIT_CHARS a $MAX_MODEL_LEN-token window promises at the pessimistic chars/3 — raise FRANK_QWEN38_MAX_MODEL_LEN, and the container's declared --max-ctx with it, before booting the reference" >&2 + exit 2 + fi + echo "corpus fits: longest record $REC_CHARS chars (number $REC_NO) inside the $MAX_MODEL_LEN-token window (limit $LIMIT_CHARS chars)" +fi + +if [ "$DRY" = 1 ]; then + echo "dry run: $MODE — the production unit is left running, nothing is mounted, nothing is written" + printf '%q\n' "${CMD[@]}" + exit 0 +fi + +# ---- gates that only a real run needs --------------------------------------- +if [ "$MODE" = score ]; then + if [ ! -d "$KLD_REF_DIR" ]; then + echo "REFUSED: no KLD reference at $KLD_REF_DIR — record it first in a drained window: scripts/run-qwen38-bf16.sh kldref" >&2 + exit 2 + fi + if [ ! -f "$SCORE_RAD" ]; then + echo "REFUSED: no container to score at $SCORE_RAD" >&2 + exit 2 + fi + if [ -e "$SCORE_OUT" ]; then + echo "REFUSED: $SCORE_OUT already exists — one report per name; move the old one to trash before re-scoring" >&2 + exit 2 + fi + mkdir -p "$OUT_DIR" +else + if [ ! -f "$RAD" ]; then + echo "REFUSED: no reference container at $RAD — build it first: scripts/build-qwen38-bf16-rad.sh" >&2 + exit 2 + fi + [ "$MODE" = kldref ] && [ ! -f "$KLD_CORPUS" ] && { + echo "REFUSED: no KLD corpus at $KLD_CORPUS — the record would be empty" >&2 + exit 2 + } +fi +if [ ! -f "$TEMPLATE" ]; then + echo "REFUSED: no chat template at $TEMPLATE" >&2 + exit 2 +fi +mkdir -p "$LOG_DIR" +# The prod template, staged where the ro /models mount reaches it: a reference +# recorded with the container's baked template scores a different tokenisation +# than the arms served with froggeric's, and the KL would be measuring that. +cp -f "$TEMPLATE" "$TPL" || { + echo "REFUSED: cannot stage the chat template at $TPL (the container reads it through /models)" >&2 + exit 2 +} +[ "$MODE" = kldref ] && mkdir -p "$KLD_REF_DIR" +[ "$MODE" = calib ] && mkdir -p "$CALIB_DIR" + +restore() { sudo systemctl start "$SERVICE" >/dev/null 2>&1 || true; } +trap restore EXIT +echo "run-qwen38-bf16 $MODE: stopping $SERVICE ($(date -Is)); log: $LOG" +sudo systemctl stop "$SERVICE" >/dev/null 2>&1 || true +podman rm -f "$C" >/dev/null 2>&1 || true + +"${CMD[@]}" 2>&1 | tee "$LOG" +rc=${PIPESTATUS[0]} +echo "run-qwen38-bf16 $MODE exited rc=$rc; $SERVICE restored; log: $LOG" +exit "$rc" diff --git a/tests/qwen38_reference.bats b/tests/qwen38_reference.bats new file mode 100755 index 0000000..b999fb0 --- /dev/null +++ b/tests/qwen38_reference.bats @@ -0,0 +1,337 @@ +#!/usr/bin/env bats +# Seam: the BF16 REFERENCE container of the Qwen3.8-Flash-Next five-bit +# campaign (feat/qwen38-w5nl64) — scripts/build-qwen38-bf16-rad.sh and +# scripts/run-qwen38-bf16.sh. +# +# The reference is what every later stage is judged or derived from (the KLD +# record scores the arms, the Grams derive the protected experts), so the +# deliverable here is the set of refusals: a half checkpoint, a second convert, +# a volume that cannot hold the convert, an engine-refused plan, and any run — +# including a dry one — that would take the GPUs from frank-archon. Each test +# drives the script's own exit status and printed argv. Offline: no engine, no +# GPU, no container, no systemctl, no network. Path knobs aim at +# $BATS_TEST_TMPDIR. +# +# The third mode, score, carries the campaign's comparability law: its engine +# argv must equal kldref's except for --model and the --kld-* pair, or the KL +# numbers of the FP8-native anchor, the served payload and the arms are not one +# measurement series. The seam proves that by diffing the two dry runs. + +# The 1.8.1 law asserted in two places below: --vram-kv-mib was REMOVED at +# 1.3.0, so naming it is a boot refusal — the reference serve must carry the +# budget-resolver knobs (--host-pool-mib / --gpu-headroom-mib) instead. + +load helpers + +BUILD="$REPO_ROOT/scripts/build-qwen38-bf16-rad.sh" +RUNNER="$REPO_ROOT/scripts/run-qwen38-bf16.sh" + +# The reference shape, spelled exactly as run-qwen38-bf16.sh must pass it: this +# is the contract the recorded reference is measured at, so a silent edit here +# is a silent edit to every KLD number in the campaign. +SHAPE="--tp 2 --placement expert_tiered --weights-disk-tier --host-pool-mib 12288 --gpu-headroom-mib 768 --deterministic --num-speculative-tokens 0 --kv-cache-dtype fp8 --max-model-len 32768 --ngram-placement ram --override-chat-template /models/tpl.jinja" + +# oneline — both dry runs print one shell-quoted argument a line; joining them +# lets an option and its value be asserted as the pair they are. +oneline() { local s="${output//$'\n'/ }"; printf '%s' "$s"; } + +# engine_argv — the arguments AFTER the image line of a dry run, with --model +# and every --kld-* flag and its value removed: what is left is the measuring +# shape, and it must be identical between kldref and score. +engine_argv() { + local seen=0 skip=0 a + while IFS= read -r a; do + if [ "$seen" = 0 ]; then + [[ "$a" == *"radiance@sha256:"* ]] && seen=1 + continue + fi + case "$a" in + --model|--kld-record|--kld-corpus|--kld-ref|--kld-out) skip=1; continue ;; + esac + if [ "$skip" = 1 ]; then skip=0; continue; fi + printf '%s\n' "$a" + done <<< "$1" +} + +# flag_value FLAG — the value a dry run printed for FLAG (the argv is one +# argument a line, so the value is the line after it). Lets a test ask what +# number the script chose instead of re-declaring the literal it guards. +flag_value() { + local want="$1" prev="" a + while IFS= read -r a; do + if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi + prev="$a" + done + return 1 +} + +@test "the build dry run prints the rad-convert argv for the checkpoint and the bf16 output" { + M="$BATS_TEST_TMPDIR/models" + run env FRANK_QWEN38_MODELS="$M" bash "$BUILD" --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"rad-convert"* ]] + # the argv is the container's view: the models dir rides the /models mount + [[ "$output" == *"$M:/models"* ]] + flat="$(oneline)" + [[ "$flat" == *"/models/Qwen3.8-Flash-Next -o /models/rad/qwen38-bf16.rad"* ]] + # a reference is the checkpoint re-headed, not re-encoded, and it declares a + # window the long KLD corpus can be read inside of + [[ "$output" != *"--recipe"* ]] + [[ "$output" != *"--reuse"* ]] + [[ "$flat" == *"--max-ctx 32768"* ]] + # and a dry run looked at nothing: the fixture dirs were never created + [ ! -e "$M" ] +} + +@test "the build refuses a checkpoint that is not on disk and names the fetch command" { + M="$BATS_TEST_TMPDIR/models" + run env FRANK_QWEN38_MODELS="$M" bash "$BUILD" + [ "$status" -eq 2 ] + [[ "$output" == *"$M/Qwen3.8-Flash-Next"* ]] + [[ "$output" == *"scripts/download-qwen38-bf16.sh --fetch"* ]] + [ ! -e "$M/rad/qwen38-bf16.rad" ] +} + +@test "the build refuses an existing reference container and names --force as the way past it" { + M="$BATS_TEST_TMPDIR/models" + mkdir -p "$M/Qwen3.8-Flash-Next" "$M/rad" + echo "not a container" > "$M/rad/qwen38-bf16.rad" + run env FRANK_QWEN38_MODELS="$M" QWEN38_MIN_FREE_MIB=1 \ + QWEN38_TRASH="$BATS_TEST_TMPDIR/trash" bash "$BUILD" + [ "$status" -eq 2 ] + [[ "$output" == *"$M/rad/qwen38-bf16.rad"* ]] + [[ "$output" == *"--force"* ]] + # refusing retires nothing: the trash law means the file survives a refusal + [ -f "$M/rad/qwen38-bf16.rad" ] + [ ! -e "$BATS_TEST_TMPDIR/trash" ] +} + +@test "the build refuses when the output filesystem cannot hold a second 335 GiB" { + # The campaign's peak-disk law: 343,323 MiB of container written next to + # 343,323 MiB of checkpoint. The wanted figure is set above whatever this + # box really has, which is what the gate reads. + M="$BATS_TEST_TMPDIR/models" + mkdir -p "$M/Qwen3.8-Flash-Next" "$M/rad" + run env FRANK_QWEN38_MODELS="$M" QWEN38_MIN_FREE_MIB=999999999 bash "$BUILD" + [ "$status" -eq 2 ] + [[ "$output" == *"REFUSED"* ]] + [[ "$output" =~ [0-9]+" MiB free" ]] + [[ "$output" == *"999999999 MiB wanted"* ]] + # the disk gate is the cheap one: it refuses long before the Hub sweep runs + [[ "$output" != *"VERIFY_OK"* ]] + [ ! -e "$M/rad/qwen38-bf16.rad" ] +} + +@test "the reference serve rejects an unknown mode and names every valid mode" { + # `probe` is the swift15 era's third mode; it is retired and must not boot + # a container here either. + run bash "$RUNNER" probe + [ "$status" -eq 2 ] + [[ "$output" == *"probe"* ]] + [[ "$output" == *"kldref"* ]] + [[ "$output" == *"calib"* ]] + [[ "$output" == *"score"* ]] + [[ "$output" != *"podman"* ]] + run bash "$RUNNER" + [ "$status" -eq 2 ] + [[ "$output" == *"kldref"* ]] + [[ "$output" == *"calib"* ]] + [[ "$output" == *"score"* ]] + # score without its two positionals is the same usage refusal + run bash "$RUNNER" score + [ "$status" -eq 2 ] + [[ "$output" == *"score RAD OUT"* ]] +} + +@test "the kldref dry run carries the record flags at the reference shape and no --vram-kv-mib" { + run bash "$RUNNER" kldref --dry-run + [ "$status" -eq 0 ] + flat="$(oneline)" + [[ "$flat" == *"--kld-record /data/kld-ref"* ]] + [[ "$flat" == *"--kld-corpus /data/kld.jsonl"* ]] + # the corpus arrives read-only, the record dir writable + [[ "$flat" == *":/data/kld.jsonl:ro"* ]] + [[ "$flat" == *":/data/kld-ref "* ]] + [[ "$flat" == *"$SHAPE"* ]] + # 1.8.1: the flag does not exist — naming it is a boot refusal + [[ "$output" != *"--vram-kv-mib"* ]] +} + +@test "the calib dry run opens the tap at 12M tokens and serves on 7770" { + run bash "$RUNNER" calib --dry-run + [ "$status" -eq 0 ] + flat="$(oneline)" + [[ "$flat" == *"-e RADIANCE_CALIB_TOKENS=12000000"* ]] + [[ "$flat" == *"-e RADIANCE_CALIB_DIR=/data/calib"* ]] + [[ "$flat" == *"--port 7770"* ]] + [[ "$flat" == *":/data/calib "* ]] + # calib serves; it does not record + [[ "$output" != *"--kld-record"* ]] +} + +@test "a dry run prints argv only: no drain, no container, nothing written" { + L="$BATS_TEST_TMPDIR/logs" + run env FRANK_QWEN38_LOG_DIR="$L" \ + bash "$RUNNER" kldref --dry-run --image example.invalid/radiance@sha256:0000000000000000000000000000000000000000000000000000000000000000 + [ "$status" -eq 0 ] + [[ "$output" != *"systemctl"* ]] + [[ "$output" != *"frank-archon stopped"* ]] + [[ "$output" != *"podman rm"* ]] + # not vacuous: the podman argv WAS printed, and --image replaced the pin + [[ "$output" == *"podman"* ]] + [[ "$output" == *"example.invalid/radiance@sha256:000"* ]] + [[ "$output" != *"stilldeadcode/radiance"* ]] + [ ! -d "$L" ] +} + +@test "the score dry run is the kldref argv with only --model and the --kld-* pair changed" { + M="$BATS_TEST_TMPDIR/models" + C="$BATS_TEST_TMPDIR/campaign" + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \ + bash "$RUNNER" kldref --dry-run + [ "$status" -eq 0 ] + ref_argv="$(engine_argv "$output")" + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \ + bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run + [ "$status" -eq 0 ] + score_argv="$(engine_argv "$output")" + # The whole value of the campaign's three KL numbers is that they rode this + # one shape: same pin, --tp 2, fp8 KV, step size, --deterministic, no spec. + if [[ "$score_argv" != "$ref_argv" ]]; then + echo "--- kldref engine argv ---"; echo "$ref_argv" + echo "--- score engine argv ---"; echo "$score_argv" + return 1 + fi + flat="$(oneline)" + [[ "$flat" == *"--model /models/rad/arm-w5nl64.rad"* ]] + [[ "$flat" == *"--kld-ref /data/kld-ref --kld-out /data/out/arm1.json"* ]] + # the zero point mounts read-only; the report needs a writable bind because + # --kld-out F also writes F.rows beside it + [[ "$flat" == *":/data/kld-ref:ro "* ]] + [[ "$flat" == *":/data/out "* ]] + [[ "$output" != *"--kld-record"* ]] + [[ "$output" != *"--kld-corpus"* ]] + [[ "$output" != *"--vram-kv-mib"* ]] +} + +@test "score refuses a missing reference, a missing container and an existing report" { + M="$BATS_TEST_TMPDIR/models" + C="$BATS_TEST_TMPDIR/campaign" + L="$BATS_TEST_TMPDIR/logs" + mkdir -p "$M/rad" + : > "$M/rad/arm-w5nl64.rad" + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \ + bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" + [ "$status" -eq 2 ] + [[ "$output" == *"$C/kld-ref"* ]] + [[ "$output" == *"run-qwen38-bf16.sh kldref"* ]] + # refused before the log dir, so before anything that could reach systemctl + [ ! -d "$L" ] + + mkdir -p "$C/kld-ref" + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \ + bash "$RUNNER" score "$M/rad/gone.rad" "$C/scores/arm1.json" + [ "$status" -eq 2 ] + [[ "$output" == *"$M/rad/gone.rad"* ]] + [ ! -d "$L" ] + + mkdir -p "$C/scores" + : > "$C/scores/arm1.json" + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \ + bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" + [ "$status" -eq 2 ] + [[ "$output" == *"$C/scores/arm1.json"* ]] + [[ "$output" == *"trash"* ]] + # the trash law: a refusal keeps the old report, it does not overwrite it + [ -f "$C/scores/arm1.json" ] + [ ! -d "$L" ] +} + +@test "the reference and a score pass ride ONE step size, legal on the file tier" { + # The gate's claim is one shape; that shape's ceiling is set by the tightest + # payload it boots, and the served iq4r and every arm are file-tier + # containers, which stream only at <=2048. A reference recorded above that is + # a KL between two shapes, not two weights. + M="$BATS_TEST_TMPDIR/models" + C="$BATS_TEST_TMPDIR/campaign" + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" bash "$RUNNER" kldref --dry-run + [ "$status" -eq 0 ] + ref_step="$(flag_value --max-num-batched-tokens <<< "$output")" + [ -n "$ref_step" ] + [ "$ref_step" -le 2048 ] + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \ + bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run + [ "$status" -eq 0 ] + score_step="$(flag_value --max-num-batched-tokens <<< "$output")" + [ "$ref_step" = "$score_step" ] + # one knob moves both, so the operator can lower it at 2 a.m. without an edit + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \ + bash "$RUNNER" kldref --dry-run + [ "$(flag_value --max-num-batched-tokens <<< "$output")" = "1024" ] + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \ + bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run + [ "$(flag_value --max-num-batched-tokens <<< "$output")" = "1024" ] + # calibration is exempt: grams are not logits, so the tap keeps the wide step + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \ + bash "$RUNNER" calib --dry-run + [ "$(flag_value --max-num-batched-tokens <<< "$output")" = "65536" ] + # a step size that is not a token count is refused, never handed to the engine + run env FRANK_QWEN38_MNBT=broad bash "$RUNNER" kldref --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"FRANK_QWEN38_MNBT"* ]] + [[ "$output" == *"must be a token count"* ]] +} + +@test "kldref refuses a corpus its window cannot hold, before the service is touched" { + M="$BATS_TEST_TMPDIR/models" R="$M/rad" + mkdir -p "$M/tpl" "$R" "$BATS_TEST_TMPDIR/out" + : > "$M/tpl/qwen38-froggeric-v22.5.jinja" + : > "$R/qwen38-bf16.rad" + CORPUS="$BATS_TEST_TMPDIR/long.jsonl" + { + printf '{"prompt":"short one"}\n' + printf '{"prompt":"%s"}\n' "$(head -c 120000 /dev/zero | tr '\0' 'x')" + printf '{"prompt":"short three"}\n' + } > "$CORPUS" + # 32,768 tokens x 3 chars is the pessimistic limit; the 120,002-char line + # that opens record 2 is well past it, so the campaign says no up front. + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \ + bash "$RUNNER" kldref --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"REFUSED"* ]] + [[ "$output" == *"record 2"* ]] + [[ "$output" == *"120013 chars"* ]] + [[ "$output" == *"past the 98304 a 32768-token window"* ]] + [[ "$output" == *"FRANK_QWEN38_MAX_MODEL_LEN"* ]] + # the same corpus is fine once the window is raised past it, and the raised + # window is what the boot is asked for + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \ + FRANK_QWEN38_MAX_MODEL_LEN=65536 FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" \ + bash "$RUNNER" kldref --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"corpus fits"* ]] + flat="$(oneline)" + [[ "$flat" == *"--max-model-len 65536"* ]] + # the shipped corpus of three short records fits the shipped window + SHORT="$BATS_TEST_TMPDIR/short.jsonl" + printf '{"prompt":"tiny"}\n{"prompt":"also tiny"}\n' > "$SHORT" + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$SHORT" \ + FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" bash "$RUNNER" kldref --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"corpus fits: longest record 22 chars (number 2)"* ]] + # a real kldref run refuses at the same place: no log dir, no drain, and the + # service would still be up when the script gave up + L="$BATS_TEST_TMPDIR/logs-real" + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \ + FRANK_QWEN38_LOG_DIR="$L" bash "$RUNNER" kldref + [ "$status" -eq 2 ] + [ ! -d "$L" ] + # and it is kldref that reads the corpus: score and calib never consult it + run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \ + FRANK_QWEN38_KLD_REF_DIR="$BATS_TEST_TMPDIR/ref" \ + FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" \ + bash "$RUNNER" score "$R/qwen38-bf16.rad" "$BATS_TEST_TMPDIR/out/arm.json" --dry-run + [ "$status" -eq 0 ] + [[ "$output" != *"corpus fits"* ]] + [[ "$output" != *"REFUSED"* ]] +}