feat(qwen38): the w5nl64 heavier-quant campaign toolchain (fetch, BF16 reference, arms, FP8-anchored gate) #76
No files matched your search
Executable
+220
@@ -0,0 +1,220 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# build-qwen38-bf16-rad.sh — RUNS ON DANKSERVER. rad-convert the vendor BF16
|
||||||
|
# checkpoint /home/dank/models/Qwen3.8-Flash-Next into
|
||||||
|
# /home/dank/models/rad/qwen38-bf16.rad with NO recipe and NO --reuse: a weight
|
||||||
|
# no rule matches keeps the checkpoint's encoding, so every tensor stays the
|
||||||
|
# bf16 it is in the safetensors and the container is the checkpoint's own
|
||||||
|
# numbers behind a .rad header.
|
||||||
|
#
|
||||||
|
# This file is the campaign's measuring stick, twice over: scripts/
|
||||||
|
# run-qwen38-bf16.sh kldref records the KLD reference logits from it (every
|
||||||
|
# arm's --kld-ref is scored against that record) and the same container serves
|
||||||
|
# with the calibration tap to write the Grams that derive the protected-expert
|
||||||
|
# list and feed GPTQ. A reference that is not the vendor checkpoint, byte for
|
||||||
|
# byte in bf16, silently moves every gate in the campaign — hence no recipe, no
|
||||||
|
# --reuse, and the four refusal gates below.
|
||||||
|
#
|
||||||
|
# Why a container at all: a checkpoint dir has no mapped/file tiers, and a
|
||||||
|
# weight only a host kernel reads (the 47.7 GiB PLE n-gram table) cannot be
|
||||||
|
# served from one; rad-convert also bakes the tokenizer. The convert runs on
|
||||||
|
# the HOST backend — no --device /dev/kfd, no /dev/dri — so declare resolves
|
||||||
|
# kernels on CPU and production serving is untouched for the whole run (the
|
||||||
|
# precedent that proved it: scripts/convert-swift15-w4dn8.sh).
|
||||||
|
#
|
||||||
|
# Gates, every one a refusal with exit 2 and a named reason, BEFORE anything is
|
||||||
|
# written, cheapest first — the verifier runs last of them because its Hub gate
|
||||||
|
# hashes 343 GiB and must not be paid for by a run that was going to refuse on
|
||||||
|
# the output file anyway.
|
||||||
|
# checkpoint the tree must be on disk; an absent dir names the command that
|
||||||
|
# makes one (scripts/download-qwen38-bf16.sh --fetch) and stops.
|
||||||
|
# free space the convert writes ~343,323 MiB next to the ~343,323 MiB
|
||||||
|
# checkpoint; QWEN38_MIN_FREE_MIB (default 350000 = that figure
|
||||||
|
# plus 6.5 GiB of slack) is the floor. The volume that cannot
|
||||||
|
# hold both is discovered at minute 2, not hour 5.
|
||||||
|
# output the campaign converts the reference ONCE. An existing
|
||||||
|
# qwen38-bf16.rad is refused; --force retires it to
|
||||||
|
# ~/.AGENTTRASH/<utc>-qwen38-bf16.rad (trash law: nothing is
|
||||||
|
# deleted, and the move is a rename on the same volume).
|
||||||
|
# verified scripts/download-qwen38-bf16.sh's verify mode must pass. A
|
||||||
|
# failed verify exits 1, not 2: the tree being the wrong payload
|
||||||
|
# is a result, a missing environment is a refusal.
|
||||||
|
# plan rad-convert --plan-only -v first — the oracle for "does this
|
||||||
|
# shape boot". A refused plan costs seconds and saves a convert.
|
||||||
|
#
|
||||||
|
# usage: scripts/build-qwen38-bf16-rad.sh [--dry-run] [--plan-only] [--force]
|
||||||
|
# --dry-run print the argv it would exec (podman wrapping rad-convert),
|
||||||
|
# one shell-quoted argument a line, and exit 0. No gate runs, no
|
||||||
|
# container starts, nothing is written — the offline test seam.
|
||||||
|
# It does resolve the checkpoint and output paths through the
|
||||||
|
# /models mount, because an argv that carries host paths would
|
||||||
|
# not run either.
|
||||||
|
# --plan-only run the plan and every gate except the disk and output ones —
|
||||||
|
# those two protect bytes the plan never writes, and this is the
|
||||||
|
# probe to run when the disk gate itself is what refuses
|
||||||
|
# --force retire an existing output container to the trash dir first
|
||||||
|
#
|
||||||
|
# knobs: FRANK_QWEN38_MODELS the host dir mounted at /models (default
|
||||||
|
# /home/dank/models); FRANK_QWEN38_CKPT (default $MODELS/
|
||||||
|
# Qwen3.8-Flash-Next, the same knob the verifier reads);
|
||||||
|
# FRANK_QWEN38_BF16_OUT (default $MODELS/rad/qwen38-bf16.rad);
|
||||||
|
# QWEN38_MIN_FREE_MIB; QWEN38_TRASH (default $HOME/.AGENTTRASH). The
|
||||||
|
# verifier's own census knobs (FRANK_QWEN38_SHARDS / _BYTES / _REPO)
|
||||||
|
# reach it through the environment untouched.
|
||||||
|
# exit: 0 converted (or planned), 1 the checkpoint failed verification,
|
||||||
|
# 2 refused before anything was written
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# ONE owner of the engine pin: the digest frank-archon serves on. The reference
|
||||||
|
# must be built by the SAME engine build that scores it, or a KLD delta is
|
||||||
|
# partly a version delta.
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
source "$SCRIPT_DIR/frank-archon.env"
|
||||||
|
IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}"
|
||||||
|
|
||||||
|
MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
|
||||||
|
CKPT="${FRANK_QWEN38_CKPT:-$MODELS/Qwen3.8-Flash-Next}"
|
||||||
|
OUT="${FRANK_QWEN38_BF16_OUT:-$MODELS/rad/qwen38-bf16.rad}"
|
||||||
|
TRASH="${QWEN38_TRASH:-$HOME/.AGENTTRASH}"
|
||||||
|
MIN_FREE_MIB="${QWEN38_MIN_FREE_MIB:-350000}"
|
||||||
|
LOG="/home/dank/logs/qwen38_bf16_convert.log"
|
||||||
|
|
||||||
|
DRY=0
|
||||||
|
PLAN_ONLY=0
|
||||||
|
FORCE=0
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--dry-run) DRY=1 ;;
|
||||||
|
--plan-only) PLAN_ONLY=1 ;;
|
||||||
|
--force) FORCE=1 ;;
|
||||||
|
*) echo "usage: ${0##*/} [--dry-run] [--plan-only] [--force]" >&2
|
||||||
|
exit 2 ;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
# The convert sees the tree only through the /models mount, so both paths must
|
||||||
|
# be under it; the container-side spelling is what goes into the argv.
|
||||||
|
case "$CKPT" in
|
||||||
|
"$MODELS"/*) CKPT_IN="/models/${CKPT#"$MODELS"/}" ;;
|
||||||
|
*) echo "REFUSED: checkpoint $CKPT is not under the mounted models dir $MODELS — the convert cannot see it" >&2
|
||||||
|
exit 2 ;;
|
||||||
|
esac
|
||||||
|
case "$OUT" in
|
||||||
|
"$MODELS"/*) OUT_IN="/models/${OUT#"$MODELS"/}" ;;
|
||||||
|
*) echo "REFUSED: output $OUT is not under the mounted models dir $MODELS — rad-convert could not write it" >&2
|
||||||
|
exit 2 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# Declare with NO recipe line — that is what keeps every tensor bf16 — and at the
|
||||||
|
# bound this container will actually be served at. Both flags come off the pinned
|
||||||
|
# image's own help: --kv-cache-dtype at convert "only picks the kernels it
|
||||||
|
# checks" and the container is identical either way, so naming fp8 is free
|
||||||
|
# insurance — if the fp8 KV kernels the fleet serves would not resolve, the
|
||||||
|
# convert refuses here instead of the boot failing on a drained window.
|
||||||
|
# --max-ctx is the bound declare is run at, whose default is the checkpoint's
|
||||||
|
# training context (262,144 here, whose fp8 cache is 258.8 GiB against a 192 GiB
|
||||||
|
# machine, so the default is not an option). 32,768 is set by the corpus, not by
|
||||||
|
# taste: kld.jsonl's longest record is 38,263 characters — ~9,600 tokens at the
|
||||||
|
# pessimistic chars/4, ~7,700 at the chars/4.97 this repo measured on the K3
|
||||||
|
# corpus — so the window the rig boots at clears every record 2.6x over and the
|
||||||
|
# reference boot cannot die on a sequence that will not fit. The rig's window is
|
||||||
|
# one knob (FRANK_QWEN38_MAX_MODEL_LEN in scripts/run-qwen38-bf16.sh, default
|
||||||
|
# 32,768) and that script refuses a corpus its window cannot hold, so raising one
|
||||||
|
# without the other is caught before the drained window, not inside it. The
|
||||||
|
# payload containers the gate scores declare at the training context because
|
||||||
|
# production must hold the 200k shape; a declared bound is a declare-time check,
|
||||||
|
# not a weight change, so the reference and the payloads carry the same numbers
|
||||||
|
# whatever each declared at — which is why one can be measured against the other.
|
||||||
|
CONVERT_ARGS=("$CKPT_IN" -o "$OUT_IN" --max-ctx 32768 --kv-cache-dtype fp8)
|
||||||
|
PODMAN=(podman run --rm --entrypoint rad-convert -v "$MODELS:/models")
|
||||||
|
PLAN_CMD=("${PODMAN[@]}" --name qwen38-bf16-plan "$IMAGE" "${CONVERT_ARGS[@]}" --plan-only -v)
|
||||||
|
CONV_CMD=("${PODMAN[@]}" --name qwen38-bf16-convert "$IMAGE" "${CONVERT_ARGS[@]}")
|
||||||
|
|
||||||
|
if [ "$DRY" = "1" ]; then
|
||||||
|
echo "dry run: no gate checked, no container started, nothing written"
|
||||||
|
echo "# $CKPT -> $OUT (the convert sees them as $CKPT_IN and $OUT_IN)"
|
||||||
|
echo "# plan argv:"
|
||||||
|
printf '%q\n' "${PLAN_CMD[@]}"
|
||||||
|
echo "# convert argv:"
|
||||||
|
printf '%q\n' "${CONV_CMD[@]}"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ---- gate: the checkpoint is on disk ---------------------------------------
|
||||||
|
if [ ! -d "$CKPT" ]; then
|
||||||
|
echo "REFUSED: no checkpoint at $CKPT — fetch and verify it first: scripts/download-qwen38-bf16.sh --fetch" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ---- gate: peak disk --------------------------------------------------------
|
||||||
|
# The deepest existing ancestor is the right thing to ask: a fresh
|
||||||
|
# /home/dank/models/rad has no free space of its own to report.
|
||||||
|
OUT_DIR="$(dirname "$OUT")"
|
||||||
|
PROBE="$OUT_DIR"
|
||||||
|
while [ ! -d "$PROBE" ] && [ "$PROBE" != "/" ]; do PROBE="$(dirname "$PROBE")"; done
|
||||||
|
FREE_MIB="$(df --output=avail -m "$PROBE" | tail -1 | tr -d ' ')"
|
||||||
|
if ! [[ "$FREE_MIB" =~ ^[0-9]+$ ]]; then
|
||||||
|
echo "REFUSED: cannot read the free space under $PROBE (df said: $FREE_MIB)" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
if [ "$PLAN_ONLY" != "1" ] && [ "$FREE_MIB" -lt "$MIN_FREE_MIB" ]; then
|
||||||
|
echo "REFUSED peak-disk: $FREE_MIB MiB free under $PROBE, $MIN_FREE_MIB MiB wanted (QWEN38_MIN_FREE_MIB) — the bf16 container rewrites the checkpoint's own 343,323 MiB of tensors beside it" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ---- gate: one conversion ---------------------------------------------------
|
||||||
|
if [ -e "$OUT" ]; then
|
||||||
|
if [ "$PLAN_ONLY" = "1" ]; then
|
||||||
|
echo "note: $OUT already exists; --plan-only writes nothing, so it stays"
|
||||||
|
elif [ "$FORCE" = "1" ]; then
|
||||||
|
mkdir -p "$TRASH"
|
||||||
|
DEST="$TRASH/$(date -u +%Y%m%dT%H%M%SZ)-$(basename "$OUT")"
|
||||||
|
mv "$OUT" "$DEST"
|
||||||
|
echo "TRASHED: retired the existing reference container $OUT -> $DEST"
|
||||||
|
else
|
||||||
|
echo "REFUSED: $OUT already exists — the campaign converts the reference once; re-run with --force to retire it to $TRASH first" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ---- gate: the tree really is the checkpoint (the full verify) -------------
|
||||||
|
echo "== verifying $CKPT (scripts/download-qwen38-bf16.sh, full gates)"
|
||||||
|
rc=0
|
||||||
|
bash "$SCRIPT_DIR/download-qwen38-bf16.sh" --dir "$CKPT" || rc=$?
|
||||||
|
if [ "$rc" -ne 0 ]; then
|
||||||
|
if [ "$rc" -eq 2 ]; then
|
||||||
|
echo "REFUSED: the checkpoint verifier refused the environment (rc=2); nothing was converted" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
echo "REFUSED: $CKPT failed verification (rc=$rc) — a half tree converts to a half reference" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ---- plan, then convert ------------------------------------------------------
|
||||||
|
mkdir -p "$(dirname "$LOG")"
|
||||||
|
# rad-convert opens -o during declare as well, so the output directory exists
|
||||||
|
# before the plan; that empty directory is all this creates ahead of it.
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
echo "== rad-convert --plan-only -v (declare + plan; writes nothing)"
|
||||||
|
"${PLAN_CMD[@]}" 2>&1 | tee -a "$LOG"
|
||||||
|
rc=${PIPESTATUS[0]}
|
||||||
|
if [ "$rc" -ne 0 ]; then
|
||||||
|
echo "REFUSED plan: rad-convert --plan-only exited rc=$rc — the engine refuses this encoding/shape, no container was written (log: $LOG)" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
if [ "$PLAN_ONLY" = "1" ]; then
|
||||||
|
echo "PLAN_OK: $CKPT_IN declares for $OUT_IN; no container written (log: $LOG)"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "== convert: $CKPT -> $OUT ($(date -Is))"
|
||||||
|
"${CONV_CMD[@]}" 2>&1 | tee -a "$LOG"
|
||||||
|
rc=${PIPESTATUS[0]}
|
||||||
|
if [ "$rc" -ne 0 ]; then
|
||||||
|
echo "rad-convert rc=$rc; log: $LOG" >&2
|
||||||
|
exit "$rc"
|
||||||
|
fi
|
||||||
|
MIB=$(( $(stat -c %s "$OUT") / 1048576 ))
|
||||||
|
echo "BUILD_DONE: $OUT is ${MIB} MiB (log: $LOG)"
|
||||||
|
echo "NEXT: scripts/run-qwen38-bf16.sh kldref records the reference logits from it"
|
||||||
Executable
+329
@@ -0,0 +1,329 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# run-qwen38-bf16.sh — RUNS ON DANKSERVER. Boot a container over the campaign's
|
||||||
|
# ONE measuring shape for the phases that need both R9700s. The engine owns the
|
||||||
|
# cards, so frank-archon is stopped first and restored on every exit path — the
|
||||||
|
# autoresearch pattern: stop, never mask (the unit file in /etc/systemd/system
|
||||||
|
# is a symlink into the checkout and mask fails on a relinked unit; the deploy
|
||||||
|
# restores it).
|
||||||
|
#
|
||||||
|
# kldref --kld-record the reference log-probabilities: the engine
|
||||||
|
# runs /home/dank/corpora/out/kld.jsonl (197 prompts, 50,432
|
||||||
|
# scored positions, score tail 256) through prefill, writes
|
||||||
|
# /home/dank/campaign/qwen38-w5nl64/kld-ref, and EXITS. The
|
||||||
|
# model behind it is the BF16 container from
|
||||||
|
# scripts/build-qwen38-bf16-rad.sh — the vendor checkpoint's
|
||||||
|
# own numbers, which is what makes it the zero point.
|
||||||
|
# score RAD OUT --kld-ref score ONE container against that recording and
|
||||||
|
# EXIT: the FP8-native anchor, the served iq4r, and every
|
||||||
|
# arm. Deliberately the SAME engine pin, the same TP, the
|
||||||
|
# same step size, the same fp8 KV, --deterministic and no
|
||||||
|
# speculation as kldref — KL and top-1 agreement are only
|
||||||
|
# comparable across boots when everything except the weights
|
||||||
|
# is identical, so this mode is one argv with --model and
|
||||||
|
# the two --kld-* flags changed, never a second shape.
|
||||||
|
# RAD must sit under the models dir (the container reads it
|
||||||
|
# through the ro /models mount); OUT lands on a writable
|
||||||
|
# mount because --kld-out F also writes F.rows beside it.
|
||||||
|
# calib serve on :7770 with the calibration tap open
|
||||||
|
# (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS=12000000). A
|
||||||
|
# separate driver — scripts/calib-push.py --port 7770
|
||||||
|
# --container qwen38-bf16 --corpus
|
||||||
|
# /home/dank/corpora/out/calib.jsonl — pushes the corpus; at
|
||||||
|
# 12M pooled positions the engine writes the Grams and the
|
||||||
|
# tap switches itself off (calib-push prints TAP_DONE). Stop
|
||||||
|
# this script after that; the trap brings frank-archon back.
|
||||||
|
#
|
||||||
|
# Shape: the vendor's bf16 calibration lines — expert_tiered placement with the
|
||||||
|
# weights disk tier, deterministic (the resident set pinned to the loader's plan,
|
||||||
|
# so a recorded reference and a scored candidate see the same one), no
|
||||||
|
# speculation, fp8 KV, a 32,768-token window, the PLE n-gram table pinned to RAM,
|
||||||
|
# and the production chat template rather than the one baked into the container —
|
||||||
|
# every scored container is asked the same tokenised questions the reference
|
||||||
|
# answered. The window is 32,768 because this corpus is long: measured on
|
||||||
|
# Dankserver 2026-10-27, kld.jsonl is 197 records with a longest of 38,263
|
||||||
|
# characters (longest prompt FIELD 36,855, median record 27,228), which is ~9,600
|
||||||
|
# tokens at the pessimistic chars/4 and ~7,700 at the ratio this repo actually
|
||||||
|
# measured on the K3 corpus (380,143 chars = 76,568 tokens, chars/4.97). A 16,384
|
||||||
|
# window cleared that by 1.7x; 32,768 clears every record of both corpora by at
|
||||||
|
# least 2.6x, and the kldref gate below re-checks it against the file on disk.
|
||||||
|
#
|
||||||
|
# NO --vram-kv-mib: it was REMOVED at 1.3.0 and naming it is a boot refusal, not
|
||||||
|
# a no-op — verified on the pinned image 2026-10-27: "E --vram-kv-mib was
|
||||||
|
# removed: the card is split between the weights and the cache from what the
|
||||||
|
# model needs." The budget resolver derives the per-card split from
|
||||||
|
# --host-pool-mib and --gpu-headroom-mib, and those two are deliberately NOT
|
||||||
|
# production's 49,152 / 1,536 (frank-archon.env:77): the pool decides WHICH
|
||||||
|
# experts stay resident, so it buys speed and never logits — the shape the gate
|
||||||
|
# has to hold identical is the kv dtype, the slot count, the step size, the spec
|
||||||
|
# depth and the window. It is also the only figure that fits: a 335 GiB bf16
|
||||||
|
# container cannot be booted against a 49 GiB resident set on a 192 GiB machine,
|
||||||
|
# and a prefill-only pass does not care that the disk tier paces at ~3 GiB/s.
|
||||||
|
#
|
||||||
|
# The step size is ONE value for the recording and every scoring pass:
|
||||||
|
# FRANK_QWEN38_MNBT, default 2,048. The gate's whole claim is that the reference,
|
||||||
|
# the served iq4r and every arm rode one shape, so the ceiling is set by the
|
||||||
|
# TIGHTEST payload the gate boots rather than by the bf16 reference: the quantised
|
||||||
|
# expert FILE tiers (the served iq4r, the arms) stream only at MNBT<=2048 and
|
||||||
|
# refuse the widened form above it, and 2,048 is what production holds anyway.
|
||||||
|
# Recording the reference at 32,768 would therefore measure a shape the payloads
|
||||||
|
# cannot reproduce; at 2,048 the 50,432 positions cost more steps, which is still
|
||||||
|
# one drained window. Calibration keeps the wide 65,536 — grams are not logits,
|
||||||
|
# nothing downstream is compared against a step size, and that is where a big
|
||||||
|
# step pays (a disk-tier step costs the same at 1 token as at 64k).
|
||||||
|
#
|
||||||
|
# usage: scripts/run-qwen38-bf16.sh kldref|calib|score RAD OUT [--dry-run] [--image REF]
|
||||||
|
# (RAD and OUT are score's two positionals: RAD is a container under the models
|
||||||
|
# dir, OUT the report path whose directory is created if missing)
|
||||||
|
# --dry-run print the exact `podman run` argv, one shell-quoted argument a
|
||||||
|
# line, and exit 0: the service is NOT stopped, nothing is
|
||||||
|
# mounted, nothing is written — the offline test seam. It still
|
||||||
|
# validates the argv's own shape (paths through the /models mount,
|
||||||
|
# a step size that is a number), because an argv that could not
|
||||||
|
# boot is not worth printing.
|
||||||
|
# --image REF override the digest pin (a probe boot, or the test seam)
|
||||||
|
#
|
||||||
|
# knobs: FRANK_QWEN38_MODELS (the dir mounted at /models), FRANK_QWEN38_RAD (the
|
||||||
|
# bf16 container, kldref only), FRANK_QWEN38_CAMPAIGN (holds kld-ref/ and
|
||||||
|
# calib/), FRANK_QWEN38_KLD_CORPUS, FRANK_QWEN38_TPL (the template copy
|
||||||
|
# the container reads through /models), FRANK_QWEN38_LOG_DIR, and
|
||||||
|
# FRANK_QWEN38_MNBT — the step size shared by kldref and score (2,048 is
|
||||||
|
# the file tier's ceiling and production's own value, four slots and all),
|
||||||
|
# and FRANK_QWEN38_MAX_MODEL_LEN — the window every mode boots at (32,768;
|
||||||
|
# see the corpus census above, and raise it together with the corpus).
|
||||||
|
# kldref refuses (exit 2) when the longest record of the corpus it is about to
|
||||||
|
# read is longer than MAX_MODEL_LEN x 3 characters — chars/3 being the
|
||||||
|
# pessimistic token:char ratio, so the script says no before the engine finds out
|
||||||
|
# inside a drained window, after the checkpoint work is already done. score
|
||||||
|
# refuses when the reference dir is absent, when RAD is not on disk or not under
|
||||||
|
# the models mount, or when OUT already exists: the trash law means a second
|
||||||
|
# score over the same name is a deliberate act, and two numbers in one file is
|
||||||
|
# how a campaign loses its own history.
|
||||||
|
# Engine stdout goes to $LOG_DIR/qwen38_bf16_<mode>.log — the boot ledger is the
|
||||||
|
# record of what the resolver actually gave the cards; the journal never sees it.
|
||||||
|
# exit: the engine's code; 2 = refused before the service was touched
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
source "$SCRIPT_DIR/frank-archon.env"
|
||||||
|
IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}"
|
||||||
|
|
||||||
|
MODE=""
|
||||||
|
POS=()
|
||||||
|
DRY=0
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
kldref|calib|score) MODE="$1" ;;
|
||||||
|
--dry-run) DRY=1 ;;
|
||||||
|
--image) IMAGE="$2"; shift ;;
|
||||||
|
*)
|
||||||
|
# score's two positional arguments, and nothing else may be bare.
|
||||||
|
if [ "$MODE" = score ] && [ "${#POS[@]}" -lt 2 ]; then
|
||||||
|
POS+=("$1")
|
||||||
|
else
|
||||||
|
echo "usage: ${0##*/} kldref|calib|score RAD OUT [--dry-run] [--image REF] (unknown argument: $1)" >&2
|
||||||
|
exit 2
|
||||||
|
fi ;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
if [ -z "$MODE" ]; then
|
||||||
|
echo "usage: ${0##*/} kldref|calib|score RAD OUT [--dry-run] [--image REF] (no mode given)" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
if [ "$MODE" = score ] && [ "${#POS[@]}" -ne 2 ]; then
|
||||||
|
echo "usage: ${0##*/} score RAD OUT — RAD is the container to score, OUT the report to write (got ${#POS[@]} argument(s))" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
|
||||||
|
CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
|
||||||
|
KLD_CORPUS="${FRANK_QWEN38_KLD_CORPUS:-/home/dank/corpora/out/kld.jsonl}"
|
||||||
|
RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}"
|
||||||
|
TPL="${FRANK_QWEN38_TPL:-$MODELS/tpl.jinja}"
|
||||||
|
TEMPLATE="$SCRIPT_DIR/qwen38-froggeric-v22.5.jinja"
|
||||||
|
LOG_DIR="${FRANK_QWEN38_LOG_DIR:-/home/dank/logs}"
|
||||||
|
LOG="$LOG_DIR/qwen38_bf16_${MODE}.log"
|
||||||
|
KLD_REF_DIR="$CAMPAIGN/kld-ref"
|
||||||
|
CALIB_DIR="$CAMPAIGN/calib"
|
||||||
|
C=qwen38-bf16
|
||||||
|
SERVICE=frank-archon
|
||||||
|
|
||||||
|
# ONE step size for both logits modes. The file tier streams only at <=2,048 and
|
||||||
|
# the gate scores file-tier payloads (the served iq4r, the arms) against the
|
||||||
|
# reference, so the reference is recorded at the ceiling the payloads can meet —
|
||||||
|
# never two numbers that a KL between them would have to forgive.
|
||||||
|
MNBT="${FRANK_QWEN38_MNBT:-2048}"
|
||||||
|
if ! [[ "$MNBT" =~ ^[0-9]+$ ]]; then
|
||||||
|
echo "REFUSED: FRANK_QWEN38_MNBT must be a token count (got: $MNBT)" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The window every mode boots at. Over-length is a REFUSAL, not a truncation (the
|
||||||
|
# env file says so for production), so this is the number the corpus must fit in.
|
||||||
|
MAX_MODEL_LEN="${FRANK_QWEN38_MAX_MODEL_LEN:-32768}"
|
||||||
|
if ! [[ "$MAX_MODEL_LEN" =~ ^[0-9]+$ ]]; then
|
||||||
|
echo "REFUSED: FRANK_QWEN38_MAX_MODEL_LEN must be a token count (got: $MAX_MODEL_LEN)" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Every container this campaign boots is read through the ro /models mount, so
|
||||||
|
# the host path must live under it; the container-side spelling is what goes
|
||||||
|
# into the argv. Same rule for the container a score pass opens.
|
||||||
|
under_models() {
|
||||||
|
case "$1" in
|
||||||
|
"$MODELS"/*) printf '/models/%s' "${1#"$MODELS"/}" ;;
|
||||||
|
*) return 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# same_dir A B — the two name one directory, judged only when BOTH exist: a dry
|
||||||
|
# run has no fixtures on disk, and two absent paths are not the same place.
|
||||||
|
same_dir() {
|
||||||
|
[ -d "$1" ] && [ -d "$2" ] && [ "$(cd "$1" && pwd)" = "$(cd "$2" && pwd)" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ "$MODE" = score ]; then
|
||||||
|
SCORE_RAD="${POS[0]}"
|
||||||
|
SCORE_OUT="${POS[1]}"
|
||||||
|
OUT_DIR="$(dirname "$SCORE_OUT")"
|
||||||
|
MODEL_IN="$(under_models "$SCORE_RAD")" || {
|
||||||
|
echo "REFUSED: container to score $SCORE_RAD is not under the mounted models dir $MODELS — the engine cannot read it" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
else
|
||||||
|
MODEL_IN="$(under_models "$RAD")" || {
|
||||||
|
echo "REFUSED: reference container $RAD is not under the mounted models dir $MODELS — the engine cannot read it" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The measuring shape. Identical for kldref and score by construction — this
|
||||||
|
# array is the only place the campaign's KL numbers are defined.
|
||||||
|
ARGS=(--tp 2 --placement expert_tiered --weights-disk-tier
|
||||||
|
--host-pool-mib 12288 --gpu-headroom-mib 768
|
||||||
|
--deterministic --num-speculative-tokens 0
|
||||||
|
--kv-cache-dtype fp8 --max-model-len "$MAX_MODEL_LEN"
|
||||||
|
--ngram-placement ram
|
||||||
|
--override-chat-template /models/tpl.jinja)
|
||||||
|
MOUNTS=(-v "$MODELS:/models:ro")
|
||||||
|
TAPENV=()
|
||||||
|
case "$MODE" in
|
||||||
|
kldref)
|
||||||
|
# Recording is prefill-only over four slots and the shared step, which is
|
||||||
|
# production's own operating point (frank-archon.env: FRANK_ARCHON_MAX_SEQS=4
|
||||||
|
# with FRANK_ARCHON_MAX_BATCHED_TOKENS=2048). Corpus read-only, record dir rw.
|
||||||
|
ARGS+=(--model "$MODEL_IN"
|
||||||
|
--kld-record /data/kld-ref --kld-corpus /data/kld.jsonl
|
||||||
|
--max-num-seqs 4 --max-num-batched-tokens "$MNBT")
|
||||||
|
MOUNTS+=(-v "$KLD_CORPUS:/data/kld.jsonl:ro"
|
||||||
|
-v "$KLD_REF_DIR:/data/kld-ref")
|
||||||
|
;;
|
||||||
|
score)
|
||||||
|
# The same argv with three words changed: which container, and that this
|
||||||
|
# time the engine reads a recording and writes a report instead of the
|
||||||
|
# reverse. The reference dir mounts READ-ONLY — a scoring pass must not be
|
||||||
|
# able to touch the zero point — and the report gets its own writable bind.
|
||||||
|
if [ "$OUT_DIR" = "$KLD_REF_DIR" ] || same_dir "$OUT_DIR" "$KLD_REF_DIR"; then
|
||||||
|
echo "REFUSED: score output $SCORE_OUT is inside the reference dir $KLD_REF_DIR, which this mode mounts read-only" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
ARGS+=(--model "$MODEL_IN"
|
||||||
|
--kld-ref /data/kld-ref --kld-out "/data/out/$(basename "$SCORE_OUT")"
|
||||||
|
--max-num-seqs 4 --max-num-batched-tokens "$MNBT")
|
||||||
|
MOUNTS+=(-v "$KLD_REF_DIR:/data/kld-ref:ro"
|
||||||
|
-v "$OUT_DIR:/data/out")
|
||||||
|
;;
|
||||||
|
calib)
|
||||||
|
# Calibration is exempt from the shared step: the tap counts pooled
|
||||||
|
# positions and nothing downstream is compared against a step size, so the
|
||||||
|
# wide step is free throughput (a disk-tier step costs the same at 1 token
|
||||||
|
# as at 64k).
|
||||||
|
ARGS+=(--model "$MODEL_IN" --port 7770
|
||||||
|
--max-num-seqs 8 --max-num-batched-tokens 65536)
|
||||||
|
MOUNTS+=(-v "$CALIB_DIR:/data/calib")
|
||||||
|
TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib
|
||||||
|
-e RADIANCE_CALIB_TOKENS=12000000)
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
CMD=(podman run --rm --name "$C"
|
||||||
|
--network host --init
|
||||||
|
--security-opt seccomp=unconfined
|
||||||
|
--ulimit memlock=-1:-1
|
||||||
|
--device /dev/kfd --device /dev/dri
|
||||||
|
"${MOUNTS[@]}" "${TAPENV[@]}"
|
||||||
|
"$IMAGE" "${ARGS[@]}")
|
||||||
|
|
||||||
|
# ---- gate: the window must hold the corpus (kldref alone can meet it) --------
|
||||||
|
# score reads a recording instead of a corpus, and every arm declares at the
|
||||||
|
# checkpoint's training context, so neither can meet a sequence too long to boot.
|
||||||
|
# Measured on the raw JSON line — the pessimistic direction, and the same figure
|
||||||
|
# the census in the header quotes. It runs on a dry run too, because a refusal
|
||||||
|
# this cheap should never have to wait for the drained window to be discovered.
|
||||||
|
if [ "$MODE" = kldref ] && [ -f "$KLD_CORPUS" ]; then
|
||||||
|
read -r REC_NO REC_CHARS < <(awk '{ if (length($0) > best) { best = length($0); where = NR } } END { print where + 0, best + 0 }' "$KLD_CORPUS")
|
||||||
|
LIMIT_CHARS=$(( MAX_MODEL_LEN * 3 ))
|
||||||
|
if [ "$REC_CHARS" -gt "$LIMIT_CHARS" ]; then
|
||||||
|
echo "REFUSED: record $REC_NO of $KLD_CORPUS is $REC_CHARS chars, past the $LIMIT_CHARS a $MAX_MODEL_LEN-token window promises at the pessimistic chars/3 — raise FRANK_QWEN38_MAX_MODEL_LEN, and the container's declared --max-ctx with it, before booting the reference" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
echo "corpus fits: longest record $REC_CHARS chars (number $REC_NO) inside the $MAX_MODEL_LEN-token window (limit $LIMIT_CHARS chars)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$DRY" = 1 ]; then
|
||||||
|
echo "dry run: $MODE — the production unit is left running, nothing is mounted, nothing is written"
|
||||||
|
printf '%q\n' "${CMD[@]}"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ---- gates that only a real run needs ---------------------------------------
|
||||||
|
if [ "$MODE" = score ]; then
|
||||||
|
if [ ! -d "$KLD_REF_DIR" ]; then
|
||||||
|
echo "REFUSED: no KLD reference at $KLD_REF_DIR — record it first in a drained window: scripts/run-qwen38-bf16.sh kldref" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
if [ ! -f "$SCORE_RAD" ]; then
|
||||||
|
echo "REFUSED: no container to score at $SCORE_RAD" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
if [ -e "$SCORE_OUT" ]; then
|
||||||
|
echo "REFUSED: $SCORE_OUT already exists — one report per name; move the old one to trash before re-scoring" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
else
|
||||||
|
if [ ! -f "$RAD" ]; then
|
||||||
|
echo "REFUSED: no reference container at $RAD — build it first: scripts/build-qwen38-bf16-rad.sh" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
[ "$MODE" = kldref ] && [ ! -f "$KLD_CORPUS" ] && {
|
||||||
|
echo "REFUSED: no KLD corpus at $KLD_CORPUS — the record would be empty" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
fi
|
||||||
|
if [ ! -f "$TEMPLATE" ]; then
|
||||||
|
echo "REFUSED: no chat template at $TEMPLATE" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
# The prod template, staged where the ro /models mount reaches it: a reference
|
||||||
|
# recorded with the container's baked template scores a different tokenisation
|
||||||
|
# than the arms served with froggeric's, and the KL would be measuring that.
|
||||||
|
cp -f "$TEMPLATE" "$TPL" || {
|
||||||
|
echo "REFUSED: cannot stage the chat template at $TPL (the container reads it through /models)" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
[ "$MODE" = kldref ] && mkdir -p "$KLD_REF_DIR"
|
||||||
|
[ "$MODE" = calib ] && mkdir -p "$CALIB_DIR"
|
||||||
|
|
||||||
|
restore() { sudo systemctl start "$SERVICE" >/dev/null 2>&1 || true; }
|
||||||
|
trap restore EXIT
|
||||||
|
echo "run-qwen38-bf16 $MODE: stopping $SERVICE ($(date -Is)); log: $LOG"
|
||||||
|
sudo systemctl stop "$SERVICE" >/dev/null 2>&1 || true
|
||||||
|
podman rm -f "$C" >/dev/null 2>&1 || true
|
||||||
|
|
||||||
|
"${CMD[@]}" 2>&1 | tee "$LOG"
|
||||||
|
rc=${PIPESTATUS[0]}
|
||||||
|
echo "run-qwen38-bf16 $MODE exited rc=$rc; $SERVICE restored; log: $LOG"
|
||||||
|
exit "$rc"
|
||||||
Executable
+337
@@ -0,0 +1,337 @@
|
|||||||
|
#!/usr/bin/env bats
|
||||||
|
# Seam: the BF16 REFERENCE container of the Qwen3.8-Flash-Next five-bit
|
||||||
|
# campaign (feat/qwen38-w5nl64) — scripts/build-qwen38-bf16-rad.sh and
|
||||||
|
# scripts/run-qwen38-bf16.sh.
|
||||||
|
#
|
||||||
|
# The reference is what every later stage is judged or derived from (the KLD
|
||||||
|
# record scores the arms, the Grams derive the protected experts), so the
|
||||||
|
# deliverable here is the set of refusals: a half checkpoint, a second convert,
|
||||||
|
# a volume that cannot hold the convert, an engine-refused plan, and any run —
|
||||||
|
# including a dry one — that would take the GPUs from frank-archon. Each test
|
||||||
|
# drives the script's own exit status and printed argv. Offline: no engine, no
|
||||||
|
# GPU, no container, no systemctl, no network. Path knobs aim at
|
||||||
|
# $BATS_TEST_TMPDIR.
|
||||||
|
#
|
||||||
|
# The third mode, score, carries the campaign's comparability law: its engine
|
||||||
|
# argv must equal kldref's except for --model and the --kld-* pair, or the KL
|
||||||
|
# numbers of the FP8-native anchor, the served payload and the arms are not one
|
||||||
|
# measurement series. The seam proves that by diffing the two dry runs.
|
||||||
|
|
||||||
|
# The 1.8.1 law asserted in two places below: --vram-kv-mib was REMOVED at
|
||||||
|
# 1.3.0, so naming it is a boot refusal — the reference serve must carry the
|
||||||
|
# budget-resolver knobs (--host-pool-mib / --gpu-headroom-mib) instead.
|
||||||
|
|
||||||
|
load helpers
|
||||||
|
|
||||||
|
BUILD="$REPO_ROOT/scripts/build-qwen38-bf16-rad.sh"
|
||||||
|
RUNNER="$REPO_ROOT/scripts/run-qwen38-bf16.sh"
|
||||||
|
|
||||||
|
# The reference shape, spelled exactly as run-qwen38-bf16.sh must pass it: this
|
||||||
|
# is the contract the recorded reference is measured at, so a silent edit here
|
||||||
|
# is a silent edit to every KLD number in the campaign.
|
||||||
|
SHAPE="--tp 2 --placement expert_tiered --weights-disk-tier --host-pool-mib 12288 --gpu-headroom-mib 768 --deterministic --num-speculative-tokens 0 --kv-cache-dtype fp8 --max-model-len 32768 --ngram-placement ram --override-chat-template /models/tpl.jinja"
|
||||||
|
|
||||||
|
# oneline — both dry runs print one shell-quoted argument a line; joining them
|
||||||
|
# lets an option and its value be asserted as the pair they are.
|
||||||
|
oneline() { local s="${output//$'\n'/ }"; printf '%s' "$s"; }
|
||||||
|
|
||||||
|
# engine_argv — the arguments AFTER the image line of a dry run, with --model
|
||||||
|
# and every --kld-* flag and its value removed: what is left is the measuring
|
||||||
|
# shape, and it must be identical between kldref and score.
|
||||||
|
engine_argv() {
|
||||||
|
local seen=0 skip=0 a
|
||||||
|
while IFS= read -r a; do
|
||||||
|
if [ "$seen" = 0 ]; then
|
||||||
|
[[ "$a" == *"radiance@sha256:"* ]] && seen=1
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
case "$a" in
|
||||||
|
--model|--kld-record|--kld-corpus|--kld-ref|--kld-out) skip=1; continue ;;
|
||||||
|
esac
|
||||||
|
if [ "$skip" = 1 ]; then skip=0; continue; fi
|
||||||
|
printf '%s\n' "$a"
|
||||||
|
done <<< "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
# flag_value FLAG — the value a dry run printed for FLAG (the argv is one
|
||||||
|
# argument a line, so the value is the line after it). Lets a test ask what
|
||||||
|
# number the script chose instead of re-declaring the literal it guards.
|
||||||
|
flag_value() {
|
||||||
|
local want="$1" prev="" a
|
||||||
|
while IFS= read -r a; do
|
||||||
|
if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi
|
||||||
|
prev="$a"
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the build dry run prints the rad-convert argv for the checkpoint and the bf16 output" {
|
||||||
|
M="$BATS_TEST_TMPDIR/models"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" bash "$BUILD" --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"rad-convert"* ]]
|
||||||
|
# the argv is the container's view: the models dir rides the /models mount
|
||||||
|
[[ "$output" == *"$M:/models"* ]]
|
||||||
|
flat="$(oneline)"
|
||||||
|
[[ "$flat" == *"/models/Qwen3.8-Flash-Next -o /models/rad/qwen38-bf16.rad"* ]]
|
||||||
|
# a reference is the checkpoint re-headed, not re-encoded, and it declares a
|
||||||
|
# window the long KLD corpus can be read inside of
|
||||||
|
[[ "$output" != *"--recipe"* ]]
|
||||||
|
[[ "$output" != *"--reuse"* ]]
|
||||||
|
[[ "$flat" == *"--max-ctx 32768"* ]]
|
||||||
|
# and a dry run looked at nothing: the fixture dirs were never created
|
||||||
|
[ ! -e "$M" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the build refuses a checkpoint that is not on disk and names the fetch command" {
|
||||||
|
M="$BATS_TEST_TMPDIR/models"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" bash "$BUILD"
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"$M/Qwen3.8-Flash-Next"* ]]
|
||||||
|
[[ "$output" == *"scripts/download-qwen38-bf16.sh --fetch"* ]]
|
||||||
|
[ ! -e "$M/rad/qwen38-bf16.rad" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the build refuses an existing reference container and names --force as the way past it" {
|
||||||
|
M="$BATS_TEST_TMPDIR/models"
|
||||||
|
mkdir -p "$M/Qwen3.8-Flash-Next" "$M/rad"
|
||||||
|
echo "not a container" > "$M/rad/qwen38-bf16.rad"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" QWEN38_MIN_FREE_MIB=1 \
|
||||||
|
QWEN38_TRASH="$BATS_TEST_TMPDIR/trash" bash "$BUILD"
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"$M/rad/qwen38-bf16.rad"* ]]
|
||||||
|
[[ "$output" == *"--force"* ]]
|
||||||
|
# refusing retires nothing: the trash law means the file survives a refusal
|
||||||
|
[ -f "$M/rad/qwen38-bf16.rad" ]
|
||||||
|
[ ! -e "$BATS_TEST_TMPDIR/trash" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the build refuses when the output filesystem cannot hold a second 335 GiB" {
|
||||||
|
# The campaign's peak-disk law: 343,323 MiB of container written next to
|
||||||
|
# 343,323 MiB of checkpoint. The wanted figure is set above whatever this
|
||||||
|
# box really has, which is what the gate reads.
|
||||||
|
M="$BATS_TEST_TMPDIR/models"
|
||||||
|
mkdir -p "$M/Qwen3.8-Flash-Next" "$M/rad"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" QWEN38_MIN_FREE_MIB=999999999 bash "$BUILD"
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"REFUSED"* ]]
|
||||||
|
[[ "$output" =~ [0-9]+" MiB free" ]]
|
||||||
|
[[ "$output" == *"999999999 MiB wanted"* ]]
|
||||||
|
# the disk gate is the cheap one: it refuses long before the Hub sweep runs
|
||||||
|
[[ "$output" != *"VERIFY_OK"* ]]
|
||||||
|
[ ! -e "$M/rad/qwen38-bf16.rad" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the reference serve rejects an unknown mode and names every valid mode" {
|
||||||
|
# `probe` is the swift15 era's third mode; it is retired and must not boot
|
||||||
|
# a container here either.
|
||||||
|
run bash "$RUNNER" probe
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"probe"* ]]
|
||||||
|
[[ "$output" == *"kldref"* ]]
|
||||||
|
[[ "$output" == *"calib"* ]]
|
||||||
|
[[ "$output" == *"score"* ]]
|
||||||
|
[[ "$output" != *"podman"* ]]
|
||||||
|
run bash "$RUNNER"
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"kldref"* ]]
|
||||||
|
[[ "$output" == *"calib"* ]]
|
||||||
|
[[ "$output" == *"score"* ]]
|
||||||
|
# score without its two positionals is the same usage refusal
|
||||||
|
run bash "$RUNNER" score
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"score RAD OUT"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the kldref dry run carries the record flags at the reference shape and no --vram-kv-mib" {
|
||||||
|
run bash "$RUNNER" kldref --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
flat="$(oneline)"
|
||||||
|
[[ "$flat" == *"--kld-record /data/kld-ref"* ]]
|
||||||
|
[[ "$flat" == *"--kld-corpus /data/kld.jsonl"* ]]
|
||||||
|
# the corpus arrives read-only, the record dir writable
|
||||||
|
[[ "$flat" == *":/data/kld.jsonl:ro"* ]]
|
||||||
|
[[ "$flat" == *":/data/kld-ref "* ]]
|
||||||
|
[[ "$flat" == *"$SHAPE"* ]]
|
||||||
|
# 1.8.1: the flag does not exist — naming it is a boot refusal
|
||||||
|
[[ "$output" != *"--vram-kv-mib"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the calib dry run opens the tap at 12M tokens and serves on 7770" {
|
||||||
|
run bash "$RUNNER" calib --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
flat="$(oneline)"
|
||||||
|
[[ "$flat" == *"-e RADIANCE_CALIB_TOKENS=12000000"* ]]
|
||||||
|
[[ "$flat" == *"-e RADIANCE_CALIB_DIR=/data/calib"* ]]
|
||||||
|
[[ "$flat" == *"--port 7770"* ]]
|
||||||
|
[[ "$flat" == *":/data/calib "* ]]
|
||||||
|
# calib serves; it does not record
|
||||||
|
[[ "$output" != *"--kld-record"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a dry run prints argv only: no drain, no container, nothing written" {
|
||||||
|
L="$BATS_TEST_TMPDIR/logs"
|
||||||
|
run env FRANK_QWEN38_LOG_DIR="$L" \
|
||||||
|
bash "$RUNNER" kldref --dry-run --image example.invalid/radiance@sha256:0000000000000000000000000000000000000000000000000000000000000000
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" != *"systemctl"* ]]
|
||||||
|
[[ "$output" != *"frank-archon stopped"* ]]
|
||||||
|
[[ "$output" != *"podman rm"* ]]
|
||||||
|
# not vacuous: the podman argv WAS printed, and --image replaced the pin
|
||||||
|
[[ "$output" == *"podman"* ]]
|
||||||
|
[[ "$output" == *"example.invalid/radiance@sha256:000"* ]]
|
||||||
|
[[ "$output" != *"stilldeadcode/radiance"* ]]
|
||||||
|
[ ! -d "$L" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the score dry run is the kldref argv with only --model and the --kld-* pair changed" {
|
||||||
|
M="$BATS_TEST_TMPDIR/models"
|
||||||
|
C="$BATS_TEST_TMPDIR/campaign"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \
|
||||||
|
bash "$RUNNER" kldref --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
ref_argv="$(engine_argv "$output")"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \
|
||||||
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
score_argv="$(engine_argv "$output")"
|
||||||
|
# The whole value of the campaign's three KL numbers is that they rode this
|
||||||
|
# one shape: same pin, --tp 2, fp8 KV, step size, --deterministic, no spec.
|
||||||
|
if [[ "$score_argv" != "$ref_argv" ]]; then
|
||||||
|
echo "--- kldref engine argv ---"; echo "$ref_argv"
|
||||||
|
echo "--- score engine argv ---"; echo "$score_argv"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
flat="$(oneline)"
|
||||||
|
[[ "$flat" == *"--model /models/rad/arm-w5nl64.rad"* ]]
|
||||||
|
[[ "$flat" == *"--kld-ref /data/kld-ref --kld-out /data/out/arm1.json"* ]]
|
||||||
|
# the zero point mounts read-only; the report needs a writable bind because
|
||||||
|
# --kld-out F also writes F.rows beside it
|
||||||
|
[[ "$flat" == *":/data/kld-ref:ro "* ]]
|
||||||
|
[[ "$flat" == *":/data/out "* ]]
|
||||||
|
[[ "$output" != *"--kld-record"* ]]
|
||||||
|
[[ "$output" != *"--kld-corpus"* ]]
|
||||||
|
[[ "$output" != *"--vram-kv-mib"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "score refuses a missing reference, a missing container and an existing report" {
|
||||||
|
M="$BATS_TEST_TMPDIR/models"
|
||||||
|
C="$BATS_TEST_TMPDIR/campaign"
|
||||||
|
L="$BATS_TEST_TMPDIR/logs"
|
||||||
|
mkdir -p "$M/rad"
|
||||||
|
: > "$M/rad/arm-w5nl64.rad"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \
|
||||||
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json"
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"$C/kld-ref"* ]]
|
||||||
|
[[ "$output" == *"run-qwen38-bf16.sh kldref"* ]]
|
||||||
|
# refused before the log dir, so before anything that could reach systemctl
|
||||||
|
[ ! -d "$L" ]
|
||||||
|
|
||||||
|
mkdir -p "$C/kld-ref"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \
|
||||||
|
bash "$RUNNER" score "$M/rad/gone.rad" "$C/scores/arm1.json"
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"$M/rad/gone.rad"* ]]
|
||||||
|
[ ! -d "$L" ]
|
||||||
|
|
||||||
|
mkdir -p "$C/scores"
|
||||||
|
: > "$C/scores/arm1.json"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \
|
||||||
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json"
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"$C/scores/arm1.json"* ]]
|
||||||
|
[[ "$output" == *"trash"* ]]
|
||||||
|
# the trash law: a refusal keeps the old report, it does not overwrite it
|
||||||
|
[ -f "$C/scores/arm1.json" ]
|
||||||
|
[ ! -d "$L" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the reference and a score pass ride ONE step size, legal on the file tier" {
|
||||||
|
# The gate's claim is one shape; that shape's ceiling is set by the tightest
|
||||||
|
# payload it boots, and the served iq4r and every arm are file-tier
|
||||||
|
# containers, which stream only at <=2048. A reference recorded above that is
|
||||||
|
# a KL between two shapes, not two weights.
|
||||||
|
M="$BATS_TEST_TMPDIR/models"
|
||||||
|
C="$BATS_TEST_TMPDIR/campaign"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" bash "$RUNNER" kldref --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
ref_step="$(flag_value --max-num-batched-tokens <<< "$output")"
|
||||||
|
[ -n "$ref_step" ]
|
||||||
|
[ "$ref_step" -le 2048 ]
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \
|
||||||
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
score_step="$(flag_value --max-num-batched-tokens <<< "$output")"
|
||||||
|
[ "$ref_step" = "$score_step" ]
|
||||||
|
# one knob moves both, so the operator can lower it at 2 a.m. without an edit
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \
|
||||||
|
bash "$RUNNER" kldref --dry-run
|
||||||
|
[ "$(flag_value --max-num-batched-tokens <<< "$output")" = "1024" ]
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \
|
||||||
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run
|
||||||
|
[ "$(flag_value --max-num-batched-tokens <<< "$output")" = "1024" ]
|
||||||
|
# calibration is exempt: grams are not logits, so the tap keeps the wide step
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \
|
||||||
|
bash "$RUNNER" calib --dry-run
|
||||||
|
[ "$(flag_value --max-num-batched-tokens <<< "$output")" = "65536" ]
|
||||||
|
# a step size that is not a token count is refused, never handed to the engine
|
||||||
|
run env FRANK_QWEN38_MNBT=broad bash "$RUNNER" kldref --dry-run
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"FRANK_QWEN38_MNBT"* ]]
|
||||||
|
[[ "$output" == *"must be a token count"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "kldref refuses a corpus its window cannot hold, before the service is touched" {
|
||||||
|
M="$BATS_TEST_TMPDIR/models" R="$M/rad"
|
||||||
|
mkdir -p "$M/tpl" "$R" "$BATS_TEST_TMPDIR/out"
|
||||||
|
: > "$M/tpl/qwen38-froggeric-v22.5.jinja"
|
||||||
|
: > "$R/qwen38-bf16.rad"
|
||||||
|
CORPUS="$BATS_TEST_TMPDIR/long.jsonl"
|
||||||
|
{
|
||||||
|
printf '{"prompt":"short one"}\n'
|
||||||
|
printf '{"prompt":"%s"}\n' "$(head -c 120000 /dev/zero | tr '\0' 'x')"
|
||||||
|
printf '{"prompt":"short three"}\n'
|
||||||
|
} > "$CORPUS"
|
||||||
|
# 32,768 tokens x 3 chars is the pessimistic limit; the 120,002-char line
|
||||||
|
# that opens record 2 is well past it, so the campaign says no up front.
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
||||||
|
bash "$RUNNER" kldref --dry-run
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"REFUSED"* ]]
|
||||||
|
[[ "$output" == *"record 2"* ]]
|
||||||
|
[[ "$output" == *"120013 chars"* ]]
|
||||||
|
[[ "$output" == *"past the 98304 a 32768-token window"* ]]
|
||||||
|
[[ "$output" == *"FRANK_QWEN38_MAX_MODEL_LEN"* ]]
|
||||||
|
# the same corpus is fine once the window is raised past it, and the raised
|
||||||
|
# window is what the boot is asked for
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
||||||
|
FRANK_QWEN38_MAX_MODEL_LEN=65536 FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" \
|
||||||
|
bash "$RUNNER" kldref --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"corpus fits"* ]]
|
||||||
|
flat="$(oneline)"
|
||||||
|
[[ "$flat" == *"--max-model-len 65536"* ]]
|
||||||
|
# the shipped corpus of three short records fits the shipped window
|
||||||
|
SHORT="$BATS_TEST_TMPDIR/short.jsonl"
|
||||||
|
printf '{"prompt":"tiny"}\n{"prompt":"also tiny"}\n' > "$SHORT"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$SHORT" \
|
||||||
|
FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" bash "$RUNNER" kldref --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"corpus fits: longest record 22 chars (number 2)"* ]]
|
||||||
|
# a real kldref run refuses at the same place: no log dir, no drain, and the
|
||||||
|
# service would still be up when the script gave up
|
||||||
|
L="$BATS_TEST_TMPDIR/logs-real"
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
||||||
|
FRANK_QWEN38_LOG_DIR="$L" bash "$RUNNER" kldref
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[ ! -d "$L" ]
|
||||||
|
# and it is kldref that reads the corpus: score and calib never consult it
|
||||||
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
||||||
|
FRANK_QWEN38_KLD_REF_DIR="$BATS_TEST_TMPDIR/ref" \
|
||||||
|
FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" \
|
||||||
|
bash "$RUNNER" score "$R/qwen38-bf16.rad" "$BATS_TEST_TMPDIR/out/arm.json" --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" != *"corpus fits"* ]]
|
||||||
|
[[ "$output" != *"REFUSED"* ]]
|
||||||
|
}
|
||||||
Reference in new issue
Block a user