Frank
f6298e3501
Measured from the engine's own counters on the 2026-10-29 drain: prefill 71 tok/s steady state, a 180 s window showing 7 prefill steps of the capped 2,048 tokens and 3 preemptions. One 7,168-token document is therefore ~100 s, round 1 (2,367 docs, 10.03M tokens) ~39 h, and the 12M-position floor ~47 h of drained production — not the ~18 h the brief assumed. The tap is the only phase whose cost is read volume per document: at the inherited 2,048-token step a document is four steps, and each step re-streams every layer's routed expert union from the container (43.4 GiB a rank live on disk). One wide step reads each layer's union once. The 'one step, one slot count' audit law binds kldref and score because their KL numbers are only comparable at an identical shape. Calib produces layer Hessians and no number any report compares, so Q38_CALIB_MNBT / Q38_CALIB_SEQS are its own, defaulting to the inherited value — nothing moves unless a probe asks. The KL modes ignore both. Tests: 589/0 — the knob reaches calib's argv, does not reach kldref's, and a non-numeric value is refused by name.
734 lines
30 KiB
Bash
Executable File
734 lines
30 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the Qwen3.8-Flash-Next five-bit campaign (feat/qwen38-w5nl64).
|
|
#
|
|
# The campaign converts the vendor BF16 checkpoint into a w5nl64a8h .rad on
|
|
# dankserver while frank-archon keeps serving. Every gate that protects the box
|
|
# or the result lives in a script under scripts/, and each gate is tested here
|
|
# through that script's own exit status and printed output — no engine, no GPU,
|
|
# no network. Fixtures are synthetic; the expected numbers are worked examples,
|
|
# not values recomputed the way the script computes them.
|
|
#
|
|
# Seams under test (the campaign's public boundaries):
|
|
# scripts/download-qwen38-bf16.sh fetch/verify of the BF16 checkpoint
|
|
# scripts/grams-to-imatrix.py calibration recording -> imatrix GGUF
|
|
# scripts/derive-protected-experts.py --target-count sweep + the parity law
|
|
# scripts/build-qwen38-bf16-rad.sh the reference container's convert argv
|
|
# scripts/run-qwen38-bf16.sh the drained-serve argv, mode selection
|
|
# scripts/convert-qwen38-w5nl64.sh the arm driver's refusal gates
|
|
# scripts/kld-gate-qwen38.sh the three-anchor interval law
|
|
# scripts/deploy-plan.py campaign files must deploy nothing
|
|
|
|
load helpers
|
|
|
|
DOWNLOAD="$REPO_ROOT/scripts/download-qwen38-bf16.sh"
|
|
PLANNER="$REPO_ROOT/scripts/deploy-plan.py"
|
|
|
|
# make_shard <dir> <name> <total-bytes> — a REAL safetensors file whose size on
|
|
# disk is exactly <total-bytes>: 8-byte header length, JSON header, tensor data.
|
|
# Real, because the verifier reads each shard's own header to learn how many
|
|
# bytes it claims to be, which is the only per-file truth available offline.
|
|
make_shard() {
|
|
mkdir -p "$1"
|
|
python3 - "$1/$2" "$3" <<'PY'
|
|
import json, struct, sys
|
|
path, want = sys.argv[1], int(sys.argv[2])
|
|
payload = max(0, want - 96)
|
|
header = b""
|
|
for _ in range(16):
|
|
# The header is part of the file, so its own length moves the target: solve
|
|
# payload = want - 8 - len(header(payload)) by fixed-point iteration.
|
|
header = json.dumps({"w": {"dtype": "F32", "shape": [payload // 4],
|
|
"data_offsets": [0, payload]}},
|
|
separators=(",", ":")).encode()
|
|
want_payload = want - 8 - len(header)
|
|
if want_payload == payload:
|
|
break
|
|
if want_payload < 0:
|
|
raise SystemExit("fixture too small to carry a header")
|
|
payload = want_payload
|
|
else:
|
|
raise SystemExit("fixture header never converged")
|
|
with open(path, "wb") as f:
|
|
f.write(struct.pack("<Q", len(header)) + header + b"\x00" * payload)
|
|
PY
|
|
}
|
|
|
|
# make_index <dir> <shard>... — a safetensors index that references them
|
|
make_index() {
|
|
local dir="$1"; shift
|
|
python3 - "$dir/model.safetensors.index.json" "$@" <<'PY'
|
|
import json, sys
|
|
out, names = sys.argv[1], sys.argv[2:]
|
|
json.dump({"metadata": {"total_size": 0},
|
|
"weight_map": {f"model.layers.{i}.attn.q.weight": n
|
|
for i, n in enumerate(names)}},
|
|
open(out, "w"))
|
|
PY
|
|
}
|
|
|
|
# three_fixture <dir> — three valid 1000-byte shards plus the index naming them
|
|
three_fixture() {
|
|
make_shard "$1" model-00001-of-00003.safetensors 1000
|
|
make_shard "$1" model-00002-of-00003.safetensors 1000
|
|
make_shard "$1" model-00003-of-00003.safetensors 1000
|
|
make_index "$1" model-00001-of-00003.safetensors \
|
|
model-00002-of-00003.safetensors \
|
|
model-00003-of-00003.safetensors
|
|
}
|
|
|
|
# --- download / verify -------------------------------------------------------
|
|
|
|
@test "the checkpoint verifier accepts a complete checkpoint and reports its footprint" {
|
|
C="$BATS_TEST_TMPDIR/complete"
|
|
three_fixture "$C"
|
|
# 3 x 1000 bytes of shards. The index file itself is deliberately not
|
|
# counted, so a Hub-side edit to a config file cannot fail this gate.
|
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"VERIFY_OK"*"3 shards"*"3000 bytes"* ]]
|
|
}
|
|
|
|
@test "the verifier names the shard whose own header disagrees with its size on disk" {
|
|
C="$BATS_TEST_TMPDIR/truncated"
|
|
three_fixture "$C"
|
|
# Cut the tail, keep the header: that is a torn transfer, which is what the
|
|
# box actually gets from an interrupted fetch. Zero-filling instead would
|
|
# destroy the header and be caught as "not a safetensors file" — a different
|
|
# gate, and it would prove nothing about the size check.
|
|
python3 -c 'import sys
|
|
p = sys.argv[1]
|
|
keep = open(p, "rb").read(400)
|
|
open(p, "wb").write(keep)' "$C/model-00002-of-00003.safetensors"
|
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"TRUNCATED"*"model-00002-of-00003.safetensors"* ]]
|
|
}
|
|
|
|
@test "the verifier refuses a missing shard and names it" {
|
|
C="$BATS_TEST_TMPDIR/missing"
|
|
three_fixture "$C"
|
|
mv "$C/model-00003-of-00003.safetensors" "$C/hidden.bin"
|
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"MISSING"*"model-00003-of-00003.safetensors"* ]]
|
|
}
|
|
|
|
@test "the verifier refuses when the shard census disagrees with the expected count" {
|
|
# The fragment law this repo learned the hard way: large quants ship as
|
|
# .partNofM fragments and a naive glob reports success on a partial tree.
|
|
C="$BATS_TEST_TMPDIR/count"
|
|
three_fixture "$C"
|
|
make_shard "$C" model-00004-of-00004.safetensors 1000
|
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"SHARD_COUNT"*"4"*"3"* ]]
|
|
}
|
|
|
|
@test "the verifier refuses a shard the index names that never landed" {
|
|
C="$BATS_TEST_TMPDIR/orphan-index"
|
|
three_fixture "$C"
|
|
rm "$C/model-00003-of-00003.safetensors"
|
|
make_index "$C" model-00001-of-00003.safetensors \
|
|
model-00002-of-00003.safetensors \
|
|
model-00003-of-00003.safetensors
|
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 2 --bytes 2000
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"MISSING"*"model-00003-of-00003.safetensors"* ]]
|
|
}
|
|
|
|
@test "the verifier treats an HTTP error body written as a shard as a failure, not a download" {
|
|
# curl -o FILE exits 0 on 401/403 and writes the error body; three shards
|
|
# of identical tiny size must never look like a checkpoint.
|
|
C="$BATS_TEST_TMPDIR/errorbody"
|
|
mkdir -p "$C"
|
|
for n in 1 2 3; do
|
|
printf '{"error":"You are not logged in"}' \
|
|
> "$C/model-0000${n}-of-00003.safetensors"
|
|
done
|
|
make_index "$C" model-00001-of-00003.safetensors \
|
|
model-00002-of-00003.safetensors \
|
|
model-00003-of-00003.safetensors
|
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"NOT_SAFETENSORS"* ]]
|
|
}
|
|
|
|
@test "the verifier refuses a shard census whose bytes do not total what the Hub declared" {
|
|
# Every shard header can be self-consistent and the tree still be the wrong
|
|
# payload: the total is the cross-check against the Hub's declared size.
|
|
C="$BATS_TEST_TMPDIR/wrongtotal"
|
|
three_fixture "$C"
|
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 2500
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"TOTAL_BYTES"*"3000"*"2500"* ]]
|
|
}
|
|
|
|
@test "the fetch path refuses when the destination cannot hold the checkpoint" {
|
|
# The campaign's peak-disk law starts at the fetch: the BF16 checkpoint and
|
|
# the reference container built from it are both ~335 GiB, and the tree that
|
|
# does not fit is discovered hours into a transfer.
|
|
C="$BATS_TEST_TMPDIR/tiny"
|
|
three_fixture "$C"
|
|
run env FRANK_QWEN38_MIN_FREE_MIB=999999999 \
|
|
bash "$DOWNLOAD" --fetch --no-hub --dir "$C" --shards 3 --bytes 3000
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"REFUSED"*"free"* ]]
|
|
}
|
|
|
|
# --- deployment scoping ------------------------------------------------------
|
|
|
|
@test "the campaign tooling deploys nothing: merging it cannot drain frank-archon" {
|
|
# Existence is asserted first: a planner that plans nothing because the
|
|
# files vanished would otherwise pass.
|
|
[ -f "$REPO_ROOT/scripts/download-qwen38-bf16.sh" ]
|
|
run python3 "$PLANNER" --target dankserver --paths \
|
|
scripts/download-qwen38-bf16.sh \
|
|
scripts/build-qwen38-bf16-rad.sh \
|
|
scripts/run-qwen38-bf16.sh \
|
|
scripts/convert-qwen38-w5nl64.sh \
|
|
scripts/kld-gate-qwen38.sh \
|
|
scripts/grams-to-imatrix.py \
|
|
scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe \
|
|
tests/qwen38_campaign.bats
|
|
[ "$status" -eq 0 ]
|
|
[ -z "$output" ]
|
|
}
|
|
|
|
# --- protected experts -------------------------------------------------------
|
|
|
|
DERIVE="$REPO_ROOT/scripts/derive-protected-experts.py"
|
|
APPLY="$REPO_ROOT/scripts/apply-protected-cast.sh"
|
|
CONVERT="$REPO_ROOT/scripts/convert-qwen38-w5nl64.sh"
|
|
GATE="$REPO_ROOT/scripts/kld-gate-qwen38.sh"
|
|
RUNNER="$REPO_ROOT/scripts/run-qwen38-bf16.sh"
|
|
PUSHER="$REPO_ROOT/scripts/calib-push.py"
|
|
WINDOW="$REPO_ROOT/scripts/qwen38-calib-window.sh"
|
|
TEMPLATE="$REPO_ROOT/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe"
|
|
|
|
# make_gram <dir> <layer> <expert> <energy> <rows> — a RADGRAM2 down-Gram whose
|
|
# trace (its down-input energy) is <energy> and whose routing count is <rows>.
|
|
make_gram() {
|
|
python3 - "$@" <<'PY'
|
|
import os, struct, sys
|
|
d, layer, e, energy, rows = sys.argv[1:6]
|
|
os.makedirs(d, exist_ok=True)
|
|
hdr = struct.pack("<8sIIQI4x", b"RADGRAM2", 2, 0, int(rows), 0)
|
|
mat = struct.pack("<4f", float(energy), 0.0, 0.0, 0.0)
|
|
with open(os.path.join(d, f"gram.r0.blk.{layer}.ffn_down_exps.{e}.bin"), "wb") as f:
|
|
f.write(hdr + mat)
|
|
PY
|
|
}
|
|
|
|
# eight_grams <dir> — one layer, eight experts inside rank half 0, energies
|
|
# 60/15/10/8/4/2/1/0.5 against equal routing. Under the default bars (3% of the
|
|
# layer's energy AND 3x its routing share) exactly ONE expert is disproportionate,
|
|
# so the parity law has to hand it a partner.
|
|
eight_grams() {
|
|
local d="$1"
|
|
make_gram "$d" 0 0 60 100
|
|
make_gram "$d" 0 1 15 100
|
|
make_gram "$d" 0 2 10 100
|
|
make_gram "$d" 0 3 8 100
|
|
make_gram "$d" 0 4 4 100
|
|
make_gram "$d" 0 5 2 100
|
|
make_gram "$d" 0 6 1 100
|
|
make_gram "$d" 0 7 0.5 100
|
|
}
|
|
|
|
cast_count() { grep -c 'cast dtype=bf16' <<< "$output"; }
|
|
|
|
@test "the bar-only pass keeps the disproportionate expert and repairs the odd rank half" {
|
|
C="$BATS_TEST_TMPDIR/calib"
|
|
eight_grams "$C"
|
|
run python3 "$DERIVE" "$C"
|
|
[ "$status" -eq 0 ]
|
|
# e0 holds 59.7% of the layer's energy on 12.5% of the routing: the only expert
|
|
# that clears three times its routing share. e1 joins it because half 0 was odd.
|
|
[ "$(cast_count)" -eq 2 ]
|
|
[[ "$output" == *"blk.0.ffn_*_exps.0.weight"* ]]
|
|
[[ "$output" == *"blk.0.ffn_*_exps.1.weight"* ]]
|
|
[[ "$output" != *"ffn_*_exps.2.weight"* ]]
|
|
[[ "$output" == *"half 0 odd -> added e1"* ]]
|
|
}
|
|
|
|
@test "--target-count lands the byte bill that was asked for, and never buys more" {
|
|
# The count is a disk and VRAM decision (each protected expert costs bf16 twice,
|
|
# once per rank half's copy), so landing it matters as much as not missing it.
|
|
C="$BATS_TEST_TMPDIR/calib"
|
|
eight_grams "$C"
|
|
run python3 "$DERIVE" "$C" --target-count 6
|
|
[ "$status" -eq 0 ]
|
|
[ "$(cast_count)" -eq 6 ]
|
|
[[ "$output" == *"count fill -> added e2"* ]]
|
|
[[ "$output" != *"ffn_*_exps.6.weight"* ]]
|
|
# the energy share the set bottoms out at is the bar the campaign paid for
|
|
[[ "$output" == *"total protected: 6 (target 6, energy cutoff"* ]]
|
|
}
|
|
|
|
@test "--target-count cannot invent experts: the pool's ceiling is reported, not padded" {
|
|
C="$BATS_TEST_TMPDIR/calib"
|
|
eight_grams "$C"
|
|
run python3 "$DERIVE" "$C" --target-count 20
|
|
[ "$status" -eq 0 ]
|
|
[ "$(cast_count)" -eq 8 ]
|
|
[[ "$output" == *"total protected: 8 (target 20"* ]]
|
|
}
|
|
|
|
@test "a protected set that cannot be made even is refused instead of emitted" {
|
|
# One disproportionate expert in half 0, its only companion in half 1: no
|
|
# candidate could make half 0 even, and an odd half hands the four-bit GEMM two
|
|
# tables that are not equal. Refusing is the only honest answer.
|
|
C="$BATS_TEST_TMPDIR/calib"
|
|
make_gram "$C" 0 0 100 100
|
|
make_gram "$C" 0 300 1 1000
|
|
run python3 "$DERIVE" "$C"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"PARITY LAW BROKEN"* ]]
|
|
[[ "$output" == *"blk.0 half 0 holds 1"* ]]
|
|
[ "$(cast_count)" -eq 0 ]
|
|
}
|
|
|
|
@test "the splice rewrites the run's copy above the expert rule and refuses a second run" {
|
|
C="$BATS_TEST_TMPDIR/calib"
|
|
eight_grams "$C"
|
|
W="$BATS_TEST_TMPDIR/recipe-work.recipe"
|
|
cp "$TEMPLATE" "$W"
|
|
run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 0
|
|
[ "$status" -eq 0 ]
|
|
grep -q "derive-protected-experts.py rewrote" "$W"
|
|
# The derived lines must precede the expert rule: the first rule whose glob
|
|
# matches a weight decides it, so a cast line below the expert line never fires.
|
|
cast_line=$(grep -n 'cast dtype=bf16' "$W" | head -1 | cut -d: -f1)
|
|
expert_line=$(grep -n 'ffn_\*_exps\.\*\.weight' "$W" | head -1 | cut -d: -f1)
|
|
[ "$cast_line" -lt "$expert_line" ]
|
|
# The committed template stays the clean, reviewable baseline.
|
|
! grep -q "derive-protected-experts.py rewrote" "$TEMPLATE"
|
|
# A second run on the same copy would derive from a recording it already spent.
|
|
run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 0
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"refusing to re-derive"* ]]
|
|
}
|
|
|
|
@test "the splice refuses before touching the recipe when the count is not reached" {
|
|
C="$BATS_TEST_TMPDIR/calib"
|
|
eight_grams "$C"
|
|
W="$BATS_TEST_TMPDIR/recipe-work.recipe"
|
|
cp "$TEMPLATE" "$W"
|
|
run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 20
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"target was 20"* ]]
|
|
! grep -q "derive-protected-experts.py rewrote" "$W"
|
|
}
|
|
|
|
# --- the arm driver ----------------------------------------------------------
|
|
|
|
# drv_fixture <dir> — the smallest tree the driver's gates accept: a checkpoint
|
|
# with its index and tokenizer, three grams, an imatrix inside the campaign dir
|
|
# the container mounts, and an empty output dir.
|
|
drv_fixture() {
|
|
local d="$1"
|
|
mkdir -p "$d/ck/Qwen3.8-Flash-Next" "$d/campaign" "$d/calib" "$d/out"
|
|
: > "$d/ck/Qwen3.8-Flash-Next/model.safetensors.index.json"
|
|
: > "$d/ck/Qwen3.8-Flash-Next/tokenizer.json"
|
|
make_gram "$d/calib" 0 0 60 100
|
|
make_gram "$d/calib" 0 1 15 100
|
|
make_gram "$d/calib" 0 2 10 100
|
|
# four experts, not three: a thin pool that cannot be made even trips the parity
|
|
# audit, and this fixture is meant to reach the later gates
|
|
make_gram "$d/calib" 0 3 8 100
|
|
: > "$d/campaign/imatrix.gguf"
|
|
}
|
|
|
|
# run_drv <driver args and K=V overrides, any order> — the driver with every path
|
|
# pointed at the fixture. Override words are separated out because `env` only
|
|
# accepts assignments ahead of the command; later assignments win, so a test can
|
|
# break exactly one input.
|
|
run_drv() {
|
|
local d="$BATS_TEST_TMPDIR/drv" assigns=() flags=() w
|
|
for w in "$@"; do
|
|
if [[ "$w" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]]; then assigns+=("$w"); else flags+=("$w"); fi
|
|
done
|
|
run env CKPT="$d/ck/Qwen3.8-Flash-Next" CAMPAIGN="$d/campaign" STATE="$d/campaign" \
|
|
CALIB="$d/calib" OUTDIR="$d/out" IMATRIX="$d/campaign/imatrix.gguf" \
|
|
MIN_GRAMS=2 TARGET_COUNT=0 ${assigns[@]+"${assigns[@]}"} \
|
|
bash "$CONVERT" ${flags[@]+"${flags[@]}"}
|
|
}
|
|
|
|
@test "the dry run prints the plan and then the convert for every arm" {
|
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
|
run_drv --dry-run
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"== arm cd3 =="* ]]
|
|
[[ "$output" == *"== arm cd6 =="* ]]
|
|
[[ "$output" == *"== arm ao1 =="* ]]
|
|
# one --plan-only bill per arm, and one convert line per arm beside it
|
|
[ "$(grep -c -- '--plan-only' <<< "$output")" -eq 3 ]
|
|
[ "$(grep -c 'rad-convert' <<< "$output")" -eq 6 ]
|
|
awk '/arm cd3/ {next}
|
|
/qwen38-w5nl64-cd3\.rad/ { if ($0 ~ /--plan-only/) p = NR; else c = NR }
|
|
END { exit !(p && c && p < c) }' <<< "$output"
|
|
}
|
|
|
|
@test "the cd6 arm re-sends the whole expert option set with cd=6; the control sends none" {
|
|
# --quant replaces the recipe line wholesale, so an override that forgot an
|
|
# option would quietly convert with the default for it.
|
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
|
run_drv --arm cd3 --arm cd6 --dry-run
|
|
[ "$status" -eq 0 ]
|
|
cd6line=$(grep -- '-cd6\.rad --quant' <<< "$output" | head -1)
|
|
[[ "$cd6line" == *"table=w5nl"* ]]
|
|
[[ "$cd6line" == *"group=64"* ]]
|
|
[[ "$cd6line" == *"scale=fp8_e4m3"* ]]
|
|
[[ "$cd6line" == *"transform=fwht128"* ]]
|
|
[[ "$cd6line" == *"calib=\$CALIB"* ]]
|
|
[[ "$cd6line" == *",cd=6"* ]]
|
|
cd3line=$(grep -- '-cd3\.rad' <<< "$output" | grep -v -- '--plan-only' | head -1)
|
|
[[ "$cd3line" != *"--quant"* ]]
|
|
}
|
|
|
|
@test "the ao1 arm carries act_order=1 and no cd=, because libquant refuses the pair" {
|
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
|
run_drv --arm ao1 --dry-run
|
|
[ "$status" -eq 0 ]
|
|
line=$(grep -- '-ao1\.rad --quant' <<< "$output" | head -1)
|
|
[[ "$line" == *"act_order=1"* ]]
|
|
[[ "$line" != *",cd="* ]]
|
|
}
|
|
|
|
@test "the driver splices the protected plane into its own copy, never the template" {
|
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
|
run_drv --dry-run
|
|
[[ "$output" == *"apply-protected-cast.sh $BATS_TEST_TMPDIR/drv/calib "* ]]
|
|
[[ "$output" == *"/campaign/recipe-"* ]]
|
|
[[ "$output" != *"$TEMPLATE"* ]]
|
|
}
|
|
|
|
@test "the driver refuses while the BF16 reference container still holds the volume" {
|
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
|
: > "$BATS_TEST_TMPDIR/drv/out/qwen38-bf16.rad"
|
|
run_drv --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"peak-disk"* ]]
|
|
}
|
|
|
|
@test "a thin recording is refused: the driver never converts uncalibrated" {
|
|
# rad-convert converts WITHOUT a calibration recording with one stderr note,
|
|
# which is silently worse than a GPTQ run that failed.
|
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
|
run_drv MIN_GRAMS=9 --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"run-qwen38-bf16.sh calib"* ]]
|
|
}
|
|
|
|
@test "an imatrix outside the mounted campaign dir is refused before any convert" {
|
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
|
mv "$BATS_TEST_TMPDIR/drv/campaign/imatrix.gguf" "$BATS_TEST_TMPDIR/imatrix.gguf"
|
|
run_drv IMATRIX="$BATS_TEST_TMPDIR/imatrix.gguf" --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"not in the campaign dir"* ]]
|
|
}
|
|
|
|
@test "the driver never overwrites an arm that already exists" {
|
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
|
: > "$BATS_TEST_TMPDIR/drv/out/qwen38-w5nl64-cd6.rad"
|
|
run_drv --arm cd6 --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"move it to trash first"* ]]
|
|
}
|
|
|
|
@test "an unknown arm name is refused before the gates run" {
|
|
# No fixture built: the name check must not need a checkpoint to exist.
|
|
run_drv --arm cd9 --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"unknown arm: cd9"* ]]
|
|
}
|
|
|
|
@test "the driver stops when the protected derivation cannot land the target" {
|
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
|
run_drv PY=python3 TARGET_COUNT=20 --arm cd3
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"target was 20"* ]]
|
|
[[ "$output" == *"would convert unprotected"* ]]
|
|
[[ "$output" != *"--entrypoint"* ]]
|
|
}
|
|
|
|
# --- the acceptance gate -----------------------------------------------------
|
|
|
|
# kld_report <path> <model> <kl mean> <top1 fraction> — one score report, in the
|
|
# shape `run-qwen38-bf16.sh score` writes: an `all` summary plus a per-source one.
|
|
kld_report() {
|
|
python3 - "$@" <<'PY'
|
|
import json, sys
|
|
path, model, kl, top1 = sys.argv[1:5]
|
|
kl, top1 = float(kl), float(top1)
|
|
group = {"kld": {"mean": kl, "p99_9": kl * 4}, "top1_agreement": top1, "positions": 50432}
|
|
json.dump({"model": model, "all": group,
|
|
"by_source": {"ops": dict(group, positions=20000)}}, open(path, "w"))
|
|
PY
|
|
}
|
|
|
|
gate_case() { # gate_case <kl fp8> <kl iq4r> <kl cand> <top1 cand> [extra flags...]
|
|
D="$BATS_TEST_TMPDIR/kl"
|
|
mkdir -p "$D"
|
|
kld_report "$D/fp8.json" fp8-native "$1" 0.981
|
|
kld_report "$D/iq4r.json" iq4r-moe "$2" 0.962
|
|
kld_report "$D/cand.json" w5nl64 "$3" "$4"
|
|
run bash "$GATE" "$D/fp8.json" "$D/iq4r.json" "$D/cand.json" "${@:5}"
|
|
}
|
|
|
|
@test "an arm strictly inside the interval that closes half the gap passes" {
|
|
gate_case 0.0021 0.0190 0.0090 0.971
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"closes 59.2%"* ]]
|
|
[[ "$output" == *"KLD GATE PASS"* ]]
|
|
}
|
|
|
|
@test "an arm worse than the served payload fails the interval and the recovery" {
|
|
gate_case 0.0021 0.0190 0.0210 0.971
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"[FAIL] 2."* ]]
|
|
[[ "$output" == *"[FAIL] 3."* ]]
|
|
[[ "$output" == *"KLD GATE FAIL"* ]]
|
|
}
|
|
|
|
@test "anchors that are not ordered stop the gate: the instrument is wrong, not the arm" {
|
|
# If the near-lossless FP8 release does not sit CLOSER to the bf16 reference
|
|
# than the 4-bit payload does, the corpus, tokenizer or boot shape is broken and
|
|
# no arm number can mean anything.
|
|
gate_case 0.0300 0.0190 0.0120 0.971
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"[FAIL] 1."* ]]
|
|
[[ "$output" == *"gap is not positive"* ]]
|
|
}
|
|
|
|
@test "an arm measured outside the interval fails even when it looks better" {
|
|
# Closer to the reference than the achievable anchor is a measurement fault.
|
|
gate_case 0.0021 0.0190 0.0010 0.971
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"[FAIL] 2."* ]]
|
|
}
|
|
|
|
@test "an arm that buys KL with top-1 agreement fails" {
|
|
gate_case 0.0021 0.0190 0.0090 0.940
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"[PASS] 3."* ]]
|
|
[[ "$output" == *"[FAIL] 4."* ]]
|
|
}
|
|
|
|
@test "--recover tightens the same numbers until they fail" {
|
|
gate_case 0.0021 0.0190 0.0090 0.971 --recover 0.8
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"needs 80%"* ]]
|
|
[[ "$output" == *"[FAIL] 3."* ]]
|
|
}
|
|
|
|
@test "a slice the report does not carry is named, not guessed at" {
|
|
gate_case 0.0021 0.0190 0.0090 0.971 --slice code
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"has no slice 'code'"* ]]
|
|
[[ "$output" == *"it has: ops"* ]]
|
|
}
|
|
|
|
# --- the n-gram tier: the pin that starved the calibration boot ---------------
|
|
|
|
@test "the calibration boot reads the n-gram table from the file tier" {
|
|
# 2026-10-29: 95.37 GiB pinned beside a 65.59 GiB/rank pool left 8.06 GiB of
|
|
# RAM and the boot died "startup stopped: out of memory". calib only needs the
|
|
# routed weights prefilled, so it reads the table off the container's file tier.
|
|
run bash "$RUNNER" calib --dry-run
|
|
[ "$status" -eq 0 ]
|
|
grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'disk'
|
|
}
|
|
|
|
@test "the recording and every score pass keep production's n-gram placement" {
|
|
# Those numbers are only comparable when the shape is identical; the tier is
|
|
# part of the shape.
|
|
run bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'ram'
|
|
}
|
|
|
|
@test "an unknown n-gram placement is refused by name before the boot" {
|
|
run env FRANK_QWEN38_NGRAM_PLACEMENT=ssd bash "$RUNNER" calib --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"auto|vram|ram|disk"* ]]
|
|
}
|
|
|
|
# --- the pusher: a tap without a live rig is not a tap ------------------------
|
|
|
|
# stub_podman <dir> <running|absent> — a podman that answers the one question the
|
|
# pusher asks: is the rig container running?
|
|
stub_podman() {
|
|
mkdir -p "$1/bin"
|
|
if [ "$2" = running ]; then
|
|
printf '#!/usr/bin/env bash\necho true\n' > "$1/bin/podman"
|
|
elif [ "$2" = stopped ]; then
|
|
printf '#!/usr/bin/env bash\necho false\n' > "$1/bin/podman"
|
|
else
|
|
printf '#!/usr/bin/env bash\nexit 125\n' > "$1/bin/podman"
|
|
fi
|
|
chmod +x "$1/bin/podman"
|
|
}
|
|
|
|
pusher_fixture() { # pusher_fixture <marker-container or empty>
|
|
D="$BATS_TEST_TMPDIR/push"
|
|
mkdir -p "$D/tap" "$D/bin"
|
|
printf '{"prompt":"hello"}\n' > "$D/corpus.jsonl"
|
|
if [ -n "$1" ]; then
|
|
printf 'container=%s\nimage=radiance@sha256:test\nmode=calib\npid=1\n' "$1" \
|
|
> "$D/tap/.rig-boot"
|
|
fi
|
|
echo "$D"
|
|
}
|
|
|
|
@test "the pusher refuses a tap that carries no rig marker: production is not a rig" {
|
|
# With no marker, the old driver fed four rounds of 10M tokens into restored
|
|
# production after the rig's boot had died, and recorded nothing.
|
|
D="$(pusher_fixture '')"
|
|
run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" --port 1
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"no rig marker"* ]]
|
|
}
|
|
|
|
@test "the pusher refuses a marker that names another container" {
|
|
D="$(pusher_fixture arbench)"
|
|
run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" \
|
|
--container qwen38-bf16 --port 1
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"names container 'arbench'"* ]]
|
|
[[ "$output" == *"targets 'qwen38-bf16'"* ]]
|
|
}
|
|
|
|
@test "the pusher refuses when the rig container is not running" {
|
|
D="$(pusher_fixture qwen38-bf16)"
|
|
stub_podman "$D" stopped
|
|
run env PATH="$D/bin:$PATH" python3 "$PUSHER" --corpus "$D/corpus.jsonl" \
|
|
--tap-dir "$D/tap" --container qwen38-bf16 --port 1
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"is not running"* ]]
|
|
}
|
|
|
|
# --- the acceptance gate without an FP8 anchor -------------------------------
|
|
|
|
# kld_report_slices <path> <model> <kl all> <kl code> <top1 all> <top1 code>
|
|
kld_report_slices() {
|
|
python3 - "$@" <<'PY'
|
|
import json, sys
|
|
path, model, kl_all, kl_code, t_all, t_code = sys.argv[1:7]
|
|
def g(kl, t, n=25000):
|
|
return {"kld": {"mean": float(kl), "p99_9": float(kl) * 4},
|
|
"top1_agreement": float(t), "positions": n}
|
|
json.dump({"model": model, "all": g(kl_all, t_all),
|
|
"by_source": {"code": g(kl_code, t_code), "ops": g(kl_all, t_all)}},
|
|
open(path, "w"))
|
|
PY
|
|
}
|
|
|
|
gate2() { # gate2 <kl iq4r> <kl cand> <top1 iq4r> <top1 cand> [extra]
|
|
D="$BATS_TEST_TMPDIR/kl2"
|
|
mkdir -p "$D"
|
|
kld_report_slices "$D/iq4r.json" iq4r-moe "$1" "$1" "$3" "$3"
|
|
kld_report_slices "$D/cand.json" w5nl64 "$2" "$2" "$4" "$4"
|
|
run bash "$GATE" "$D/iq4r.json" "$D/cand.json" "${@:5}"
|
|
}
|
|
|
|
@test "a candidate that beats the served payload on mean, tail and top-1 passes without an anchor" {
|
|
gate2 0.0190 0.0090 0.962 0.971
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"candidate vs served (no FP8 anchor)"* ]]
|
|
[[ "$output" == *"KLD GATE PASS"* ]]
|
|
}
|
|
|
|
@test "a better mean with a worse tail fails: the tail is where a mean hides its damage" {
|
|
D="$BATS_TEST_TMPDIR/kl3"
|
|
mkdir -p "$D"
|
|
kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962
|
|
python3 - "$D/cand.json" <<'PY'
|
|
import json, sys
|
|
# mean better than the served payload, p99.9 worse: the shape a mean-only gate waves through
|
|
g = {"kld": {"mean": 0.0090, "p99_9": 0.4000}, "top1_agreement": 0.971, "positions": 25000}
|
|
json.dump({"model": "w5nl64", "all": g, "by_source": {"code": dict(g), "ops": dict(g)}},
|
|
open(sys.argv[1], "w"))
|
|
PY
|
|
run bash "$GATE" "$D/iq4r.json" "$D/cand.json"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"[FAIL] 2."* ]]
|
|
[[ "$output" == *"KL p99.9"* ]]
|
|
}
|
|
|
|
@test "a candidate that wins overall but regresses one source slice fails" {
|
|
D="$BATS_TEST_TMPDIR/kl4"
|
|
mkdir -p "$D"
|
|
kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962
|
|
# 'all' is better, 'code' is worse: exactly the trade a mean-only gate cannot see
|
|
kld_report_slices "$D/cand.json" w5nl64 0.0090 0.0300 0.971 0.950
|
|
run bash "$GATE" "$D/iq4r.json" "$D/cand.json"
|
|
[ "$status" -eq 1 ]
|
|
[[ "$output" == *"[PASS] 1."* ]]
|
|
[[ "$output" == *"[FAIL] 4."* ]]
|
|
[[ "$output" == *"regressed: code"* ]]
|
|
}
|
|
|
|
# --- the calibration window: gated, and it says so ---------------------------
|
|
|
|
window_fixture() {
|
|
D="$BATS_TEST_TMPDIR/win"
|
|
mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora"
|
|
echo '{}' > "$D/campaign/kld-ref/kld.json"
|
|
: > "$D/rad/qwen38-bf16.rad"
|
|
: > "$D/corpora/calib.jsonl"
|
|
echo "$D"
|
|
}
|
|
|
|
@test "the calibration window refuses before touching a card without a reference recording" {
|
|
D="$BATS_TEST_TMPDIR/win-less"
|
|
mkdir -p "$D/campaign/calib"
|
|
run env FRANK_QWEN38_CAMPAIGN="$D/campaign" bash "$WINDOW"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"no reference recording"* ]]
|
|
}
|
|
|
|
@test "the calibration window's dry run prints the gated plan, including what it would bank" {
|
|
D="$(window_fixture)"
|
|
run env FRANK_QWEN38_CAMPAIGN="$D/campaign" \
|
|
FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \
|
|
Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" bash "$WINDOW" --dry-run
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"run-qwen38-bf16.sh calib"* ]]
|
|
[[ "$output" == *"else kill the rig"* ]]
|
|
[[ "$output" == *"calib-push.py"* ]]
|
|
[[ "$output" == *"AGENTTRASH"* ]]
|
|
}
|
|
|
|
@test "a flat calibration round is not a failure: the tap's floor is pooled" {
|
|
# round 1 of a 10.03M-token corpus against a 12M floor seals nothing, and the
|
|
# first version of this gate aborted the run there — one round too early.
|
|
run python3 - "$PUSHER" <<'PY'
|
|
import importlib.util, sys
|
|
spec = importlib.util.spec_from_file_location("cp", sys.argv[1])
|
|
m = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(m)
|
|
print(m.round_verdict(8192, 8192, False), # flat: continue
|
|
m.round_verdict(8192, 4096, False), # shrink: fail
|
|
m.round_verdict(8192, 20000, False), # grew: continue
|
|
m.round_verdict(0, 0, True)) # done: stop
|
|
PY
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "flat shrink grew done" ]
|
|
}
|
|
|
|
@test "the calibration boot's step is its own knob; the KL modes keep production's" {
|
|
# Calib is the only phase whose cost is read volume per document: four 2,048-token steps
|
|
# re-stream each layer's routed experts four times. Measured 2026-10-29: 71 tok/s, 47 h tap.
|
|
run env Q38_CALIB_MNBT=8192 bash "$RUNNER" calib --dry-run
|
|
[ "$status" -eq 0 ]
|
|
grep -A1 -x -- '--max-num-batched-tokens' <<<"$output" | grep -qx '8192'
|
|
# and the same knob must NOT reach the measuring shape those numbers are compared in
|
|
run env Q38_CALIB_MNBT=8192 bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
! grep -qx '8192' <<<"$output"
|
|
run env Q38_CALIB_MNBT=wide bash "$RUNNER" calib --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"Q38_CALIB_MNBT must be a number"* ]]
|
|
}
|