#!/usr/bin/env bats # Seam: the Qwen3.8-Flash-Next five-bit campaign (feat/qwen38-w5nl64). # # The campaign converts the vendor BF16 checkpoint into a w5nl64a8h .rad on # dankserver while frank-archon keeps serving. Every gate that protects the box # or the result lives in a script under scripts/, and each gate is tested here # through that script's own exit status and printed output — no engine, no GPU, # no network. Fixtures are synthetic; the expected numbers are worked examples, # not values recomputed the way the script computes them. # # Seams under test (the campaign's public boundaries): # scripts/download-qwen38-bf16.sh fetch/verify of the BF16 checkpoint # scripts/grams-to-imatrix.py calibration recording -> imatrix GGUF # scripts/derive-protected-experts.py --target-count sweep + the parity law # scripts/build-qwen38-bf16-rad.sh the reference container's convert argv # scripts/run-qwen38-bf16.sh the drained-serve argv, mode selection # scripts/convert-qwen38-w5nl64.sh the arm driver's refusal gates # scripts/kld-gate-qwen38.sh the three-anchor interval law # scripts/deploy-plan.py campaign files must deploy nothing load helpers DOWNLOAD="$REPO_ROOT/scripts/download-qwen38-bf16.sh" PLANNER="$REPO_ROOT/scripts/deploy-plan.py" # make_shard
... — a safetensors index that references them make_index() { local dir="$1"; shift python3 - "$dir/model.safetensors.index.json" "$@" <<'PY' import json, sys out, names = sys.argv[1], sys.argv[2:] json.dump({"metadata": {"total_size": 0}, "weight_map": {f"model.layers.{i}.attn.q.weight": n for i, n in enumerate(names)}}, open(out, "w")) PY } # three_fixture — three valid 1000-byte shards plus the index naming them three_fixture() { make_shard "$1" model-00001-of-00003.safetensors 1000 make_shard "$1" model-00002-of-00003.safetensors 1000 make_shard "$1" model-00003-of-00003.safetensors 1000 make_index "$1" model-00001-of-00003.safetensors \ model-00002-of-00003.safetensors \ model-00003-of-00003.safetensors } # --- download / verify ------------------------------------------------------- @test "the checkpoint verifier accepts a complete checkpoint and reports its footprint" { C="$BATS_TEST_TMPDIR/complete" three_fixture "$C" # 3 x 1000 bytes of shards. The index file itself is deliberately not # counted, so a Hub-side edit to a config file cannot fail this gate. run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000 [ "$status" -eq 0 ] [[ "$output" == *"VERIFY_OK"*"3 shards"*"3000 bytes"* ]] } @test "the verifier names the shard whose own header disagrees with its size on disk" { C="$BATS_TEST_TMPDIR/truncated" three_fixture "$C" # Cut the tail, keep the header: that is a torn transfer, which is what the # box actually gets from an interrupted fetch. Zero-filling instead would # destroy the header and be caught as "not a safetensors file" — a different # gate, and it would prove nothing about the size check. python3 -c 'import sys p = sys.argv[1] keep = open(p, "rb").read(400) open(p, "wb").write(keep)' "$C/model-00002-of-00003.safetensors" run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000 [ "$status" -eq 1 ] [[ "$output" == *"TRUNCATED"*"model-00002-of-00003.safetensors"* ]] } @test "the verifier refuses a missing shard and names it" { C="$BATS_TEST_TMPDIR/missing" three_fixture "$C" mv "$C/model-00003-of-00003.safetensors" "$C/hidden.bin" run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000 [ "$status" -eq 1 ] [[ "$output" == *"MISSING"*"model-00003-of-00003.safetensors"* ]] } @test "the verifier refuses when the shard census disagrees with the expected count" { # The fragment law this repo learned the hard way: large quants ship as # .partNofM fragments and a naive glob reports success on a partial tree. C="$BATS_TEST_TMPDIR/count" three_fixture "$C" make_shard "$C" model-00004-of-00004.safetensors 1000 run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000 [ "$status" -eq 1 ] [[ "$output" == *"SHARD_COUNT"*"4"*"3"* ]] } @test "the verifier refuses a shard the index names that never landed" { C="$BATS_TEST_TMPDIR/orphan-index" three_fixture "$C" rm "$C/model-00003-of-00003.safetensors" make_index "$C" model-00001-of-00003.safetensors \ model-00002-of-00003.safetensors \ model-00003-of-00003.safetensors run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 2 --bytes 2000 [ "$status" -eq 1 ] [[ "$output" == *"MISSING"*"model-00003-of-00003.safetensors"* ]] } @test "the verifier treats an HTTP error body written as a shard as a failure, not a download" { # curl -o FILE exits 0 on 401/403 and writes the error body; three shards # of identical tiny size must never look like a checkpoint. C="$BATS_TEST_TMPDIR/errorbody" mkdir -p "$C" for n in 1 2 3; do printf '{"error":"You are not logged in"}' \ > "$C/model-0000${n}-of-00003.safetensors" done make_index "$C" model-00001-of-00003.safetensors \ model-00002-of-00003.safetensors \ model-00003-of-00003.safetensors run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000 [ "$status" -eq 1 ] [[ "$output" == *"NOT_SAFETENSORS"* ]] } @test "the verifier refuses a shard census whose bytes do not total what the Hub declared" { # Every shard header can be self-consistent and the tree still be the wrong # payload: the total is the cross-check against the Hub's declared size. C="$BATS_TEST_TMPDIR/wrongtotal" three_fixture "$C" run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 2500 [ "$status" -eq 1 ] [[ "$output" == *"TOTAL_BYTES"*"3000"*"2500"* ]] } @test "the fetch path refuses when the destination cannot hold the checkpoint" { # The campaign's peak-disk law starts at the fetch: the BF16 checkpoint and # the reference container built from it are both ~335 GiB, and the tree that # does not fit is discovered hours into a transfer. C="$BATS_TEST_TMPDIR/tiny" three_fixture "$C" run env FRANK_QWEN38_MIN_FREE_MIB=999999999 \ bash "$DOWNLOAD" --fetch --no-hub --dir "$C" --shards 3 --bytes 3000 [ "$status" -eq 2 ] [[ "$output" == *"REFUSED"*"free"* ]] } # --- deployment scoping ------------------------------------------------------ @test "the campaign tooling deploys nothing: merging it cannot drain frank-archon" { # Existence is asserted first: a planner that plans nothing because the # files vanished would otherwise pass. [ -f "$REPO_ROOT/scripts/download-qwen38-bf16.sh" ] run python3 "$PLANNER" --target dankserver --paths \ scripts/download-qwen38-bf16.sh \ scripts/build-qwen38-bf16-rad.sh \ scripts/run-qwen38-bf16.sh \ scripts/convert-qwen38-w5nl64.sh \ scripts/kld-gate-qwen38.sh \ scripts/grams-to-imatrix.py \ scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe \ tests/qwen38_campaign.bats [ "$status" -eq 0 ] [ -z "$output" ] } # --- protected experts ------------------------------------------------------- DERIVE="$REPO_ROOT/scripts/derive-protected-experts.py" APPLY="$REPO_ROOT/scripts/apply-protected-cast.sh" CONVERT="$REPO_ROOT/scripts/convert-qwen38-w5nl64.sh" GATE="$REPO_ROOT/scripts/kld-gate-qwen38.sh" TEMPLATE="$REPO_ROOT/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe" # make_gram — a RADGRAM2 down-Gram whose # trace (its down-input energy) is and whose routing count is . make_gram() { python3 - "$@" <<'PY' import os, struct, sys d, layer, e, energy, rows = sys.argv[1:6] os.makedirs(d, exist_ok=True) hdr = struct.pack("<8sIIQI4x", b"RADGRAM2", 2, 0, int(rows), 0) mat = struct.pack("<4f", float(energy), 0.0, 0.0, 0.0) with open(os.path.join(d, f"gram.r0.blk.{layer}.ffn_down_exps.{e}.bin"), "wb") as f: f.write(hdr + mat) PY } # eight_grams — one layer, eight experts inside rank half 0, energies # 60/15/10/8/4/2/1/0.5 against equal routing. Under the default bars (3% of the # layer's energy AND 3x its routing share) exactly ONE expert is disproportionate, # so the parity law has to hand it a partner. eight_grams() { local d="$1" make_gram "$d" 0 0 60 100 make_gram "$d" 0 1 15 100 make_gram "$d" 0 2 10 100 make_gram "$d" 0 3 8 100 make_gram "$d" 0 4 4 100 make_gram "$d" 0 5 2 100 make_gram "$d" 0 6 1 100 make_gram "$d" 0 7 0.5 100 } cast_count() { grep -c 'cast dtype=bf16' <<< "$output"; } @test "the bar-only pass keeps the disproportionate expert and repairs the odd rank half" { C="$BATS_TEST_TMPDIR/calib" eight_grams "$C" run python3 "$DERIVE" "$C" [ "$status" -eq 0 ] # e0 holds 59.7% of the layer's energy on 12.5% of the routing: the only expert # that clears three times its routing share. e1 joins it because half 0 was odd. [ "$(cast_count)" -eq 2 ] [[ "$output" == *"blk.0.ffn_*_exps.0.weight"* ]] [[ "$output" == *"blk.0.ffn_*_exps.1.weight"* ]] [[ "$output" != *"ffn_*_exps.2.weight"* ]] [[ "$output" == *"half 0 odd -> added e1"* ]] } @test "--target-count lands the byte bill that was asked for, and never buys more" { # The count is a disk and VRAM decision (each protected expert costs bf16 twice, # once per rank half's copy), so landing it matters as much as not missing it. C="$BATS_TEST_TMPDIR/calib" eight_grams "$C" run python3 "$DERIVE" "$C" --target-count 6 [ "$status" -eq 0 ] [ "$(cast_count)" -eq 6 ] [[ "$output" == *"count fill -> added e2"* ]] [[ "$output" != *"ffn_*_exps.6.weight"* ]] # the energy share the set bottoms out at is the bar the campaign paid for [[ "$output" == *"total protected: 6 (target 6, energy cutoff"* ]] } @test "--target-count cannot invent experts: the pool's ceiling is reported, not padded" { C="$BATS_TEST_TMPDIR/calib" eight_grams "$C" run python3 "$DERIVE" "$C" --target-count 20 [ "$status" -eq 0 ] [ "$(cast_count)" -eq 8 ] [[ "$output" == *"total protected: 8 (target 20"* ]] } @test "a protected set that cannot be made even is refused instead of emitted" { # One disproportionate expert in half 0, its only companion in half 1: no # candidate could make half 0 even, and an odd half hands the four-bit GEMM two # tables that are not equal. Refusing is the only honest answer. C="$BATS_TEST_TMPDIR/calib" make_gram "$C" 0 0 100 100 make_gram "$C" 0 300 1 1000 run python3 "$DERIVE" "$C" [ "$status" -eq 2 ] [[ "$output" == *"PARITY LAW BROKEN"* ]] [[ "$output" == *"blk.0 half 0 holds 1"* ]] [ "$(cast_count)" -eq 0 ] } @test "the splice rewrites the run's copy above the expert rule and refuses a second run" { C="$BATS_TEST_TMPDIR/calib" eight_grams "$C" W="$BATS_TEST_TMPDIR/recipe-work.recipe" cp "$TEMPLATE" "$W" run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 0 [ "$status" -eq 0 ] grep -q "derive-protected-experts.py rewrote" "$W" # The derived lines must precede the expert rule: the first rule whose glob # matches a weight decides it, so a cast line below the expert line never fires. cast_line=$(grep -n 'cast dtype=bf16' "$W" | head -1 | cut -d: -f1) expert_line=$(grep -n 'ffn_\*_exps\.\*\.weight' "$W" | head -1 | cut -d: -f1) [ "$cast_line" -lt "$expert_line" ] # The committed template stays the clean, reviewable baseline. ! grep -q "derive-protected-experts.py rewrote" "$TEMPLATE" # A second run on the same copy would derive from a recording it already spent. run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 0 [ "$status" -ne 0 ] [[ "$output" == *"refusing to re-derive"* ]] } @test "the splice refuses before touching the recipe when the count is not reached" { C="$BATS_TEST_TMPDIR/calib" eight_grams "$C" W="$BATS_TEST_TMPDIR/recipe-work.recipe" cp "$TEMPLATE" "$W" run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 20 [ "$status" -eq 1 ] [[ "$output" == *"target was 20"* ]] ! grep -q "derive-protected-experts.py rewrote" "$W" } # --- the arm driver ---------------------------------------------------------- # drv_fixture — the smallest tree the driver's gates accept: a checkpoint # with its index and tokenizer, three grams, an imatrix inside the campaign dir # the container mounts, and an empty output dir. drv_fixture() { local d="$1" mkdir -p "$d/ck/Qwen3.8-Flash-Next" "$d/campaign" "$d/calib" "$d/out" : > "$d/ck/Qwen3.8-Flash-Next/model.safetensors.index.json" : > "$d/ck/Qwen3.8-Flash-Next/tokenizer.json" make_gram "$d/calib" 0 0 60 100 make_gram "$d/calib" 0 1 15 100 make_gram "$d/calib" 0 2 10 100 # four experts, not three: a thin pool that cannot be made even trips the parity # audit, and this fixture is meant to reach the later gates make_gram "$d/calib" 0 3 8 100 : > "$d/campaign/imatrix.gguf" } # run_drv — the driver with every path # pointed at the fixture. Override words are separated out because `env` only # accepts assignments ahead of the command; later assignments win, so a test can # break exactly one input. run_drv() { local d="$BATS_TEST_TMPDIR/drv" assigns=() flags=() w for w in "$@"; do if [[ "$w" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]]; then assigns+=("$w"); else flags+=("$w"); fi done run env CKPT="$d/ck/Qwen3.8-Flash-Next" CAMPAIGN="$d/campaign" STATE="$d/campaign" \ CALIB="$d/calib" OUTDIR="$d/out" IMATRIX="$d/campaign/imatrix.gguf" \ MIN_GRAMS=2 TARGET_COUNT=0 ${assigns[@]+"${assigns[@]}"} \ bash "$CONVERT" ${flags[@]+"${flags[@]}"} } @test "the dry run prints the plan and then the convert for every arm" { drv_fixture "$BATS_TEST_TMPDIR/drv" run_drv --dry-run [ "$status" -eq 0 ] [[ "$output" == *"== arm cd3 =="* ]] [[ "$output" == *"== arm cd6 =="* ]] [[ "$output" == *"== arm ao1 =="* ]] # one --plan-only bill per arm, and one convert line per arm beside it [ "$(grep -c -- '--plan-only' <<< "$output")" -eq 3 ] [ "$(grep -c 'rad-convert' <<< "$output")" -eq 6 ] awk '/arm cd3/ {next} /qwen38-w5nl64-cd3\.rad/ { if ($0 ~ /--plan-only/) p = NR; else c = NR } END { exit !(p && c && p < c) }' <<< "$output" } @test "the cd6 arm re-sends the whole expert option set with cd=6; the control sends none" { # --quant replaces the recipe line wholesale, so an override that forgot an # option would quietly convert with the default for it. drv_fixture "$BATS_TEST_TMPDIR/drv" run_drv --arm cd3 --arm cd6 --dry-run [ "$status" -eq 0 ] cd6line=$(grep -- '-cd6\.rad --quant' <<< "$output" | head -1) [[ "$cd6line" == *"table=w5nl"* ]] [[ "$cd6line" == *"group=64"* ]] [[ "$cd6line" == *"scale=fp8_e4m3"* ]] [[ "$cd6line" == *"transform=fwht128"* ]] [[ "$cd6line" == *"calib=\$CALIB"* ]] [[ "$cd6line" == *",cd=6"* ]] cd3line=$(grep -- '-cd3\.rad' <<< "$output" | grep -v -- '--plan-only' | head -1) [[ "$cd3line" != *"--quant"* ]] } @test "the ao1 arm carries act_order=1 and no cd=, because libquant refuses the pair" { drv_fixture "$BATS_TEST_TMPDIR/drv" run_drv --arm ao1 --dry-run [ "$status" -eq 0 ] line=$(grep -- '-ao1\.rad --quant' <<< "$output" | head -1) [[ "$line" == *"act_order=1"* ]] [[ "$line" != *",cd="* ]] } @test "the driver splices the protected plane into its own copy, never the template" { drv_fixture "$BATS_TEST_TMPDIR/drv" run_drv --dry-run [[ "$output" == *"apply-protected-cast.sh $BATS_TEST_TMPDIR/drv/calib "* ]] [[ "$output" == *"/campaign/recipe-"* ]] [[ "$output" != *"$TEMPLATE"* ]] } @test "the driver refuses while the BF16 reference container still holds the volume" { drv_fixture "$BATS_TEST_TMPDIR/drv" : > "$BATS_TEST_TMPDIR/drv/out/qwen38-bf16.rad" run_drv --dry-run [ "$status" -eq 2 ] [[ "$output" == *"peak-disk"* ]] } @test "a thin recording is refused: the driver never converts uncalibrated" { # rad-convert converts WITHOUT a calibration recording with one stderr note, # which is silently worse than a GPTQ run that failed. drv_fixture "$BATS_TEST_TMPDIR/drv" run_drv MIN_GRAMS=9 --dry-run [ "$status" -eq 2 ] [[ "$output" == *"run-qwen38-bf16.sh calib"* ]] } @test "an imatrix outside the mounted campaign dir is refused before any convert" { drv_fixture "$BATS_TEST_TMPDIR/drv" mv "$BATS_TEST_TMPDIR/drv/campaign/imatrix.gguf" "$BATS_TEST_TMPDIR/imatrix.gguf" run_drv IMATRIX="$BATS_TEST_TMPDIR/imatrix.gguf" --dry-run [ "$status" -eq 2 ] [[ "$output" == *"not in the campaign dir"* ]] } @test "the driver never overwrites an arm that already exists" { drv_fixture "$BATS_TEST_TMPDIR/drv" : > "$BATS_TEST_TMPDIR/drv/out/qwen38-w5nl64-cd6.rad" run_drv --arm cd6 --dry-run [ "$status" -eq 2 ] [[ "$output" == *"move it to trash first"* ]] } @test "an unknown arm name is refused before the gates run" { # No fixture built: the name check must not need a checkpoint to exist. run_drv --arm cd9 --dry-run [ "$status" -eq 2 ] [[ "$output" == *"unknown arm: cd9"* ]] } @test "the driver stops when the protected derivation cannot land the target" { drv_fixture "$BATS_TEST_TMPDIR/drv" run_drv PY=python3 TARGET_COUNT=20 --arm cd3 [ "$status" -eq 1 ] [[ "$output" == *"target was 20"* ]] [[ "$output" == *"would convert unprotected"* ]] [[ "$output" != *"--entrypoint"* ]] } # --- the acceptance gate ----------------------------------------------------- # kld_report — one score report, in the # shape `run-qwen38-bf16.sh score` writes: an `all` summary plus a per-source one. kld_report() { python3 - "$@" <<'PY' import json, sys path, model, kl, top1 = sys.argv[1:5] kl, top1 = float(kl), float(top1) group = {"kld": {"mean": kl, "p99_9": kl * 4}, "top1_agreement": top1, "positions": 50432} json.dump({"model": model, "all": group, "by_source": {"ops": dict(group, positions=20000)}}, open(path, "w")) PY } gate_case() { # gate_case [extra flags...] D="$BATS_TEST_TMPDIR/kl" mkdir -p "$D" kld_report "$D/fp8.json" fp8-native "$1" 0.981 kld_report "$D/iq4r.json" iq4r-moe "$2" 0.962 kld_report "$D/cand.json" w5nl64 "$3" "$4" run bash "$GATE" "$D/fp8.json" "$D/iq4r.json" "$D/cand.json" "${@:5}" } @test "an arm strictly inside the interval that closes half the gap passes" { gate_case 0.0021 0.0190 0.0090 0.971 [ "$status" -eq 0 ] [[ "$output" == *"closes 59.2%"* ]] [[ "$output" == *"KLD GATE PASS"* ]] } @test "an arm worse than the served payload fails the interval and the recovery" { gate_case 0.0021 0.0190 0.0210 0.971 [ "$status" -eq 1 ] [[ "$output" == *"[FAIL] 2."* ]] [[ "$output" == *"[FAIL] 3."* ]] [[ "$output" == *"KLD GATE FAIL"* ]] } @test "anchors that are not ordered stop the gate: the instrument is wrong, not the arm" { # If the near-lossless FP8 release does not sit CLOSER to the bf16 reference # than the 4-bit payload does, the corpus, tokenizer or boot shape is broken and # no arm number can mean anything. gate_case 0.0300 0.0190 0.0120 0.971 [ "$status" -eq 1 ] [[ "$output" == *"[FAIL] 1."* ]] [[ "$output" == *"gap is not positive"* ]] } @test "an arm measured outside the interval fails even when it looks better" { # Closer to the reference than the achievable anchor is a measurement fault. gate_case 0.0021 0.0190 0.0010 0.971 [ "$status" -eq 1 ] [[ "$output" == *"[FAIL] 2."* ]] } @test "an arm that buys KL with top-1 agreement fails" { gate_case 0.0021 0.0190 0.0090 0.940 [ "$status" -eq 1 ] [[ "$output" == *"[PASS] 3."* ]] [[ "$output" == *"[FAIL] 4."* ]] } @test "--recover tightens the same numbers until they fail" { gate_case 0.0021 0.0190 0.0090 0.971 --recover 0.8 [ "$status" -eq 1 ] [[ "$output" == *"needs 80%"* ]] [[ "$output" == *"[FAIL] 3."* ]] } @test "a slice the report does not carry is named, not guessed at" { gate_case 0.0021 0.0190 0.0090 0.971 --slice code [ "$status" -eq 2 ] [[ "$output" == *"has no slice 'code'"* ]] [[ "$output" == *"it has: ops"* ]] }