test(qwen38): campaign seam for fetch, protected plane, arms and gate
32 black-box tests over the four tools the campaign runs, driven through their own argv with fixture corpora - no GPU, no container, no rad-convert. Each test is an output an operator reads: the verifier's MISSING line and exit, the byte bill the derivation lands, the refusal text, the plan and the convert the driver prints per arm, the interval verdict the gate returns. Includes the planner seam the swift15 campaign lacked: the campaign's paths plan NO deployable service, so merging this toolchain can never drain frank-archon. The driver fixture carries four grams so --min-grams has a thin-pool control, and the arm-driver invocation passes env assignments positionally after the driver's own flags, so the seam exercises the real call shape.
This commit is contained in:
Frank committed
1 parent
bb027265db
commit
5fe04e569f
1 file changed
+527
Executable
+527
@@ -0,0 +1,527 @@
|
|||||||
|
#!/usr/bin/env bats
|
||||||
|
# Seam: the Qwen3.8-Flash-Next five-bit campaign (feat/qwen38-w5nl64).
|
||||||
|
#
|
||||||
|
# The campaign converts the vendor BF16 checkpoint into a w5nl64a8h .rad on
|
||||||
|
# dankserver while frank-archon keeps serving. Every gate that protects the box
|
||||||
|
# or the result lives in a script under scripts/, and each gate is tested here
|
||||||
|
# through that script's own exit status and printed output — no engine, no GPU,
|
||||||
|
# no network. Fixtures are synthetic; the expected numbers are worked examples,
|
||||||
|
# not values recomputed the way the script computes them.
|
||||||
|
#
|
||||||
|
# Seams under test (the campaign's public boundaries):
|
||||||
|
# scripts/download-qwen38-bf16.sh fetch/verify of the BF16 checkpoint
|
||||||
|
# scripts/grams-to-imatrix.py calibration recording -> imatrix GGUF
|
||||||
|
# scripts/derive-protected-experts.py --target-count sweep + the parity law
|
||||||
|
# scripts/build-qwen38-bf16-rad.sh the reference container's convert argv
|
||||||
|
# scripts/run-qwen38-bf16.sh the drained-serve argv, mode selection
|
||||||
|
# scripts/convert-qwen38-w5nl64.sh the arm driver's refusal gates
|
||||||
|
# scripts/kld-gate-qwen38.sh the three-anchor interval law
|
||||||
|
# scripts/deploy-plan.py campaign files must deploy nothing
|
||||||
|
|
||||||
|
load helpers
|
||||||
|
|
||||||
|
DOWNLOAD="$REPO_ROOT/scripts/download-qwen38-bf16.sh"
|
||||||
|
PLANNER="$REPO_ROOT/scripts/deploy-plan.py"
|
||||||
|
|
||||||
|
# make_shard <dir> <name> <total-bytes> — a REAL safetensors file whose size on
|
||||||
|
# disk is exactly <total-bytes>: 8-byte header length, JSON header, tensor data.
|
||||||
|
# Real, because the verifier reads each shard's own header to learn how many
|
||||||
|
# bytes it claims to be, which is the only per-file truth available offline.
|
||||||
|
make_shard() {
|
||||||
|
mkdir -p "$1"
|
||||||
|
python3 - "$1/$2" "$3" <<'PY'
|
||||||
|
import json, struct, sys
|
||||||
|
path, want = sys.argv[1], int(sys.argv[2])
|
||||||
|
payload = max(0, want - 96)
|
||||||
|
header = b""
|
||||||
|
for _ in range(16):
|
||||||
|
# The header is part of the file, so its own length moves the target: solve
|
||||||
|
# payload = want - 8 - len(header(payload)) by fixed-point iteration.
|
||||||
|
header = json.dumps({"w": {"dtype": "F32", "shape": [payload // 4],
|
||||||
|
"data_offsets": [0, payload]}},
|
||||||
|
separators=(",", ":")).encode()
|
||||||
|
want_payload = want - 8 - len(header)
|
||||||
|
if want_payload == payload:
|
||||||
|
break
|
||||||
|
if want_payload < 0:
|
||||||
|
raise SystemExit("fixture too small to carry a header")
|
||||||
|
payload = want_payload
|
||||||
|
else:
|
||||||
|
raise SystemExit("fixture header never converged")
|
||||||
|
with open(path, "wb") as f:
|
||||||
|
f.write(struct.pack("<Q", len(header)) + header + b"\x00" * payload)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
# make_index <dir> <shard>... — a safetensors index that references them
|
||||||
|
make_index() {
|
||||||
|
local dir="$1"; shift
|
||||||
|
python3 - "$dir/model.safetensors.index.json" "$@" <<'PY'
|
||||||
|
import json, sys
|
||||||
|
out, names = sys.argv[1], sys.argv[2:]
|
||||||
|
json.dump({"metadata": {"total_size": 0},
|
||||||
|
"weight_map": {f"model.layers.{i}.attn.q.weight": n
|
||||||
|
for i, n in enumerate(names)}},
|
||||||
|
open(out, "w"))
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
# three_fixture <dir> — three valid 1000-byte shards plus the index naming them
|
||||||
|
three_fixture() {
|
||||||
|
make_shard "$1" model-00001-of-00003.safetensors 1000
|
||||||
|
make_shard "$1" model-00002-of-00003.safetensors 1000
|
||||||
|
make_shard "$1" model-00003-of-00003.safetensors 1000
|
||||||
|
make_index "$1" model-00001-of-00003.safetensors \
|
||||||
|
model-00002-of-00003.safetensors \
|
||||||
|
model-00003-of-00003.safetensors
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- download / verify -------------------------------------------------------
|
||||||
|
|
||||||
|
@test "the checkpoint verifier accepts a complete checkpoint and reports its footprint" {
|
||||||
|
C="$BATS_TEST_TMPDIR/complete"
|
||||||
|
three_fixture "$C"
|
||||||
|
# 3 x 1000 bytes of shards. The index file itself is deliberately not
|
||||||
|
# counted, so a Hub-side edit to a config file cannot fail this gate.
|
||||||
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"VERIFY_OK"*"3 shards"*"3000 bytes"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the verifier names the shard whose own header disagrees with its size on disk" {
|
||||||
|
C="$BATS_TEST_TMPDIR/truncated"
|
||||||
|
three_fixture "$C"
|
||||||
|
# Cut the tail, keep the header: that is a torn transfer, which is what the
|
||||||
|
# box actually gets from an interrupted fetch. Zero-filling instead would
|
||||||
|
# destroy the header and be caught as "not a safetensors file" — a different
|
||||||
|
# gate, and it would prove nothing about the size check.
|
||||||
|
python3 -c 'import sys
|
||||||
|
p = sys.argv[1]
|
||||||
|
keep = open(p, "rb").read(400)
|
||||||
|
open(p, "wb").write(keep)' "$C/model-00002-of-00003.safetensors"
|
||||||
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"TRUNCATED"*"model-00002-of-00003.safetensors"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the verifier refuses a missing shard and names it" {
|
||||||
|
C="$BATS_TEST_TMPDIR/missing"
|
||||||
|
three_fixture "$C"
|
||||||
|
mv "$C/model-00003-of-00003.safetensors" "$C/hidden.bin"
|
||||||
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"MISSING"*"model-00003-of-00003.safetensors"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the verifier refuses when the shard census disagrees with the expected count" {
|
||||||
|
# The fragment law this repo learned the hard way: large quants ship as
|
||||||
|
# .partNofM fragments and a naive glob reports success on a partial tree.
|
||||||
|
C="$BATS_TEST_TMPDIR/count"
|
||||||
|
three_fixture "$C"
|
||||||
|
make_shard "$C" model-00004-of-00004.safetensors 1000
|
||||||
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"SHARD_COUNT"*"4"*"3"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the verifier refuses a shard the index names that never landed" {
|
||||||
|
C="$BATS_TEST_TMPDIR/orphan-index"
|
||||||
|
three_fixture "$C"
|
||||||
|
rm "$C/model-00003-of-00003.safetensors"
|
||||||
|
make_index "$C" model-00001-of-00003.safetensors \
|
||||||
|
model-00002-of-00003.safetensors \
|
||||||
|
model-00003-of-00003.safetensors
|
||||||
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 2 --bytes 2000
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"MISSING"*"model-00003-of-00003.safetensors"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the verifier treats an HTTP error body written as a shard as a failure, not a download" {
|
||||||
|
# curl -o FILE exits 0 on 401/403 and writes the error body; three shards
|
||||||
|
# of identical tiny size must never look like a checkpoint.
|
||||||
|
C="$BATS_TEST_TMPDIR/errorbody"
|
||||||
|
mkdir -p "$C"
|
||||||
|
for n in 1 2 3; do
|
||||||
|
printf '{"error":"You are not logged in"}' \
|
||||||
|
> "$C/model-0000${n}-of-00003.safetensors"
|
||||||
|
done
|
||||||
|
make_index "$C" model-00001-of-00003.safetensors \
|
||||||
|
model-00002-of-00003.safetensors \
|
||||||
|
model-00003-of-00003.safetensors
|
||||||
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"NOT_SAFETENSORS"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the verifier refuses a shard census whose bytes do not total what the Hub declared" {
|
||||||
|
# Every shard header can be self-consistent and the tree still be the wrong
|
||||||
|
# payload: the total is the cross-check against the Hub's declared size.
|
||||||
|
C="$BATS_TEST_TMPDIR/wrongtotal"
|
||||||
|
three_fixture "$C"
|
||||||
|
run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 2500
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"TOTAL_BYTES"*"3000"*"2500"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the fetch path refuses when the destination cannot hold the checkpoint" {
|
||||||
|
# The campaign's peak-disk law starts at the fetch: the BF16 checkpoint and
|
||||||
|
# the reference container built from it are both ~335 GiB, and the tree that
|
||||||
|
# does not fit is discovered hours into a transfer.
|
||||||
|
C="$BATS_TEST_TMPDIR/tiny"
|
||||||
|
three_fixture "$C"
|
||||||
|
run env FRANK_QWEN38_MIN_FREE_MIB=999999999 \
|
||||||
|
bash "$DOWNLOAD" --fetch --no-hub --dir "$C" --shards 3 --bytes 3000
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"REFUSED"*"free"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- deployment scoping ------------------------------------------------------
|
||||||
|
|
||||||
|
@test "the campaign tooling deploys nothing: merging it cannot drain frank-archon" {
|
||||||
|
# Existence is asserted first: a planner that plans nothing because the
|
||||||
|
# files vanished would otherwise pass.
|
||||||
|
[ -f "$REPO_ROOT/scripts/download-qwen38-bf16.sh" ]
|
||||||
|
run python3 "$PLANNER" --target dankserver --paths \
|
||||||
|
scripts/download-qwen38-bf16.sh \
|
||||||
|
scripts/build-qwen38-bf16-rad.sh \
|
||||||
|
scripts/run-qwen38-bf16.sh \
|
||||||
|
scripts/convert-qwen38-w5nl64.sh \
|
||||||
|
scripts/kld-gate-qwen38.sh \
|
||||||
|
scripts/grams-to-imatrix.py \
|
||||||
|
scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe \
|
||||||
|
tests/qwen38_campaign.bats
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ -z "$output" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- protected experts -------------------------------------------------------
|
||||||
|
|
||||||
|
DERIVE="$REPO_ROOT/scripts/derive-protected-experts.py"
|
||||||
|
APPLY="$REPO_ROOT/scripts/apply-protected-cast.sh"
|
||||||
|
CONVERT="$REPO_ROOT/scripts/convert-qwen38-w5nl64.sh"
|
||||||
|
GATE="$REPO_ROOT/scripts/kld-gate-qwen38.sh"
|
||||||
|
TEMPLATE="$REPO_ROOT/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe"
|
||||||
|
|
||||||
|
# make_gram <dir> <layer> <expert> <energy> <rows> — a RADGRAM2 down-Gram whose
|
||||||
|
# trace (its down-input energy) is <energy> and whose routing count is <rows>.
|
||||||
|
make_gram() {
|
||||||
|
python3 - "$@" <<'PY'
|
||||||
|
import os, struct, sys
|
||||||
|
d, layer, e, energy, rows = sys.argv[1:6]
|
||||||
|
os.makedirs(d, exist_ok=True)
|
||||||
|
hdr = struct.pack("<8sIIQI4x", b"RADGRAM2", 2, 0, int(rows), 0)
|
||||||
|
mat = struct.pack("<4f", float(energy), 0.0, 0.0, 0.0)
|
||||||
|
with open(os.path.join(d, f"gram.r0.blk.{layer}.ffn_down_exps.{e}.bin"), "wb") as f:
|
||||||
|
f.write(hdr + mat)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
# eight_grams <dir> — one layer, eight experts inside rank half 0, energies
|
||||||
|
# 60/15/10/8/4/2/1/0.5 against equal routing. Under the default bars (3% of the
|
||||||
|
# layer's energy AND 3x its routing share) exactly ONE expert is disproportionate,
|
||||||
|
# so the parity law has to hand it a partner.
|
||||||
|
eight_grams() {
|
||||||
|
local d="$1"
|
||||||
|
make_gram "$d" 0 0 60 100
|
||||||
|
make_gram "$d" 0 1 15 100
|
||||||
|
make_gram "$d" 0 2 10 100
|
||||||
|
make_gram "$d" 0 3 8 100
|
||||||
|
make_gram "$d" 0 4 4 100
|
||||||
|
make_gram "$d" 0 5 2 100
|
||||||
|
make_gram "$d" 0 6 1 100
|
||||||
|
make_gram "$d" 0 7 0.5 100
|
||||||
|
}
|
||||||
|
|
||||||
|
cast_count() { grep -c 'cast dtype=bf16' <<< "$output"; }
|
||||||
|
|
||||||
|
@test "the bar-only pass keeps the disproportionate expert and repairs the odd rank half" {
|
||||||
|
C="$BATS_TEST_TMPDIR/calib"
|
||||||
|
eight_grams "$C"
|
||||||
|
run python3 "$DERIVE" "$C"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
# e0 holds 59.7% of the layer's energy on 12.5% of the routing: the only expert
|
||||||
|
# that clears three times its routing share. e1 joins it because half 0 was odd.
|
||||||
|
[ "$(cast_count)" -eq 2 ]
|
||||||
|
[[ "$output" == *"blk.0.ffn_*_exps.0.weight"* ]]
|
||||||
|
[[ "$output" == *"blk.0.ffn_*_exps.1.weight"* ]]
|
||||||
|
[[ "$output" != *"ffn_*_exps.2.weight"* ]]
|
||||||
|
[[ "$output" == *"half 0 odd -> added e1"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "--target-count lands the byte bill that was asked for, and never buys more" {
|
||||||
|
# The count is a disk and VRAM decision (each protected expert costs bf16 twice,
|
||||||
|
# once per rank half's copy), so landing it matters as much as not missing it.
|
||||||
|
C="$BATS_TEST_TMPDIR/calib"
|
||||||
|
eight_grams "$C"
|
||||||
|
run python3 "$DERIVE" "$C" --target-count 6
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ "$(cast_count)" -eq 6 ]
|
||||||
|
[[ "$output" == *"count fill -> added e2"* ]]
|
||||||
|
[[ "$output" != *"ffn_*_exps.6.weight"* ]]
|
||||||
|
# the energy share the set bottoms out at is the bar the campaign paid for
|
||||||
|
[[ "$output" == *"total protected: 6 (target 6, energy cutoff"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "--target-count cannot invent experts: the pool's ceiling is reported, not padded" {
|
||||||
|
C="$BATS_TEST_TMPDIR/calib"
|
||||||
|
eight_grams "$C"
|
||||||
|
run python3 "$DERIVE" "$C" --target-count 20
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ "$(cast_count)" -eq 8 ]
|
||||||
|
[[ "$output" == *"total protected: 8 (target 20"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a protected set that cannot be made even is refused instead of emitted" {
|
||||||
|
# One disproportionate expert in half 0, its only companion in half 1: no
|
||||||
|
# candidate could make half 0 even, and an odd half hands the four-bit GEMM two
|
||||||
|
# tables that are not equal. Refusing is the only honest answer.
|
||||||
|
C="$BATS_TEST_TMPDIR/calib"
|
||||||
|
make_gram "$C" 0 0 100 100
|
||||||
|
make_gram "$C" 0 300 1 1000
|
||||||
|
run python3 "$DERIVE" "$C"
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"PARITY LAW BROKEN"* ]]
|
||||||
|
[[ "$output" == *"blk.0 half 0 holds 1"* ]]
|
||||||
|
[ "$(cast_count)" -eq 0 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the splice rewrites the run's copy above the expert rule and refuses a second run" {
|
||||||
|
C="$BATS_TEST_TMPDIR/calib"
|
||||||
|
eight_grams "$C"
|
||||||
|
W="$BATS_TEST_TMPDIR/recipe-work.recipe"
|
||||||
|
cp "$TEMPLATE" "$W"
|
||||||
|
run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 0
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
grep -q "derive-protected-experts.py rewrote" "$W"
|
||||||
|
# The derived lines must precede the expert rule: the first rule whose glob
|
||||||
|
# matches a weight decides it, so a cast line below the expert line never fires.
|
||||||
|
cast_line=$(grep -n 'cast dtype=bf16' "$W" | head -1 | cut -d: -f1)
|
||||||
|
expert_line=$(grep -n 'ffn_\*_exps\.\*\.weight' "$W" | head -1 | cut -d: -f1)
|
||||||
|
[ "$cast_line" -lt "$expert_line" ]
|
||||||
|
# The committed template stays the clean, reviewable baseline.
|
||||||
|
! grep -q "derive-protected-experts.py rewrote" "$TEMPLATE"
|
||||||
|
# A second run on the same copy would derive from a recording it already spent.
|
||||||
|
run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 0
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"refusing to re-derive"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the splice refuses before touching the recipe when the count is not reached" {
|
||||||
|
C="$BATS_TEST_TMPDIR/calib"
|
||||||
|
eight_grams "$C"
|
||||||
|
W="$BATS_TEST_TMPDIR/recipe-work.recipe"
|
||||||
|
cp "$TEMPLATE" "$W"
|
||||||
|
run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 20
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"target was 20"* ]]
|
||||||
|
! grep -q "derive-protected-experts.py rewrote" "$W"
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- the arm driver ----------------------------------------------------------
|
||||||
|
|
||||||
|
# drv_fixture <dir> — the smallest tree the driver's gates accept: a checkpoint
|
||||||
|
# with its index and tokenizer, three grams, an imatrix inside the campaign dir
|
||||||
|
# the container mounts, and an empty output dir.
|
||||||
|
drv_fixture() {
|
||||||
|
local d="$1"
|
||||||
|
mkdir -p "$d/ck/Qwen3.8-Flash-Next" "$d/campaign" "$d/calib" "$d/out"
|
||||||
|
: > "$d/ck/Qwen3.8-Flash-Next/model.safetensors.index.json"
|
||||||
|
: > "$d/ck/Qwen3.8-Flash-Next/tokenizer.json"
|
||||||
|
make_gram "$d/calib" 0 0 60 100
|
||||||
|
make_gram "$d/calib" 0 1 15 100
|
||||||
|
make_gram "$d/calib" 0 2 10 100
|
||||||
|
# four experts, not three: a thin pool that cannot be made even trips the parity
|
||||||
|
# audit, and this fixture is meant to reach the later gates
|
||||||
|
make_gram "$d/calib" 0 3 8 100
|
||||||
|
: > "$d/campaign/imatrix.gguf"
|
||||||
|
}
|
||||||
|
|
||||||
|
# run_drv <driver args and K=V overrides, any order> — the driver with every path
|
||||||
|
# pointed at the fixture. Override words are separated out because `env` only
|
||||||
|
# accepts assignments ahead of the command; later assignments win, so a test can
|
||||||
|
# break exactly one input.
|
||||||
|
run_drv() {
|
||||||
|
local d="$BATS_TEST_TMPDIR/drv" assigns=() flags=() w
|
||||||
|
for w in "$@"; do
|
||||||
|
if [[ "$w" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]]; then assigns+=("$w"); else flags+=("$w"); fi
|
||||||
|
done
|
||||||
|
run env CKPT="$d/ck/Qwen3.8-Flash-Next" CAMPAIGN="$d/campaign" STATE="$d/campaign" \
|
||||||
|
CALIB="$d/calib" OUTDIR="$d/out" IMATRIX="$d/campaign/imatrix.gguf" \
|
||||||
|
MIN_GRAMS=2 TARGET_COUNT=0 ${assigns[@]+"${assigns[@]}"} \
|
||||||
|
bash "$CONVERT" ${flags[@]+"${flags[@]}"}
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the dry run prints the plan and then the convert for every arm" {
|
||||||
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
||||||
|
run_drv --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"== arm cd3 =="* ]]
|
||||||
|
[[ "$output" == *"== arm cd6 =="* ]]
|
||||||
|
[[ "$output" == *"== arm ao1 =="* ]]
|
||||||
|
# one --plan-only bill per arm, and one convert line per arm beside it
|
||||||
|
[ "$(grep -c -- '--plan-only' <<< "$output")" -eq 3 ]
|
||||||
|
[ "$(grep -c 'rad-convert' <<< "$output")" -eq 6 ]
|
||||||
|
awk '/arm cd3/ {next}
|
||||||
|
/qwen38-w5nl64-cd3\.rad/ { if ($0 ~ /--plan-only/) p = NR; else c = NR }
|
||||||
|
END { exit !(p && c && p < c) }' <<< "$output"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the cd6 arm re-sends the whole expert option set with cd=6; the control sends none" {
|
||||||
|
# --quant replaces the recipe line wholesale, so an override that forgot an
|
||||||
|
# option would quietly convert with the default for it.
|
||||||
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
||||||
|
run_drv --arm cd3 --arm cd6 --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
cd6line=$(grep -- '-cd6\.rad --quant' <<< "$output" | head -1)
|
||||||
|
[[ "$cd6line" == *"table=w5nl"* ]]
|
||||||
|
[[ "$cd6line" == *"group=64"* ]]
|
||||||
|
[[ "$cd6line" == *"scale=fp8_e4m3"* ]]
|
||||||
|
[[ "$cd6line" == *"transform=fwht128"* ]]
|
||||||
|
[[ "$cd6line" == *"calib=\$CALIB"* ]]
|
||||||
|
[[ "$cd6line" == *",cd=6"* ]]
|
||||||
|
cd3line=$(grep -- '-cd3\.rad' <<< "$output" | grep -v -- '--plan-only' | head -1)
|
||||||
|
[[ "$cd3line" != *"--quant"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the ao1 arm carries act_order=1 and no cd=, because libquant refuses the pair" {
|
||||||
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
||||||
|
run_drv --arm ao1 --dry-run
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
line=$(grep -- '-ao1\.rad --quant' <<< "$output" | head -1)
|
||||||
|
[[ "$line" == *"act_order=1"* ]]
|
||||||
|
[[ "$line" != *",cd="* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the driver splices the protected plane into its own copy, never the template" {
|
||||||
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
||||||
|
run_drv --dry-run
|
||||||
|
[[ "$output" == *"apply-protected-cast.sh $BATS_TEST_TMPDIR/drv/calib "* ]]
|
||||||
|
[[ "$output" == *"/campaign/recipe-"* ]]
|
||||||
|
[[ "$output" != *"$TEMPLATE"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the driver refuses while the BF16 reference container still holds the volume" {
|
||||||
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
||||||
|
: > "$BATS_TEST_TMPDIR/drv/out/qwen38-bf16.rad"
|
||||||
|
run_drv --dry-run
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"peak-disk"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a thin recording is refused: the driver never converts uncalibrated" {
|
||||||
|
# rad-convert converts WITHOUT a calibration recording with one stderr note,
|
||||||
|
# which is silently worse than a GPTQ run that failed.
|
||||||
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
||||||
|
run_drv MIN_GRAMS=9 --dry-run
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"run-qwen38-bf16.sh calib"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "an imatrix outside the mounted campaign dir is refused before any convert" {
|
||||||
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
||||||
|
mv "$BATS_TEST_TMPDIR/drv/campaign/imatrix.gguf" "$BATS_TEST_TMPDIR/imatrix.gguf"
|
||||||
|
run_drv IMATRIX="$BATS_TEST_TMPDIR/imatrix.gguf" --dry-run
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"not in the campaign dir"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the driver never overwrites an arm that already exists" {
|
||||||
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
||||||
|
: > "$BATS_TEST_TMPDIR/drv/out/qwen38-w5nl64-cd6.rad"
|
||||||
|
run_drv --arm cd6 --dry-run
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"move it to trash first"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "an unknown arm name is refused before the gates run" {
|
||||||
|
# No fixture built: the name check must not need a checkpoint to exist.
|
||||||
|
run_drv --arm cd9 --dry-run
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"unknown arm: cd9"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the driver stops when the protected derivation cannot land the target" {
|
||||||
|
drv_fixture "$BATS_TEST_TMPDIR/drv"
|
||||||
|
run_drv PY=python3 TARGET_COUNT=20 --arm cd3
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"target was 20"* ]]
|
||||||
|
[[ "$output" == *"would convert unprotected"* ]]
|
||||||
|
[[ "$output" != *"--entrypoint"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- the acceptance gate -----------------------------------------------------
|
||||||
|
|
||||||
|
# kld_report <path> <model> <kl mean> <top1 fraction> — one score report, in the
|
||||||
|
# shape `run-qwen38-bf16.sh score` writes: an `all` summary plus a per-source one.
|
||||||
|
kld_report() {
|
||||||
|
python3 - "$@" <<'PY'
|
||||||
|
import json, sys
|
||||||
|
path, model, kl, top1 = sys.argv[1:5]
|
||||||
|
kl, top1 = float(kl), float(top1)
|
||||||
|
group = {"kld": {"mean": kl, "p99_9": kl * 4}, "top1_agreement": top1, "positions": 50432}
|
||||||
|
json.dump({"model": model, "all": group,
|
||||||
|
"by_source": {"ops": dict(group, positions=20000)}}, open(path, "w"))
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
gate_case() { # gate_case <kl fp8> <kl iq4r> <kl cand> <top1 cand> [extra flags...]
|
||||||
|
D="$BATS_TEST_TMPDIR/kl"
|
||||||
|
mkdir -p "$D"
|
||||||
|
kld_report "$D/fp8.json" fp8-native "$1" 0.981
|
||||||
|
kld_report "$D/iq4r.json" iq4r-moe "$2" 0.962
|
||||||
|
kld_report "$D/cand.json" w5nl64 "$3" "$4"
|
||||||
|
run bash "$GATE" "$D/fp8.json" "$D/iq4r.json" "$D/cand.json" "${@:5}"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "an arm strictly inside the interval that closes half the gap passes" {
|
||||||
|
gate_case 0.0021 0.0190 0.0090 0.971
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"closes 59.2%"* ]]
|
||||||
|
[[ "$output" == *"KLD GATE PASS"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "an arm worse than the served payload fails the interval and the recovery" {
|
||||||
|
gate_case 0.0021 0.0190 0.0210 0.971
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"[FAIL] 2."* ]]
|
||||||
|
[[ "$output" == *"[FAIL] 3."* ]]
|
||||||
|
[[ "$output" == *"KLD GATE FAIL"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "anchors that are not ordered stop the gate: the instrument is wrong, not the arm" {
|
||||||
|
# If the near-lossless FP8 release does not sit CLOSER to the bf16 reference
|
||||||
|
# than the 4-bit payload does, the corpus, tokenizer or boot shape is broken and
|
||||||
|
# no arm number can mean anything.
|
||||||
|
gate_case 0.0300 0.0190 0.0120 0.971
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"[FAIL] 1."* ]]
|
||||||
|
[[ "$output" == *"gap is not positive"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "an arm measured outside the interval fails even when it looks better" {
|
||||||
|
# Closer to the reference than the achievable anchor is a measurement fault.
|
||||||
|
gate_case 0.0021 0.0190 0.0010 0.971
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"[FAIL] 2."* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "an arm that buys KL with top-1 agreement fails" {
|
||||||
|
gate_case 0.0021 0.0190 0.0090 0.940
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"[PASS] 3."* ]]
|
||||||
|
[[ "$output" == *"[FAIL] 4."* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "--recover tightens the same numbers until they fail" {
|
||||||
|
gate_case 0.0021 0.0190 0.0090 0.971 --recover 0.8
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[[ "$output" == *"needs 80%"* ]]
|
||||||
|
[[ "$output" == *"[FAIL] 3."* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a slice the report does not carry is named, not guessed at" {
|
||||||
|
gate_case 0.0021 0.0190 0.0090 0.971 --slice code
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"has no slice 'code'"* ]]
|
||||||
|
[[ "$output" == *"it has: ops"* ]]
|
||||||
|
}
|
||||||
Reference in new issue
Block a user