diff --git a/tests/qwen38_campaign.bats b/tests/qwen38_campaign.bats new file mode 100755 index 0000000..f061a8d --- /dev/null +++ b/tests/qwen38_campaign.bats @@ -0,0 +1,527 @@ +#!/usr/bin/env bats +# Seam: the Qwen3.8-Flash-Next five-bit campaign (feat/qwen38-w5nl64). +# +# The campaign converts the vendor BF16 checkpoint into a w5nl64a8h .rad on +# dankserver while frank-archon keeps serving. Every gate that protects the box +# or the result lives in a script under scripts/, and each gate is tested here +# through that script's own exit status and printed output — no engine, no GPU, +# no network. Fixtures are synthetic; the expected numbers are worked examples, +# not values recomputed the way the script computes them. +# +# Seams under test (the campaign's public boundaries): +# scripts/download-qwen38-bf16.sh fetch/verify of the BF16 checkpoint +# scripts/grams-to-imatrix.py calibration recording -> imatrix GGUF +# scripts/derive-protected-experts.py --target-count sweep + the parity law +# scripts/build-qwen38-bf16-rad.sh the reference container's convert argv +# scripts/run-qwen38-bf16.sh the drained-serve argv, mode selection +# scripts/convert-qwen38-w5nl64.sh the arm driver's refusal gates +# scripts/kld-gate-qwen38.sh the three-anchor interval law +# scripts/deploy-plan.py campaign files must deploy nothing + +load helpers + +DOWNLOAD="$REPO_ROOT/scripts/download-qwen38-bf16.sh" +PLANNER="$REPO_ROOT/scripts/deploy-plan.py" + +# make_shard — a REAL safetensors file whose size on +# disk is exactly : 8-byte header length, JSON header, tensor data. +# Real, because the verifier reads each shard's own header to learn how many +# bytes it claims to be, which is the only per-file truth available offline. +make_shard() { + mkdir -p "$1" + python3 - "$1/$2" "$3" <<'PY' +import json, struct, sys +path, want = sys.argv[1], int(sys.argv[2]) +payload = max(0, want - 96) +header = b"" +for _ in range(16): + # The header is part of the file, so its own length moves the target: solve + # payload = want - 8 - len(header(payload)) by fixed-point iteration. + header = json.dumps({"w": {"dtype": "F32", "shape": [payload // 4], + "data_offsets": [0, payload]}}, + separators=(",", ":")).encode() + want_payload = want - 8 - len(header) + if want_payload == payload: + break + if want_payload < 0: + raise SystemExit("fixture too small to carry a header") + payload = want_payload +else: + raise SystemExit("fixture header never converged") +with open(path, "wb") as f: + f.write(struct.pack(" ... — a safetensors index that references them +make_index() { + local dir="$1"; shift + python3 - "$dir/model.safetensors.index.json" "$@" <<'PY' +import json, sys +out, names = sys.argv[1], sys.argv[2:] +json.dump({"metadata": {"total_size": 0}, + "weight_map": {f"model.layers.{i}.attn.q.weight": n + for i, n in enumerate(names)}}, + open(out, "w")) +PY +} + +# three_fixture — three valid 1000-byte shards plus the index naming them +three_fixture() { + make_shard "$1" model-00001-of-00003.safetensors 1000 + make_shard "$1" model-00002-of-00003.safetensors 1000 + make_shard "$1" model-00003-of-00003.safetensors 1000 + make_index "$1" model-00001-of-00003.safetensors \ + model-00002-of-00003.safetensors \ + model-00003-of-00003.safetensors +} + +# --- download / verify ------------------------------------------------------- + +@test "the checkpoint verifier accepts a complete checkpoint and reports its footprint" { + C="$BATS_TEST_TMPDIR/complete" + three_fixture "$C" + # 3 x 1000 bytes of shards. The index file itself is deliberately not + # counted, so a Hub-side edit to a config file cannot fail this gate. + run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000 + [ "$status" -eq 0 ] + [[ "$output" == *"VERIFY_OK"*"3 shards"*"3000 bytes"* ]] +} + +@test "the verifier names the shard whose own header disagrees with its size on disk" { + C="$BATS_TEST_TMPDIR/truncated" + three_fixture "$C" + # Cut the tail, keep the header: that is a torn transfer, which is what the + # box actually gets from an interrupted fetch. Zero-filling instead would + # destroy the header and be caught as "not a safetensors file" — a different + # gate, and it would prove nothing about the size check. + python3 -c 'import sys +p = sys.argv[1] +keep = open(p, "rb").read(400) +open(p, "wb").write(keep)' "$C/model-00002-of-00003.safetensors" + run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000 + [ "$status" -eq 1 ] + [[ "$output" == *"TRUNCATED"*"model-00002-of-00003.safetensors"* ]] +} + +@test "the verifier refuses a missing shard and names it" { + C="$BATS_TEST_TMPDIR/missing" + three_fixture "$C" + mv "$C/model-00003-of-00003.safetensors" "$C/hidden.bin" + run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000 + [ "$status" -eq 1 ] + [[ "$output" == *"MISSING"*"model-00003-of-00003.safetensors"* ]] +} + +@test "the verifier refuses when the shard census disagrees with the expected count" { + # The fragment law this repo learned the hard way: large quants ship as + # .partNofM fragments and a naive glob reports success on a partial tree. + C="$BATS_TEST_TMPDIR/count" + three_fixture "$C" + make_shard "$C" model-00004-of-00004.safetensors 1000 + run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000 + [ "$status" -eq 1 ] + [[ "$output" == *"SHARD_COUNT"*"4"*"3"* ]] +} + +@test "the verifier refuses a shard the index names that never landed" { + C="$BATS_TEST_TMPDIR/orphan-index" + three_fixture "$C" + rm "$C/model-00003-of-00003.safetensors" + make_index "$C" model-00001-of-00003.safetensors \ + model-00002-of-00003.safetensors \ + model-00003-of-00003.safetensors + run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 2 --bytes 2000 + [ "$status" -eq 1 ] + [[ "$output" == *"MISSING"*"model-00003-of-00003.safetensors"* ]] +} + +@test "the verifier treats an HTTP error body written as a shard as a failure, not a download" { + # curl -o FILE exits 0 on 401/403 and writes the error body; three shards + # of identical tiny size must never look like a checkpoint. + C="$BATS_TEST_TMPDIR/errorbody" + mkdir -p "$C" + for n in 1 2 3; do + printf '{"error":"You are not logged in"}' \ + > "$C/model-0000${n}-of-00003.safetensors" + done + make_index "$C" model-00001-of-00003.safetensors \ + model-00002-of-00003.safetensors \ + model-00003-of-00003.safetensors + run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 3000 + [ "$status" -eq 1 ] + [[ "$output" == *"NOT_SAFETENSORS"* ]] +} + +@test "the verifier refuses a shard census whose bytes do not total what the Hub declared" { + # Every shard header can be self-consistent and the tree still be the wrong + # payload: the total is the cross-check against the Hub's declared size. + C="$BATS_TEST_TMPDIR/wrongtotal" + three_fixture "$C" + run bash "$DOWNLOAD" --no-hub --dir "$C" --shards 3 --bytes 2500 + [ "$status" -eq 1 ] + [[ "$output" == *"TOTAL_BYTES"*"3000"*"2500"* ]] +} + +@test "the fetch path refuses when the destination cannot hold the checkpoint" { + # The campaign's peak-disk law starts at the fetch: the BF16 checkpoint and + # the reference container built from it are both ~335 GiB, and the tree that + # does not fit is discovered hours into a transfer. + C="$BATS_TEST_TMPDIR/tiny" + three_fixture "$C" + run env FRANK_QWEN38_MIN_FREE_MIB=999999999 \ + bash "$DOWNLOAD" --fetch --no-hub --dir "$C" --shards 3 --bytes 3000 + [ "$status" -eq 2 ] + [[ "$output" == *"REFUSED"*"free"* ]] +} + +# --- deployment scoping ------------------------------------------------------ + +@test "the campaign tooling deploys nothing: merging it cannot drain frank-archon" { + # Existence is asserted first: a planner that plans nothing because the + # files vanished would otherwise pass. + [ -f "$REPO_ROOT/scripts/download-qwen38-bf16.sh" ] + run python3 "$PLANNER" --target dankserver --paths \ + scripts/download-qwen38-bf16.sh \ + scripts/build-qwen38-bf16-rad.sh \ + scripts/run-qwen38-bf16.sh \ + scripts/convert-qwen38-w5nl64.sh \ + scripts/kld-gate-qwen38.sh \ + scripts/grams-to-imatrix.py \ + scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe \ + tests/qwen38_campaign.bats + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +# --- protected experts ------------------------------------------------------- + +DERIVE="$REPO_ROOT/scripts/derive-protected-experts.py" +APPLY="$REPO_ROOT/scripts/apply-protected-cast.sh" +CONVERT="$REPO_ROOT/scripts/convert-qwen38-w5nl64.sh" +GATE="$REPO_ROOT/scripts/kld-gate-qwen38.sh" +TEMPLATE="$REPO_ROOT/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe" + +# make_gram — a RADGRAM2 down-Gram whose +# trace (its down-input energy) is and whose routing count is . +make_gram() { + python3 - "$@" <<'PY' +import os, struct, sys +d, layer, e, energy, rows = sys.argv[1:6] +os.makedirs(d, exist_ok=True) +hdr = struct.pack("<8sIIQI4x", b"RADGRAM2", 2, 0, int(rows), 0) +mat = struct.pack("<4f", float(energy), 0.0, 0.0, 0.0) +with open(os.path.join(d, f"gram.r0.blk.{layer}.ffn_down_exps.{e}.bin"), "wb") as f: + f.write(hdr + mat) +PY +} + +# eight_grams — one layer, eight experts inside rank half 0, energies +# 60/15/10/8/4/2/1/0.5 against equal routing. Under the default bars (3% of the +# layer's energy AND 3x its routing share) exactly ONE expert is disproportionate, +# so the parity law has to hand it a partner. +eight_grams() { + local d="$1" + make_gram "$d" 0 0 60 100 + make_gram "$d" 0 1 15 100 + make_gram "$d" 0 2 10 100 + make_gram "$d" 0 3 8 100 + make_gram "$d" 0 4 4 100 + make_gram "$d" 0 5 2 100 + make_gram "$d" 0 6 1 100 + make_gram "$d" 0 7 0.5 100 +} + +cast_count() { grep -c 'cast dtype=bf16' <<< "$output"; } + +@test "the bar-only pass keeps the disproportionate expert and repairs the odd rank half" { + C="$BATS_TEST_TMPDIR/calib" + eight_grams "$C" + run python3 "$DERIVE" "$C" + [ "$status" -eq 0 ] + # e0 holds 59.7% of the layer's energy on 12.5% of the routing: the only expert + # that clears three times its routing share. e1 joins it because half 0 was odd. + [ "$(cast_count)" -eq 2 ] + [[ "$output" == *"blk.0.ffn_*_exps.0.weight"* ]] + [[ "$output" == *"blk.0.ffn_*_exps.1.weight"* ]] + [[ "$output" != *"ffn_*_exps.2.weight"* ]] + [[ "$output" == *"half 0 odd -> added e1"* ]] +} + +@test "--target-count lands the byte bill that was asked for, and never buys more" { + # The count is a disk and VRAM decision (each protected expert costs bf16 twice, + # once per rank half's copy), so landing it matters as much as not missing it. + C="$BATS_TEST_TMPDIR/calib" + eight_grams "$C" + run python3 "$DERIVE" "$C" --target-count 6 + [ "$status" -eq 0 ] + [ "$(cast_count)" -eq 6 ] + [[ "$output" == *"count fill -> added e2"* ]] + [[ "$output" != *"ffn_*_exps.6.weight"* ]] + # the energy share the set bottoms out at is the bar the campaign paid for + [[ "$output" == *"total protected: 6 (target 6, energy cutoff"* ]] +} + +@test "--target-count cannot invent experts: the pool's ceiling is reported, not padded" { + C="$BATS_TEST_TMPDIR/calib" + eight_grams "$C" + run python3 "$DERIVE" "$C" --target-count 20 + [ "$status" -eq 0 ] + [ "$(cast_count)" -eq 8 ] + [[ "$output" == *"total protected: 8 (target 20"* ]] +} + +@test "a protected set that cannot be made even is refused instead of emitted" { + # One disproportionate expert in half 0, its only companion in half 1: no + # candidate could make half 0 even, and an odd half hands the four-bit GEMM two + # tables that are not equal. Refusing is the only honest answer. + C="$BATS_TEST_TMPDIR/calib" + make_gram "$C" 0 0 100 100 + make_gram "$C" 0 300 1 1000 + run python3 "$DERIVE" "$C" + [ "$status" -eq 2 ] + [[ "$output" == *"PARITY LAW BROKEN"* ]] + [[ "$output" == *"blk.0 half 0 holds 1"* ]] + [ "$(cast_count)" -eq 0 ] +} + +@test "the splice rewrites the run's copy above the expert rule and refuses a second run" { + C="$BATS_TEST_TMPDIR/calib" + eight_grams "$C" + W="$BATS_TEST_TMPDIR/recipe-work.recipe" + cp "$TEMPLATE" "$W" + run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 0 + [ "$status" -eq 0 ] + grep -q "derive-protected-experts.py rewrote" "$W" + # The derived lines must precede the expert rule: the first rule whose glob + # matches a weight decides it, so a cast line below the expert line never fires. + cast_line=$(grep -n 'cast dtype=bf16' "$W" | head -1 | cut -d: -f1) + expert_line=$(grep -n 'ffn_\*_exps\.\*\.weight' "$W" | head -1 | cut -d: -f1) + [ "$cast_line" -lt "$expert_line" ] + # The committed template stays the clean, reviewable baseline. + ! grep -q "derive-protected-experts.py rewrote" "$TEMPLATE" + # A second run on the same copy would derive from a recording it already spent. + run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 0 + [ "$status" -ne 0 ] + [[ "$output" == *"refusing to re-derive"* ]] +} + +@test "the splice refuses before touching the recipe when the count is not reached" { + C="$BATS_TEST_TMPDIR/calib" + eight_grams "$C" + W="$BATS_TEST_TMPDIR/recipe-work.recipe" + cp "$TEMPLATE" "$W" + run env PY=python3 TRASH_DIR="$BATS_TEST_TMPDIR/trash" bash "$APPLY" "$C" "$W" 20 + [ "$status" -eq 1 ] + [[ "$output" == *"target was 20"* ]] + ! grep -q "derive-protected-experts.py rewrote" "$W" +} + +# --- the arm driver ---------------------------------------------------------- + +# drv_fixture — the smallest tree the driver's gates accept: a checkpoint +# with its index and tokenizer, three grams, an imatrix inside the campaign dir +# the container mounts, and an empty output dir. +drv_fixture() { + local d="$1" + mkdir -p "$d/ck/Qwen3.8-Flash-Next" "$d/campaign" "$d/calib" "$d/out" + : > "$d/ck/Qwen3.8-Flash-Next/model.safetensors.index.json" + : > "$d/ck/Qwen3.8-Flash-Next/tokenizer.json" + make_gram "$d/calib" 0 0 60 100 + make_gram "$d/calib" 0 1 15 100 + make_gram "$d/calib" 0 2 10 100 + # four experts, not three: a thin pool that cannot be made even trips the parity + # audit, and this fixture is meant to reach the later gates + make_gram "$d/calib" 0 3 8 100 + : > "$d/campaign/imatrix.gguf" +} + +# run_drv — the driver with every path +# pointed at the fixture. Override words are separated out because `env` only +# accepts assignments ahead of the command; later assignments win, so a test can +# break exactly one input. +run_drv() { + local d="$BATS_TEST_TMPDIR/drv" assigns=() flags=() w + for w in "$@"; do + if [[ "$w" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]]; then assigns+=("$w"); else flags+=("$w"); fi + done + run env CKPT="$d/ck/Qwen3.8-Flash-Next" CAMPAIGN="$d/campaign" STATE="$d/campaign" \ + CALIB="$d/calib" OUTDIR="$d/out" IMATRIX="$d/campaign/imatrix.gguf" \ + MIN_GRAMS=2 TARGET_COUNT=0 ${assigns[@]+"${assigns[@]}"} \ + bash "$CONVERT" ${flags[@]+"${flags[@]}"} +} + +@test "the dry run prints the plan and then the convert for every arm" { + drv_fixture "$BATS_TEST_TMPDIR/drv" + run_drv --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"== arm cd3 =="* ]] + [[ "$output" == *"== arm cd6 =="* ]] + [[ "$output" == *"== arm ao1 =="* ]] + # one --plan-only bill per arm, and one convert line per arm beside it + [ "$(grep -c -- '--plan-only' <<< "$output")" -eq 3 ] + [ "$(grep -c 'rad-convert' <<< "$output")" -eq 6 ] + awk '/arm cd3/ {next} + /qwen38-w5nl64-cd3\.rad/ { if ($0 ~ /--plan-only/) p = NR; else c = NR } + END { exit !(p && c && p < c) }' <<< "$output" +} + +@test "the cd6 arm re-sends the whole expert option set with cd=6; the control sends none" { + # --quant replaces the recipe line wholesale, so an override that forgot an + # option would quietly convert with the default for it. + drv_fixture "$BATS_TEST_TMPDIR/drv" + run_drv --arm cd3 --arm cd6 --dry-run + [ "$status" -eq 0 ] + cd6line=$(grep -- '-cd6\.rad --quant' <<< "$output" | head -1) + [[ "$cd6line" == *"table=w5nl"* ]] + [[ "$cd6line" == *"group=64"* ]] + [[ "$cd6line" == *"scale=fp8_e4m3"* ]] + [[ "$cd6line" == *"transform=fwht128"* ]] + [[ "$cd6line" == *"calib=\$CALIB"* ]] + [[ "$cd6line" == *",cd=6"* ]] + cd3line=$(grep -- '-cd3\.rad' <<< "$output" | grep -v -- '--plan-only' | head -1) + [[ "$cd3line" != *"--quant"* ]] +} + +@test "the ao1 arm carries act_order=1 and no cd=, because libquant refuses the pair" { + drv_fixture "$BATS_TEST_TMPDIR/drv" + run_drv --arm ao1 --dry-run + [ "$status" -eq 0 ] + line=$(grep -- '-ao1\.rad --quant' <<< "$output" | head -1) + [[ "$line" == *"act_order=1"* ]] + [[ "$line" != *",cd="* ]] +} + +@test "the driver splices the protected plane into its own copy, never the template" { + drv_fixture "$BATS_TEST_TMPDIR/drv" + run_drv --dry-run + [[ "$output" == *"apply-protected-cast.sh $BATS_TEST_TMPDIR/drv/calib "* ]] + [[ "$output" == *"/campaign/recipe-"* ]] + [[ "$output" != *"$TEMPLATE"* ]] +} + +@test "the driver refuses while the BF16 reference container still holds the volume" { + drv_fixture "$BATS_TEST_TMPDIR/drv" + : > "$BATS_TEST_TMPDIR/drv/out/qwen38-bf16.rad" + run_drv --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"peak-disk"* ]] +} + +@test "a thin recording is refused: the driver never converts uncalibrated" { + # rad-convert converts WITHOUT a calibration recording with one stderr note, + # which is silently worse than a GPTQ run that failed. + drv_fixture "$BATS_TEST_TMPDIR/drv" + run_drv MIN_GRAMS=9 --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"run-qwen38-bf16.sh calib"* ]] +} + +@test "an imatrix outside the mounted campaign dir is refused before any convert" { + drv_fixture "$BATS_TEST_TMPDIR/drv" + mv "$BATS_TEST_TMPDIR/drv/campaign/imatrix.gguf" "$BATS_TEST_TMPDIR/imatrix.gguf" + run_drv IMATRIX="$BATS_TEST_TMPDIR/imatrix.gguf" --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"not in the campaign dir"* ]] +} + +@test "the driver never overwrites an arm that already exists" { + drv_fixture "$BATS_TEST_TMPDIR/drv" + : > "$BATS_TEST_TMPDIR/drv/out/qwen38-w5nl64-cd6.rad" + run_drv --arm cd6 --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"move it to trash first"* ]] +} + +@test "an unknown arm name is refused before the gates run" { + # No fixture built: the name check must not need a checkpoint to exist. + run_drv --arm cd9 --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"unknown arm: cd9"* ]] +} + +@test "the driver stops when the protected derivation cannot land the target" { + drv_fixture "$BATS_TEST_TMPDIR/drv" + run_drv PY=python3 TARGET_COUNT=20 --arm cd3 + [ "$status" -eq 1 ] + [[ "$output" == *"target was 20"* ]] + [[ "$output" == *"would convert unprotected"* ]] + [[ "$output" != *"--entrypoint"* ]] +} + +# --- the acceptance gate ----------------------------------------------------- + +# kld_report — one score report, in the +# shape `run-qwen38-bf16.sh score` writes: an `all` summary plus a per-source one. +kld_report() { + python3 - "$@" <<'PY' +import json, sys +path, model, kl, top1 = sys.argv[1:5] +kl, top1 = float(kl), float(top1) +group = {"kld": {"mean": kl, "p99_9": kl * 4}, "top1_agreement": top1, "positions": 50432} +json.dump({"model": model, "all": group, + "by_source": {"ops": dict(group, positions=20000)}}, open(path, "w")) +PY +} + +gate_case() { # gate_case [extra flags...] + D="$BATS_TEST_TMPDIR/kl" + mkdir -p "$D" + kld_report "$D/fp8.json" fp8-native "$1" 0.981 + kld_report "$D/iq4r.json" iq4r-moe "$2" 0.962 + kld_report "$D/cand.json" w5nl64 "$3" "$4" + run bash "$GATE" "$D/fp8.json" "$D/iq4r.json" "$D/cand.json" "${@:5}" +} + +@test "an arm strictly inside the interval that closes half the gap passes" { + gate_case 0.0021 0.0190 0.0090 0.971 + [ "$status" -eq 0 ] + [[ "$output" == *"closes 59.2%"* ]] + [[ "$output" == *"KLD GATE PASS"* ]] +} + +@test "an arm worse than the served payload fails the interval and the recovery" { + gate_case 0.0021 0.0190 0.0210 0.971 + [ "$status" -eq 1 ] + [[ "$output" == *"[FAIL] 2."* ]] + [[ "$output" == *"[FAIL] 3."* ]] + [[ "$output" == *"KLD GATE FAIL"* ]] +} + +@test "anchors that are not ordered stop the gate: the instrument is wrong, not the arm" { + # If the near-lossless FP8 release does not sit CLOSER to the bf16 reference + # than the 4-bit payload does, the corpus, tokenizer or boot shape is broken and + # no arm number can mean anything. + gate_case 0.0300 0.0190 0.0120 0.971 + [ "$status" -eq 1 ] + [[ "$output" == *"[FAIL] 1."* ]] + [[ "$output" == *"gap is not positive"* ]] +} + +@test "an arm measured outside the interval fails even when it looks better" { + # Closer to the reference than the achievable anchor is a measurement fault. + gate_case 0.0021 0.0190 0.0010 0.971 + [ "$status" -eq 1 ] + [[ "$output" == *"[FAIL] 2."* ]] +} + +@test "an arm that buys KL with top-1 agreement fails" { + gate_case 0.0021 0.0190 0.0090 0.940 + [ "$status" -eq 1 ] + [[ "$output" == *"[PASS] 3."* ]] + [[ "$output" == *"[FAIL] 4."* ]] +} + +@test "--recover tightens the same numbers until they fail" { + gate_case 0.0021 0.0190 0.0090 0.971 --recover 0.8 + [ "$status" -eq 1 ] + [[ "$output" == *"needs 80%"* ]] + [[ "$output" == *"[FAIL] 3."* ]] +} + +@test "a slice the report does not carry is named, not guessed at" { + gate_case 0.0021 0.0190 0.0090 0.971 --slice code + [ "$status" -eq 2 ] + [[ "$output" == *"has no slice 'code'"* ]] + [[ "$output" == *"it has: ops"* ]] +}