fix(quant): a tap is only a tap when its rig is running
Four gates, all bought with the 2026-10-29 incident: the rig's calibration boot died of an OOM, the chain's health loop expired without failing, and the pusher fed four rounds of 10M tokens (86 minutes of prefill) into RESTORED PRODUCTION, recording nothing. - run-qwen38-bf16.sh: --ngram-placement becomes a knob whose calib default is `disk` (the 95.37 GiB table pin is what starved that boot: 230.22 GiB pinned of 246, 8.06 GiB available) while kldref and score keep `ram`. calib writes .rig-boot into the tap dir and retires it on a non-zero exit, so a dead boot cannot leave the tap looking rig-authorized. - calib-push.py: refuses before the first prompt unless the marker exists, names this container, and that container is running. With the driver's existing tap-progress gate, a push can no longer spend hours feeding an endpoint that is not the rig. - qwen38-calib-window.sh: the calibration window as a gated chain — stop on a failed phase instead of falling through the health wait, bank the 336 GiB reference container into trash only when both phases exit 0 and the tap holds grams, and print the whole plan on --dry-run. - kld-gate-qwen38.sh: two-report mode (candidate vs the served payload) for the disk where the 172.8 GiB FP8 anchor cannot live, plus the laws the report always carried and the gate never read: a p99.9 tail ceiling and every by_source slice passing mean, tail and top-1. - frank-archon.md: the pipeline laws (marker, n-gram tier, three-metric acceptance) and a Files section that names the live campaign instead of the retired swift15 era. Tests: 587/0 — 13 new behavioral cases (dry-run argv, marker refusals, two-report gate math).
This commit is contained in:
Frank committed
1 parent
3e5a37a21f
commit
c238e32d6f
6 files changed
+523
-56
No files matched your search
@@ -200,6 +200,9 @@ DERIVE="$REPO_ROOT/scripts/derive-protected-experts.py"
|
||||
APPLY="$REPO_ROOT/scripts/apply-protected-cast.sh"
|
||||
CONVERT="$REPO_ROOT/scripts/convert-qwen38-w5nl64.sh"
|
||||
GATE="$REPO_ROOT/scripts/kld-gate-qwen38.sh"
|
||||
RUNNER="$REPO_ROOT/scripts/run-qwen38-bf16.sh"
|
||||
PUSHER="$REPO_ROOT/scripts/calib-push.py"
|
||||
WINDOW="$REPO_ROOT/scripts/qwen38-calib-window.sh"
|
||||
TEMPLATE="$REPO_ROOT/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe"
|
||||
|
||||
# make_gram <dir> <layer> <expert> <energy> <rows> — a RADGRAM2 down-Gram whose
|
||||
@@ -525,3 +528,174 @@ gate_case() { # gate_case <kl fp8> <kl iq4r> <kl cand> <top1 cand> [extra flags.
|
||||
[[ "$output" == *"has no slice 'code'"* ]]
|
||||
[[ "$output" == *"it has: ops"* ]]
|
||||
}
|
||||
|
||||
# --- the n-gram tier: the pin that starved the calibration boot ---------------
|
||||
|
||||
@test "the calibration boot reads the n-gram table from the file tier" {
|
||||
# 2026-10-29: 95.37 GiB pinned beside a 65.59 GiB/rank pool left 8.06 GiB of
|
||||
# RAM and the boot died "startup stopped: out of memory". calib only needs the
|
||||
# routed weights prefilled, so it reads the table off the container's file tier.
|
||||
run bash "$RUNNER" calib --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'disk'
|
||||
}
|
||||
|
||||
@test "the recording and every score pass keep production's n-gram placement" {
|
||||
# Those numbers are only comparable when the shape is identical; the tier is
|
||||
# part of the shape.
|
||||
run bash "$RUNNER" kldref --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'ram'
|
||||
}
|
||||
|
||||
@test "an unknown n-gram placement is refused by name before the boot" {
|
||||
run env FRANK_QWEN38_NGRAM_PLACEMENT=ssd bash "$RUNNER" calib --dry-run
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"auto|vram|ram|disk"* ]]
|
||||
}
|
||||
|
||||
# --- the pusher: a tap without a live rig is not a tap ------------------------
|
||||
|
||||
# stub_podman <dir> <running|absent> — a podman that answers the one question the
|
||||
# pusher asks: is the rig container running?
|
||||
stub_podman() {
|
||||
mkdir -p "$1/bin"
|
||||
if [ "$2" = running ]; then
|
||||
printf '#!/usr/bin/env bash\necho true\n' > "$1/bin/podman"
|
||||
elif [ "$2" = stopped ]; then
|
||||
printf '#!/usr/bin/env bash\necho false\n' > "$1/bin/podman"
|
||||
else
|
||||
printf '#!/usr/bin/env bash\nexit 125\n' > "$1/bin/podman"
|
||||
fi
|
||||
chmod +x "$1/bin/podman"
|
||||
}
|
||||
|
||||
pusher_fixture() { # pusher_fixture <marker-container or empty>
|
||||
D="$BATS_TEST_TMPDIR/push"
|
||||
mkdir -p "$D/tap" "$D/bin"
|
||||
printf '{"prompt":"hello"}\n' > "$D/corpus.jsonl"
|
||||
if [ -n "$1" ]; then
|
||||
printf 'container=%s\nimage=radiance@sha256:test\nmode=calib\npid=1\n' "$1" \
|
||||
> "$D/tap/.rig-boot"
|
||||
fi
|
||||
echo "$D"
|
||||
}
|
||||
|
||||
@test "the pusher refuses a tap that carries no rig marker: production is not a rig" {
|
||||
# With no marker, the old driver fed four rounds of 10M tokens into restored
|
||||
# production after the rig's boot had died, and recorded nothing.
|
||||
D="$(pusher_fixture '')"
|
||||
run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" --port 1
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"no rig marker"* ]]
|
||||
}
|
||||
|
||||
@test "the pusher refuses a marker that names another container" {
|
||||
D="$(pusher_fixture arbench)"
|
||||
run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" \
|
||||
--container qwen38-bf16 --port 1
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"names container 'arbench'"* ]]
|
||||
[[ "$output" == *"targets 'qwen38-bf16'"* ]]
|
||||
}
|
||||
|
||||
@test "the pusher refuses when the rig container is not running" {
|
||||
D="$(pusher_fixture qwen38-bf16)"
|
||||
stub_podman "$D" stopped
|
||||
run env PATH="$D/bin:$PATH" python3 "$PUSHER" --corpus "$D/corpus.jsonl" \
|
||||
--tap-dir "$D/tap" --container qwen38-bf16 --port 1
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"is not running"* ]]
|
||||
}
|
||||
|
||||
# --- the acceptance gate without an FP8 anchor -------------------------------
|
||||
|
||||
# kld_report_slices <path> <model> <kl all> <kl code> <top1 all> <top1 code>
|
||||
kld_report_slices() {
|
||||
python3 - "$@" <<'PY'
|
||||
import json, sys
|
||||
path, model, kl_all, kl_code, t_all, t_code = sys.argv[1:7]
|
||||
def g(kl, t, n=25000):
|
||||
return {"kld": {"mean": float(kl), "p99_9": float(kl) * 4},
|
||||
"top1_agreement": float(t), "positions": n}
|
||||
json.dump({"model": model, "all": g(kl_all, t_all),
|
||||
"by_source": {"code": g(kl_code, t_code), "ops": g(kl_all, t_all)}},
|
||||
open(path, "w"))
|
||||
PY
|
||||
}
|
||||
|
||||
gate2() { # gate2 <kl iq4r> <kl cand> <top1 iq4r> <top1 cand> [extra]
|
||||
D="$BATS_TEST_TMPDIR/kl2"
|
||||
mkdir -p "$D"
|
||||
kld_report_slices "$D/iq4r.json" iq4r-moe "$1" "$1" "$3" "$3"
|
||||
kld_report_slices "$D/cand.json" w5nl64 "$2" "$2" "$4" "$4"
|
||||
run bash "$GATE" "$D/iq4r.json" "$D/cand.json" "${@:5}"
|
||||
}
|
||||
|
||||
@test "a candidate that beats the served payload on mean, tail and top-1 passes without an anchor" {
|
||||
gate2 0.0190 0.0090 0.962 0.971
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"candidate vs served (no FP8 anchor)"* ]]
|
||||
[[ "$output" == *"KLD GATE PASS"* ]]
|
||||
}
|
||||
|
||||
@test "a better mean with a worse tail fails: the tail is where a mean hides its damage" {
|
||||
D="$BATS_TEST_TMPDIR/kl3"
|
||||
mkdir -p "$D"
|
||||
kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962
|
||||
python3 - "$D/cand.json" <<'PY'
|
||||
import json, sys
|
||||
# mean better than the served payload, p99.9 worse: the shape a mean-only gate waves through
|
||||
g = {"kld": {"mean": 0.0090, "p99_9": 0.4000}, "top1_agreement": 0.971, "positions": 25000}
|
||||
json.dump({"model": "w5nl64", "all": g, "by_source": {"code": dict(g), "ops": dict(g)}},
|
||||
open(sys.argv[1], "w"))
|
||||
PY
|
||||
run bash "$GATE" "$D/iq4r.json" "$D/cand.json"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"[FAIL] 2."* ]]
|
||||
[[ "$output" == *"KL p99.9"* ]]
|
||||
}
|
||||
|
||||
@test "a candidate that wins overall but regresses one source slice fails" {
|
||||
D="$BATS_TEST_TMPDIR/kl4"
|
||||
mkdir -p "$D"
|
||||
kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962
|
||||
# 'all' is better, 'code' is worse: exactly the trade a mean-only gate cannot see
|
||||
kld_report_slices "$D/cand.json" w5nl64 0.0090 0.0300 0.971 0.950
|
||||
run bash "$GATE" "$D/iq4r.json" "$D/cand.json"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"[PASS] 1."* ]]
|
||||
[[ "$output" == *"[FAIL] 4."* ]]
|
||||
[[ "$output" == *"regressed: code"* ]]
|
||||
}
|
||||
|
||||
# --- the calibration window: gated, and it says so ---------------------------
|
||||
|
||||
window_fixture() {
|
||||
D="$BATS_TEST_TMPDIR/win"
|
||||
mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora"
|
||||
echo '{}' > "$D/campaign/kld-ref/kld.json"
|
||||
: > "$D/rad/qwen38-bf16.rad"
|
||||
: > "$D/corpora/calib.jsonl"
|
||||
echo "$D"
|
||||
}
|
||||
|
||||
@test "the calibration window refuses before touching a card without a reference recording" {
|
||||
D="$BATS_TEST_TMPDIR/win-less"
|
||||
mkdir -p "$D/campaign/calib"
|
||||
run env FRANK_QWEN38_CAMPAIGN="$D/campaign" bash "$WINDOW"
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"no reference recording"* ]]
|
||||
}
|
||||
|
||||
@test "the calibration window's dry run prints the gated plan, including what it would bank" {
|
||||
D="$(window_fixture)"
|
||||
run env FRANK_QWEN38_CAMPAIGN="$D/campaign" \
|
||||
FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" bash "$WINDOW" --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"run-qwen38-bf16.sh calib"* ]]
|
||||
[[ "$output" == *"else kill the rig"* ]]
|
||||
[[ "$output" == *"calib-push.py"* ]]
|
||||
[[ "$output" == *"AGENTTRASH"* ]]
|
||||
}
|
||||
Reference in new issue
Block a user