Merge pull request 'fix(quant): a tap is only a tap when its rig is running — marker gate, gated calib window, tail-and-slices gate' (#84) from fix/qwen38-tap-marker into main

scripts/calib-push.py

@@ -25,6 +25,11 @@ Mechanics notes:
 - a round that seals no tap bytes is a FAILURE, not a retry: the engine
   answering 200 while the tap directory is wrong, full or read-only is
   how a corpus gets pushed four times for nothing.
+- before any prompt, the tap must carry the rig's own marker
+  (`--tap-marker`, written by run-qwen38-bf16.sh calib) naming THIS
+  container, and that container must be running. Measured 2026-10-29:
+  with neither check, four rounds of 10M tokens went into restored
+  production after the rig's boot had died, and recorded nothing.
 
 Usage: calib-push.py [--port 7770] [--parallel 64] [--container arbench]
        [--corpus /home/dank/corpora/out/calib.jsonl] [--max-rounds 4]
@@ -107,6 +112,40 @@ def tap_bytes(path):
     return total
 
 
+def require_rig(a):
+    """Refuse unless the rig that opened this tap is up and serving right now.
+
+    Three facts, in order: the runner wrote its marker into the tap dir, the
+    marker names the container this pusher targets, and that container is
+    running. A marker without a live container is a boot that died; a live
+    container without a marker is some other engine on the port — on 2026-10-29
+    it was restored production, which took four rounds of 10M tokens before the
+    old driver admitted it had recorded nothing.
+    """
+    marker = os.path.join(a.tap_dir, a.tap_marker)
+    if not os.path.isfile(marker):
+        qc.refuse(f"no rig marker at {marker}: this tap was not opened by "
+                  "run-qwen38-bf16.sh calib, so a push here would feed whatever "
+                  "answers the port and record nothing")
+    try:
+        lines = open(marker, encoding="utf-8", errors="replace").read().splitlines()
+    except OSError as exc:
+        qc.refuse(f"cannot read the rig marker {marker}: {exc}")
+    meta = dict(line.split("=", 1) for line in lines if "=" in line)
+    who = meta.get("container", "")
+    if who != a.container:
+        qc.refuse(f"the rig marker names container {who!r} and this pusher targets "
+                  f"{a.container!r} — wrong tap, or a stale marker")
+    try:
+        p = subprocess.run(["podman", "inspect", "-f", "{{.State.Running}}", a.container],
+                           capture_output=True, text=True, timeout=30)
+    except Exception as exc:
+        qc.refuse(f"cannot inspect the rig container {a.container}: {exc}")
+    if p.returncode != 0 or p.stdout.strip() != "true":
+        qc.refuse(f"the rig container {a.container} is not running — whatever is "
+                  "answering the port, it is not the calibration rig")
+
+
 def parse_args():
     ap = argparse.ArgumentParser()
     ap.add_argument("--port", type=int, default=7770)
@@ -114,6 +153,8 @@ def parse_args():
     ap.add_argument("--container", default="arbench")
     ap.add_argument("--model", default="frank-archon")
     ap.add_argument("--tap-dir", default="/home/dank/campaign/qwen38-w5nl64/calib")
+    ap.add_argument("--tap-marker", default=".rig-boot",
+                    help="the file run-qwen38-bf16.sh calib writes into the tap dir; without it the pusher refuses, because an unmarked tap is not a rig")
     ap.add_argument("--corpus", default="/home/dank/corpora/out/calib.jsonl")
     ap.add_argument("--max-rounds", type=int, default=4)
     return ap.parse_args()
@@ -136,6 +177,7 @@ def main():
     a = parse_args()
     url = f"http://127.0.0.1:{a.port}/v1/completions"
     prompts = load_prompts(a)
+    require_rig(a)
     chars = sum(len(p) for p in prompts)
     print(f"corpus: {len(prompts)} prompts, ~{chars // 4} tokens (chars/4)",
           flush=True)

scripts/kld-gate-qwen38.sh

@@ -1,44 +1,53 @@
 #!/usr/bin/env bash
 # kld-gate-qwen38.sh — RUNS ON DANKSERVER (or anywhere the reports are). The campaign's
-# acceptance gate: three KLD reports against ONE reference recording, one verdict.
+# acceptance gate: one verdict from the KLD reports of the containers that ride one reference
+# recording.
 #
 # The reference is the BF16 container built from the vendor checkpoint (run-qwen38-bf16.sh
 # kldref, in the same drained window that records the calibration grams). Every scored
 # container is measured against that one recording, so the KL scale runs from 0 = the
-# checkpoint itself downward in faithfulness. Three containers ride it:
+# checkpoint itself downward in faithfulness.
 #
-#   fp8.json    the published FP8-native release   — the achievable end of the interval
+# TWO MODES, because the disk decides how many anchors can exist at once:
-#   iq4r.json   the payload frank-archon serves    — the bar to beat
-#   cand.json   the arm under test                 — the subject
 #
-# The FP8 number is what makes the interval absolute instead of relative: "half the gap"
+#   gate IQ4R.json CAND.json                    candidate vs the served payload
-# is half of the distance between the 4-bit payload and the near-lossless release, not half
+#   gate FP8.json IQ4R.json CAND.json           the same, inside the absolute interval
-# of an invented distance to the checkpoint. That is also why the FP8 payload is FETCHED and
-# scored, never converted: it is a second opinion about what this checkpoint tolerates, not a
-# step of the campaign. It lands LAST (disk sequencing) — by then the 335 GiB bf16 container
-# is gone, so its 172.8 GiB fits.
 #
-# The law, in order of severity, on the mean KL of one slice (--slice all | a corpus source):
+# iq4r.json is the payload frank-archon serves — the bar to beat. When the FP8-native release
-#   1. KL(fp8) < KL(iq4r)                 the anchors are ordered. If the near-lossless
+# (172.8 GiB, fetched last because of the peak-disk law) is on disk, pass its report first:
-#                                         release does NOT sit closer to the checkpoint than
+# "half the gap" is then half of the distance between the 4-bit payload and the near-lossless
-#                                         the 4-bit payload, the corpus, the tokenizer or the
+# release, not half of an invented distance to the checkpoint. Without it the gate still
-#                                         boot shape is wrong and no arm number means anything.
+# answers the campaign's question — does the candidate beat what is served — and says so.
-#   2. KL(fp8) < KL(cand) < KL(iq4r)      the candidate lies strictly inside the interval —
+#
-#                                         outside it, something is mis-measured, not better
+# The law, on the summary of one slice (--slice all | a corpus source):
-#   3. KL(cand) <= KL(iq4r) - R x gap     it closes at least R (--recover, default 0.5) of the
+#
-#                                         gap = KL(iq4r) - KL(fp8): at least half the way from
+#   candidate vs served (2 reports)
-#                                         what is served toward the release (agreed 2026-10-27)
+#   1. mean KL(cand) < mean KL(iq4r)         the served payload is beaten on the corpus
-#   4. top-1(cand) >= top-1(iq4r)         the agreement the harness keys on does not fall
+#   2. p99.9(cand) <= p99.9(iq4r)            and the tail is no worse — a mean can hide a few
+#                                            positions taking the whole hit
+#   3. top-1(cand) >= top-1(iq4r)            the agreement the harness keys on does not fall
+#   4. every source slice passes 1-3         a win bought by regressing one distribution
+#                                            (code, prose, tools) is not a win
+#
+#   inside the interval (3 reports), laws 1-4 above (1-3 renumbered to the interval and the
+#   recovery fraction, as agreed 2026-10-27) plus:
+#   5. p99.9(cand) <= p99.9(iq4r)            the tail law
+#   6. every source slice passes            the slice law
 #
 # Booting the containers is NOT this script's job: `run-qwen38-bf16.sh score RAD OUT.json`
 # writes the reports and owns the single engine argv (same pin, --tp 2, --kv-cache-dtype fp8,
-# same step size, --deterministic, no speculation) that makes the three numbers comparable.
+# same step size, --deterministic, no speculation) that makes the numbers comparable.
 # This script is arithmetic, so it re-runs against old reports and is testable without a GPU.
 #
-# Usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]
+# Usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]
+#        kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]
 set -uo pipefail
 
-[ "$#" -ge 3 ] || { echo "usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]" >&2; exit 2; }
+if [ "$#" -lt 2 ]; then
+    echo "usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]" >&2
+    echo "       kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]" >&2
+    exit 2
+fi
 python3 - "$@" <<'PY'
 import json, sys
 
@@ -52,13 +61,15 @@ while i < len(a):
     else:
         files.append(a[i])
         i += 1
-if len(files) != 3:
+if len(files) not in (2, 3):
-    sys.exit("usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]")
+    sys.exit("usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]\n"
+             "       kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]")
 o = dict(zip(opts[0::2], opts[1::2]))
 slice_ = o.get("--slice", "all")
 recover = float(o.get("--recover", 0.5))
 if not 0.0 < recover <= 1.0:
     sys.exit(f"--recover must be in (0,1], got {recover}")
+interval = len(files) == 3
 
 
 def die(msg):
@@ -66,11 +77,15 @@ def die(msg):
     raise SystemExit(2)
 
 
-def read(path):
+def load(path):
     try:
-        d = json.load(open(path))
+        return json.load(open(path))
     except Exception as e:
         die(f"cannot read the KLD report {path}: {e}")
+
+
+def summary(path, d):
+    """The slice this gate grades: 'all', or one corpus source."""
     if slice_ == "all":
         g = d.get("all")
     else:
@@ -80,23 +95,25 @@ def read(path):
                 f"{', '.join(sorted(d.get('by_source') or {})) or '(none)'}")
     if g is None or "kld" not in g or "top1_agreement" not in g:
         die(f"{path} has no {slice_!r} summary — wrong report version?")
-    return d, g
+    return g
 
 
+docs = {label: load(path) for label, path in
+        zip(("fp8", "iq4r", "cand") if interval else ("iq4r", "cand"), files)}
 rows = []
-for label, path in zip(("fp8", "iq4r", "cand"), files):
+for label, path in zip(("fp8", "iq4r", "cand") if interval else ("iq4r", "cand"), files):
-    d, g = read(path)
+    g = summary(path, docs[label])
-    rows.append((label, path, d.get("model", "?"), g["kld"]["mean"], g["kld"]["p99_9"],
+    rows.append((label, path, docs[label].get("model", "?"), g["kld"]["mean"],
-                 g["top1_agreement"], g["positions"]))
+                 g["kld"]["p99_9"], g["top1_agreement"], g["positions"]))
 
-print(f"reference: BF16 container; slice: {slice_}")
+print(f"reference: BF16 container; slice: {slice_}; "
+      + ("inside the FP8 interval" if interval else "candidate vs served (no FP8 anchor)"))
 print(f"{'':6} {'model':38} {'KL mean':>10} {'KL p99.9':>10} {'top-1':>8}  positions")
 for label, path, model, kl, p999, top1, n in rows:
     print(f"{label:6} {str(model)[:38]:38} {kl:10.6f} {p999:10.4f} {100*top1:7.2f}%  {n}")
-r = {label: (kl, top1) for label, _p, _m, kl, _q, top1, _n in rows}
+r = {label: (kl, p999, top1) for label, _p, _m, kl, p999, top1, _n in rows}
-kl_fp8, kl_iq4r, kl_cand = r["fp8"][0], r["iq4r"][0], r["cand"][0]
+kl_cand, p_cand, t_cand = r["cand"]
-t_iq4r, t_cand = r["iq4r"][1], r["cand"][1]
+kl_iq4r, p_iq4r, t_iq4r = r["iq4r"]
-gap = kl_iq4r - kl_fp8
 ok = True
 
 
@@ -106,17 +123,64 @@ def law(n, text, passed):
     print(f"  [{'PASS' if passed else 'FAIL'}] {n}. {text}")
 
 
-law(1, f"anchors ordered: fp8 {kl_fp8:.6f} < iq4r {kl_iq4r:.6f} (gap {gap:.6f})",
+def tail_law(n):
-    gap > 0)
+    law(n, f"candidate KL p99.9 {p_cand:.4f} <= served iq4r {p_iq4r:.4f}",
-law(2, f"candidate inside the interval: {kl_fp8:.6f} < {kl_cand:.6f} < {kl_iq4r:.6f}",
+        p_cand <= p_iq4r)
-    kl_fp8 < kl_cand < kl_iq4r)
+
-need = kl_iq4r - recover * gap
+
-law(3, f"candidate closes {100*max((kl_iq4r - kl_cand)/gap, 0):.1f}% of the "
+# --- the slice law: every corpus source must pass the metric laws -------------
-       f"{gap:.6f} gap (needs {100*recover:.0f}%, i.e. KL <= {need:.6f})"
+def slice_law(n):
-       if gap > 0 else "gap is not positive; cannot score recovery",
+    """A win that regresses one source class is not a win: check them all."""
-    gap > 0 and kl_cand <= need)
+    mine = docs["cand"].get("by_source") or {}
-law(4, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%",
+    theirs = docs["iq4r"].get("by_source") or {}
-    t_cand >= t_iq4r)
+    if not mine or not theirs:
+        law(n, "no per-source summaries to check", True)
+        return
+    missing = sorted(set(theirs) - set(mine))
+    extra = sorted(set(mine) - set(theirs))
+    if missing or extra:
+        law(n, "same source slices in both reports"
+            + (f" (missing from cand: {', '.join(missing)})" if missing else "")
+            + (f" (only in cand: {', '.join(extra)})" if extra else ""), False)
+        return
+    bad = []
+    for src in sorted(theirs):
+        c, q = mine[src], theirs[src]
+        worse_mean = c["kld"]["mean"] >= q["kld"]["mean"]
+        worse_tail = c["kld"]["p99_9"] > q["kld"]["p99_9"]
+        worse_top1 = c["top1_agreement"] < q["top1_agreement"]
+        if worse_mean or worse_tail or worse_top1:
+            why = ",".join(m for m, w in (("mean", worse_mean), ("p99.9", worse_tail),
+                                          ("top-1", worse_top1)) if w)
+            bad.append(f"{src} ({why})")
+    law(n, "every source slice passes (mean, p99.9, top-1)"
+        + (f" — regressed: {'; '.join(bad)}" if bad else ""), not bad)
+
+
+if interval:
+    kl_fp8 = r["fp8"][0]
+    gap = kl_iq4r - kl_fp8
+    law(1, f"anchors ordered: fp8 {kl_fp8:.6f} < iq4r {kl_iq4r:.6f} (gap {gap:.6f})",
+        gap > 0)
+    law(2, f"candidate inside the interval: {kl_fp8:.6f} < {kl_cand:.6f} < {kl_iq4r:.6f}",
+        kl_fp8 < kl_cand < kl_iq4r)
+    need = kl_iq4r - recover * gap
+    law(3, f"candidate closes {100*max((kl_iq4r - kl_cand)/gap, 0):.1f}% of the "
+           f"{gap:.6f} gap (needs {100*recover:.0f}%, i.e. KL <= {need:.6f})"
+           if gap > 0 else "gap is not positive; cannot score recovery",
+        gap > 0 and kl_cand <= need)
+    law(4, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%",
+        t_cand >= t_iq4r)
+    tail_law(5)
+    slice_law(6)
+else:
+    law(1, f"candidate beats the served payload: KL {kl_cand:.6f} < iq4r {kl_iq4r:.6f}",
+        kl_cand < kl_iq4r)
+    tail_law(2)
+    law(3, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%",
+        t_cand >= t_iq4r)
+    slice_law(4)
+
 print("KLD GATE " + ("PASS" if ok else "FAIL"))
 sys.exit(0 if ok else 1)
 PY

scripts/qwen38-calib-window.sh

@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# qwen38-calib-window.sh — RUNS ON DANKSERVER. The calibration window, and nothing else:
+# boot the BF16 rig, wait until it is actually serving, push the corpus until the engine says
+# the tap is full, stop the rig, restore frank-archon, and bank the reference container.
+#
+# The reference recording is already taken (kld-ref/kld.json exists), so this window is the
+# calibration tap alone — the one job that needs both cards after it. The kldref phase is NOT
+# re-run here: the chained launcher that used to do that moved the banked kld-ref into trash
+# first, which is how a 16-hour phase gets paid for twice.
+#
+# Every phase is gated on the previous one, because the last time this ran unguarded
+# (2026-10-29) the rig's boot died of an OOM, the health loop quietly fell through, and the
+# pusher fed four rounds of 10M tokens into RESTORED PRODUCTION with nothing recorded:
+#
+#   1. the runner must still be alive and its container must answer /health, or this script
+#      kills it, restores the service and exits non-zero — no push, ever;
+#   2. the pusher refuses on its own unless the rig wrote its marker into the tap dir, the
+#      marker names this container, and the container is running (scripts/calib-push.py
+#      --tap-marker), so a second lock survives a mistake in this file;
+#   3. the reference container is banked into trash only when BOTH the serve and the push
+#      exited 0 and the tap dir holds grams. The checkpoint that reproduces it stays.
+#
+# knobs: Q38_WINDOW_CALIB_CORPUS (calib-7168.jsonl — the re-chunked file the rig's pool can
+#        hold), Q38_WINDOW_HEALTH_TIMEOUT (seconds to wait for /health), Q38_WINDOW_ROUNDS
+#        (pusher rounds; one is 10M tokens), Q38_WINDOW_KEEP_RAD=1 to leave the .rad in place.
+# Usage: scripts/qwen38-calib-window.sh [--dry-run]
+set -uo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 9
+
+DRY=0
+[ "${1:-}" = "--dry-run" ] && DRY=1
+say() { printf '%s %s\n' "$(date -Is)" "$*"; }
+
+MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
+CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
+RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}"
+CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}"
+KLDREF="$CAMPAIGN/kld-ref/kld.json"
+CALIB="$CAMPAIGN/calib"
+HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}"
+ROUNDS="${Q38_WINDOW_ROUNDS:-6}"
+RIG=qwen38-bf16
+SERVICE=frank-archon
+PY="${PY:-python3}"
+
+# ---- preflight: everything this window needs, before a card is touched -------
+[ -f "$KLDREF" ] || { echo "REFUSED: no reference recording at $KLDREF — record it first (scripts/run-qwen38-bf16.sh kldref); this window only fills the calibration tap" >&2; exit 2; }
+[ -f "$RAD" ]    || { echo "REFUSED: no BF16 reference container at $RAD — nothing to serve; scripts/build-qwen38-bf16-rad.sh" >&2; exit 2; }
+[ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; }
+[ -d "$CALIB" ]  || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; }
+[ -w "$CALIB" ]  || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; }
+! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; }
+! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; }
+
+if [ "$DRY" = 1 ]; then
+    echo "dry run: calib window — the service is left running, nothing is mounted or written"
+    echo "would run: bash scripts/run-qwen38-bf16.sh calib          (rig boots, writes $CALIB/.rig-boot)"
+    echo "would wait: /health up within ${HEALTH_TIMEOUT}s, else kill the rig, restore $SERVICE, exit 1"
+    echo "would run: $PY scripts/calib-push.py --container $RIG --corpus $CORPUS --max-rounds $ROUNDS"
+    echo "would bank: $RAD -> ~/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad/  (only if both phases exit 0)"
+    exit 0
+fi
+
+say "calib window start pid=$$ corpus=$CORPUS rounds=$ROUNDS"
+[ -n "$(ls -A "$CALIB" 2>/dev/null)" ] && say "tap dir $CALIB is not empty; the engine's token floor is pooled, so an earlier partial tap still counts"
+
+# ---- phase 1: the rig -------------------------------------------------------
+say "phase=calib serve begin"
+bash scripts/run-qwen38-bf16.sh calib & spid=$!
+
+rig_up() {
+    podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true \
+        && curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health
+}
+
+waited=0
+until rig_up; do
+    if ! kill -0 "$spid" 2>/dev/null; then
+        wait "$spid"; crc=$?
+        say "phase=calib serve end rc=$crc (the rig exited before it ever answered /health)"
+        echo "FAIL: the rig boot died (rc=$crc) — nothing was pushed; read /home/dank/logs/qwen38_bf16_calib.log" >&2
+        exit 1
+    fi
+    if [ "$waited" -ge "$HEALTH_TIMEOUT" ]; then
+        say "phase=calib health never came after ${HEALTH_TIMEOUT}s; killing the rig"
+        podman stop "$RIG" >/dev/null 2>&1 || true
+        wait "$spid"
+        echo "FAIL: the rig never answered /health within ${HEALTH_TIMEOUT}s — nothing was pushed" >&2
+        exit 1
+    fi
+    sleep 15
+    waited=$((waited + 15))
+done
+say "phase=calib health ok waited=${waited}s"
+
+# ---- phase 2: the tap -------------------------------------------------------
+say "phase=calib push begin"
+"$PY" scripts/calib-push.py --port 7770 --parallel 8 --container "$RIG" \
+    --tap-dir "$CALIB" --corpus "$CORPUS" --max-rounds "$ROUNDS"
+prc=$?
+say "phase=calib push end rc=$prc"
+
+podman stop "$RIG" >/dev/null 2>&1 || true
+wait "$spid"; crc=$?
+say "phase=calib serve end rc=$crc"
+
+if [ "$prc" -ne 0 ] || [ "$crc" -ne 0 ]; then
+    echo "FAIL: serve rc=$crc push rc=$prc — the tap is NOT banked and $RAD stays: $(basename "$RAD")" >&2
+    exit 1
+fi
+[ -n "$(ls -A "$CALIB" 2>/dev/null)" ] || { echo "FAIL: both phases exited 0 but the tap dir $CALIB is empty" >&2; exit 1; }
+say "TAP BANKED: $(ls -1 "$CALIB" | wc -l) file(s) in $CALIB"
+
+# ---- phase 3: bank the reference container ----------------------------------
+if [ "${Q38_WINDOW_KEEP_RAD:-0}" = "1" ]; then
+    say "reference .rad kept (Q38_WINDOW_KEEP_RAD=1)"
+else
+    T="$HOME/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad"
+    mkdir -p "$T" || { echo "FAIL: cannot create trash dir $T" >&2; exit 1; }
+    mv "$RAD" "$T/" && say "reference .rad banked in $T (the checkpoint that rebuilds it stays)"
+fi
+say "calib window end"

scripts/run-qwen38-bf16.sh

@@ -193,6 +193,25 @@ if ! [[ "$MAX_MODEL_LEN" =~ ^[0-9]+$ ]]; then
     exit 2
 fi
 
+# Where the n-gram table lives. On the bf16 payload it is 95.37 GiB and `ram`
+# PINS it: the calibration boot died at "startup stopped: out of memory" with
+# 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available (measured 2026-10-29),
+# while the reference recording and every score boot had already succeeded with
+# the same pool and slots. The difference that matters is the job: calib only
+# needs the routed weights prefilled, so it reads the table from the container's
+# own file tier and the pin never happens; kldref and score keep production's
+# placement, because their numbers are only comparable when every other term of
+# the shape is identical. FRANK_QWEN38_NGRAM_PLACEMENT overrides either default.
+case "$MODE" in
+    calib) NGRAM_PLACEMENT="${FRANK_QWEN38_NGRAM_PLACEMENT:-disk}" ;;
+    *)     NGRAM_PLACEMENT="${FRANK_QWEN38_NGRAM_PLACEMENT:-ram}" ;;
+esac
+case "$NGRAM_PLACEMENT" in
+    auto|vram|ram|disk) ;;
+    *) echo "REFUSED: FRANK_QWEN38_NGRAM_PLACEMENT must be auto|vram|ram|disk (got: $NGRAM_PLACEMENT)" >&2
+       exit 2 ;;
+esac
+
 # Every container this campaign boots is read through the ro /models mount, so
 # the host path must live under it; the container-side spelling is what goes
 # into the argv. Same rule for the container a score pass opens.
@@ -230,7 +249,7 @@ ARGS=(--tp 2 --placement expert_tiered --weights-disk-tier
       --host-pool-mib "$HOST_POOL_MIB" --gpu-headroom-mib "$GPU_HEADROOM_MIB"
       --deterministic --num-speculative-tokens 0
       --kv-cache-dtype fp8 --max-model-len "$MAX_MODEL_LEN"
-      --ngram-placement ram
+      --ngram-placement "$NGRAM_PLACEMENT"
       --override-chat-template /models/tpl.jinja)
 MOUNTS=(-v "$MODELS:/models:ro")
 TAPENV=()
@@ -289,6 +308,17 @@ case "$MODE" in
     MOUNTS+=(-v "$CALIB_DIR:/data/calib")
     TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib
             -e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-12000000}")
+    # The tap's own authorization. A pusher must see this marker before it sends
+    # a prompt (calib-push.py --tap-marker), it names the container so a push
+    # aimed anywhere else refuses, and a boot that fails removes it again below.
+    # Measured 2026-10-29: with no marker, the chain pushed 4 rounds x 10M tokens
+    # into RESTORED PRODUCTION after the rig's boot died, and nothing recorded.
+    if [ "$DRY" = 1 ]; then
+        :
+    else
+        printf 'container=%s\nimage=%s\nmode=calib\nstarted=%s\npid=%s\n' \
+            "$C" "$IMAGE" "$(date -Is)" "$$" > "$CALIB_DIR/.rig-boot"
+    fi
     ;;
 esac
 
@@ -377,5 +407,12 @@ podman rm -f "$C" >/dev/null 2>&1 || true
 
 "${CMD[@]}" 2>&1 | tee "$LOG"
 rc=${PIPESTATUS[0]}
+# A boot that failed must not leave the tap marked as rig-authorized: the marker
+# is what calib-push.py trusts, and the point of it is that a dead rig can never
+# be pushed into again.
+if [ "$MODE" = calib ] && [ "$rc" -ne 0 ] && [ -f "$CALIB_DIR/.rig-boot" ]; then
+    mv -f "$CALIB_DIR/.rig-boot" "$CALIB_DIR/.rig-boot.failed-$(date +%Y%m%d-%H%M%S)" 2>/dev/null || true
+    echo "the rig boot failed (rc=$rc): the tap marker is retired, so no pusher can take this tap for a live rig"
+fi
 echo "run-qwen38-bf16 $MODE exited rc=$rc; $SERVICE restored; log: $LOG"
 exit "$rc"

serverAppConfigs-docs/LLM/frank-archon.md

@@ -562,6 +562,29 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
   fork bomb inside the container. Kill with `podman rm -f` (the cgroup owns every generation)
   never pattern-kill an interpreter name on this host; use `TORCH_LOGS` for dynamo questions.
 
+### The quantization pipeline
+
+- **A calibration tap is only a tap when the rig that opened it is running.** The runner writes
+  `.rig-boot` into the tap dir (container, image, pid) and retires it when its boot exits non-zero;
+  `calib-push.py` refuses without that marker, refuses when it names another container, and refuses
+  when the named container is not running; `qwen38-calib-window.sh` stops on a failed phase instead
+  of falling through its health wait. All three exist because on 2026-10-29 the unguarded chain's
+  rig boot died of an OOM, the health loop expired without failing, and the pusher fed four rounds
+  of 10M tokens into **restored production** — 86 minutes of live prefill that recorded nothing.
+- **The n-gram table's pin is what starves a rig boot, and the tier is a knob.** On the BF16 payload
+  the table is 95.37 GiB, `--ngram-placement ram` pins it, and the calibration boot died at
+  `startup stopped: out of memory` with 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available —
+  while the reference recording and every score boot succeeded at the same pool and slot count.
+  Calibration only needs the routed weights prefilled, so it reads the table from the container's
+  file tier (`--ngram-placement disk`); kldref and score keep production's placement, because their
+  numbers are only comparable at an identical shape.
+- **Acceptance is three metrics and every slice, not a mean.** `scripts/kld-gate-qwen38.sh` grades
+  mean KL, the p99.9 tail, top-1 agreement and every `by_source` slice (code, prose, tools):
+  a mean can hide a few positions taking the whole hit, and an overall win can be bought by
+  regressing one distribution. The FP8-native anchor is an interval, not a requirement — without
+  the 172.8 GiB fetch the gate runs in two-report mode (candidate vs the served payload) and says
+  so on its first line.
+
 ## Contract (the wire)
 
 - **Live surface:** `/health` (== `/ping`), `/v1/models`, `/v1/chat/completions`
@@ -638,14 +661,19 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
   rebuild), `scripts/download-frank-archon-swift15.sh` (the BF16 source a w4dn8 re-convert
   needs), `scripts/hfget.sh` / `scripts/hf-fetch.sh`; template
   `scripts/qwen38-froggeric-v22.5.jinja`.
-- Convert campaign (dormant, kept for a future quant): `scripts/convert-swift15-w4dn8.sh`
+- The campaign (qwen38-w5nl64, live): `scripts/run-qwen38-bf16.sh` (kldref / calib / score — the
-  `scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe`, `scripts/run-swift15-bf16.sh`
+  ONE measuring shape), `scripts/qwen38-calib-window.sh` (the gated calibration window),
-  `scripts/kld-gate-swift15.sh`, `scripts/kld-then-calib.sh`, `scripts/calib-push.py`
+  `scripts/convert-qwen38-w5nl64.sh` with
+  `scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe` (the arm driver and its template),
+  `scripts/kld-gate-qwen38.sh` (acceptance), `scripts/calib-push.py`,
   `scripts/make-calib-corpus.py`, `scripts/rechunk-kld-corpus.py`,
   `scripts/qwen38_common.py` (the tokenizer, JSONL and refusal plumbing those three
   corpus tools share — one `n_tokens`, so every tool counts the same way),
-  `scripts/derive-protected-experts.py`
+  `scripts/derive-protected-experts.py`, `scripts/apply-protected-cast.sh`.
-  `scripts/apply-protected-cast.sh`, `scripts/verify-swift15-ple.py`.
+- Retired swift15-era campaign scripts, superseded by the above and kept only as reference:
+  `scripts/convert-swift15-w4dn8.sh`, `scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe`,
+  `scripts/run-swift15-bf16.sh`, `scripts/kld-gate-swift15.sh`, `scripts/kld-then-calib.sh`,
+  `scripts/verify-swift15-ple.py`.
 - Probes: `scripts/probe-frank-archon-kv.py` (reads slots/window/dtype from the deployed env so
   it cannot drift; leg D = decode), `scripts/frank-archon-expert-activity.py`
   `scripts/autoresearch-frank-archon.sh` + `scripts/autoresearch/frank-archon-measure.py`.

tests/qwen38_campaign.bats

@@ -200,6 +200,9 @@ DERIVE="$REPO_ROOT/scripts/derive-protected-experts.py"
 APPLY="$REPO_ROOT/scripts/apply-protected-cast.sh"
 CONVERT="$REPO_ROOT/scripts/convert-qwen38-w5nl64.sh"
 GATE="$REPO_ROOT/scripts/kld-gate-qwen38.sh"
+RUNNER="$REPO_ROOT/scripts/run-qwen38-bf16.sh"
+PUSHER="$REPO_ROOT/scripts/calib-push.py"
+WINDOW="$REPO_ROOT/scripts/qwen38-calib-window.sh"
 TEMPLATE="$REPO_ROOT/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe"
 
 # make_gram <dir> <layer> <expert> <energy> <rows> — a RADGRAM2 down-Gram whose
@@ -525,3 +528,174 @@ gate_case() { # gate_case <kl fp8> <kl iq4r> <kl cand> <top1 cand> [extra flags.
     [[ "$output" == *"has no slice 'code'"* ]]
     [[ "$output" == *"it has: ops"* ]]
 }
+
+# --- the n-gram tier: the pin that starved the calibration boot ---------------
+
+@test "the calibration boot reads the n-gram table from the file tier" {
+    # 2026-10-29: 95.37 GiB pinned beside a 65.59 GiB/rank pool left 8.06 GiB of
+    # RAM and the boot died "startup stopped: out of memory". calib only needs the
+    # routed weights prefilled, so it reads the table off the container's file tier.
+    run bash "$RUNNER" calib --dry-run
+    [ "$status" -eq 0 ]
+    grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'disk'
+}
+
+@test "the recording and every score pass keep production's n-gram placement" {
+    # Those numbers are only comparable when the shape is identical; the tier is
+    # part of the shape.
+    run bash "$RUNNER" kldref --dry-run
+    [ "$status" -eq 0 ]
+    grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'ram'
+}
+
+@test "an unknown n-gram placement is refused by name before the boot" {
+    run env FRANK_QWEN38_NGRAM_PLACEMENT=ssd bash "$RUNNER" calib --dry-run
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"auto|vram|ram|disk"* ]]
+}
+
+# --- the pusher: a tap without a live rig is not a tap ------------------------
+
+# stub_podman <dir> <running|absent> — a podman that answers the one question the
+# pusher asks: is the rig container running?
+stub_podman() {
+    mkdir -p "$1/bin"
+    if [ "$2" = running ]; then
+        printf '#!/usr/bin/env bash\necho true\n' > "$1/bin/podman"
+    elif [ "$2" = stopped ]; then
+        printf '#!/usr/bin/env bash\necho false\n' > "$1/bin/podman"
+    else
+        printf '#!/usr/bin/env bash\nexit 125\n' > "$1/bin/podman"
+    fi
+    chmod +x "$1/bin/podman"
+}
+
+pusher_fixture() { # pusher_fixture <marker-container or empty>
+    D="$BATS_TEST_TMPDIR/push"
+    mkdir -p "$D/tap" "$D/bin"
+    printf '{"prompt":"hello"}\n' > "$D/corpus.jsonl"
+    if [ -n "$1" ]; then
+        printf 'container=%s\nimage=radiance@sha256:test\nmode=calib\npid=1\n' "$1" \
+            > "$D/tap/.rig-boot"
+    fi
+    echo "$D"
+}
+
+@test "the pusher refuses a tap that carries no rig marker: production is not a rig" {
+    # With no marker, the old driver fed four rounds of 10M tokens into restored
+    # production after the rig's boot had died, and recorded nothing.
+    D="$(pusher_fixture '')"
+    run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" --port 1
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"no rig marker"* ]]
+}
+
+@test "the pusher refuses a marker that names another container" {
+    D="$(pusher_fixture arbench)"
+    run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" \
+        --container qwen38-bf16 --port 1
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"names container 'arbench'"* ]]
+    [[ "$output" == *"targets 'qwen38-bf16'"* ]]
+}
+
+@test "the pusher refuses when the rig container is not running" {
+    D="$(pusher_fixture qwen38-bf16)"
+    stub_podman "$D" stopped
+    run env PATH="$D/bin:$PATH" python3 "$PUSHER" --corpus "$D/corpus.jsonl" \
+        --tap-dir "$D/tap" --container qwen38-bf16 --port 1
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"is not running"* ]]
+}
+
+# --- the acceptance gate without an FP8 anchor -------------------------------
+
+# kld_report_slices <path> <model> <kl all> <kl code> <top1 all> <top1 code>
+kld_report_slices() {
+    python3 - "$@" <<'PY'
+import json, sys
+path, model, kl_all, kl_code, t_all, t_code = sys.argv[1:7]
+def g(kl, t, n=25000):
+    return {"kld": {"mean": float(kl), "p99_9": float(kl) * 4},
+            "top1_agreement": float(t), "positions": n}
+json.dump({"model": model, "all": g(kl_all, t_all),
+           "by_source": {"code": g(kl_code, t_code), "ops": g(kl_all, t_all)}},
+          open(path, "w"))
+PY
+}
+
+gate2() { # gate2 <kl iq4r> <kl cand> <top1 iq4r> <top1 cand> [extra]
+    D="$BATS_TEST_TMPDIR/kl2"
+    mkdir -p "$D"
+    kld_report_slices "$D/iq4r.json" iq4r-moe "$1" "$1" "$3" "$3"
+    kld_report_slices "$D/cand.json" w5nl64 "$2" "$2" "$4" "$4"
+    run bash "$GATE" "$D/iq4r.json" "$D/cand.json" "${@:5}"
+}
+
+@test "a candidate that beats the served payload on mean, tail and top-1 passes without an anchor" {
+    gate2 0.0190 0.0090 0.962 0.971
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"candidate vs served (no FP8 anchor)"* ]]
+    [[ "$output" == *"KLD GATE PASS"* ]]
+}
+
+@test "a better mean with a worse tail fails: the tail is where a mean hides its damage" {
+    D="$BATS_TEST_TMPDIR/kl3"
+    mkdir -p "$D"
+    kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962
+    python3 - "$D/cand.json" <<'PY'
+import json, sys
+# mean better than the served payload, p99.9 worse: the shape a mean-only gate waves through
+g = {"kld": {"mean": 0.0090, "p99_9": 0.4000}, "top1_agreement": 0.971, "positions": 25000}
+json.dump({"model": "w5nl64", "all": g, "by_source": {"code": dict(g), "ops": dict(g)}},
+          open(sys.argv[1], "w"))
+PY
+    run bash "$GATE" "$D/iq4r.json" "$D/cand.json"
+    [ "$status" -eq 1 ]
+    [[ "$output" == *"[FAIL] 2."* ]]
+    [[ "$output" == *"KL p99.9"* ]]
+}
+
+@test "a candidate that wins overall but regresses one source slice fails" {
+    D="$BATS_TEST_TMPDIR/kl4"
+    mkdir -p "$D"
+    kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962
+    # 'all' is better, 'code' is worse: exactly the trade a mean-only gate cannot see
+    kld_report_slices "$D/cand.json" w5nl64 0.0090 0.0300 0.971 0.950
+    run bash "$GATE" "$D/iq4r.json" "$D/cand.json"
+    [ "$status" -eq 1 ]
+    [[ "$output" == *"[PASS] 1."* ]]
+    [[ "$output" == *"[FAIL] 4."* ]]
+    [[ "$output" == *"regressed: code"* ]]
+}
+
+# --- the calibration window: gated, and it says so ---------------------------
+
+window_fixture() {
+    D="$BATS_TEST_TMPDIR/win"
+    mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora"
+    echo '{}' > "$D/campaign/kld-ref/kld.json"
+    : > "$D/rad/qwen38-bf16.rad"
+    : > "$D/corpora/calib.jsonl"
+    echo "$D"
+}
+
+@test "the calibration window refuses before touching a card without a reference recording" {
+    D="$BATS_TEST_TMPDIR/win-less"
+    mkdir -p "$D/campaign/calib"
+    run env FRANK_QWEN38_CAMPAIGN="$D/campaign" bash "$WINDOW"
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"no reference recording"* ]]
+}
+
+@test "the calibration window's dry run prints the gated plan, including what it would bank" {
+    D="$(window_fixture)"
+    run env FRANK_QWEN38_CAMPAIGN="$D/campaign" \
+        FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \
+        Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" bash "$WINDOW" --dry-run
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"run-qwen38-bf16.sh calib"* ]]
+    [[ "$output" == *"else kill the rig"* ]]
+    [[ "$output" == *"calib-push.py"* ]]
+    [[ "$output" == *"AGENTTRASH"* ]]
+}