Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f6298e3501 | ||
|
|
a6fb0209e8 | ||
|
|
c238e32d6f |
No files matched your search
+77
-7
@@ -22,9 +22,17 @@ Mechanics notes:
|
||||
prompts do not bias the Hessians (the corpus is drawn round-robin over
|
||||
source classes, so loss is class-neutral as long as failures are rare;
|
||||
a systematic 4xx is loud in the log and must be fixed, not ignored).
|
||||
- a round that seals no tap bytes is a FAILURE, not a retry: the engine
|
||||
answering 200 while the tap directory is wrong, full or read-only is
|
||||
how a corpus gets pushed four times for nothing.
|
||||
- a round that seals no tap bytes is NORMAL and continues: RADIANCE_CALIB_TOKENS
|
||||
is a pooled floor and this campaign's corpus is 10.03M tokens against a 12M
|
||||
floor, so round 1 seals nothing by construction. A tap that LOSES bytes fails
|
||||
the run, and a run that never sees the engine's completion line exits non-zero.
|
||||
The first version failed on a flat round, which aborted exactly one round too
|
||||
early (2026-10-29).
|
||||
- before any prompt, the tap must carry the rig's own marker
|
||||
(`--tap-marker`, written by run-qwen38-bf16.sh calib) naming THIS
|
||||
container, and that container must be running. Measured 2026-10-29:
|
||||
with neither check, four rounds of 10M tokens went into restored
|
||||
production after the rig's boot had died, and recorded nothing.
|
||||
|
||||
Usage: calib-push.py [--port 7770] [--parallel 64] [--container arbench]
|
||||
[--corpus /home/dank/corpora/out/calib.jsonl] [--max-rounds 4]
|
||||
@@ -107,6 +115,61 @@ def tap_bytes(path):
|
||||
return total
|
||||
|
||||
|
||||
def round_verdict(before, after, done):
|
||||
"""What one round's tap bytes mean.
|
||||
|
||||
'done' the engine logged that it sealed the tap — the run is over
|
||||
'shrink' the tap LOST bytes: reset, pruned or replaced under us
|
||||
'grew' sealed bytes this round
|
||||
'flat' sealed nothing — which is legitimate: RADIANCE_CALIB_TOKENS is a
|
||||
pooled floor, and a round that does not reach it has nothing to
|
||||
write yet (the campaign's corpus is 10.03M tokens against a 12M
|
||||
floor, so round 1 is always flat). Failing on it aborted the run
|
||||
exactly one round too early.
|
||||
"""
|
||||
if done:
|
||||
return "done"
|
||||
if after < before:
|
||||
return "shrink"
|
||||
if after == before:
|
||||
return "flat"
|
||||
return "grew"
|
||||
|
||||
|
||||
def require_rig(a):
|
||||
"""Refuse unless the rig that opened this tap is up and serving right now.
|
||||
|
||||
Three facts, in order: the runner wrote its marker into the tap dir, the
|
||||
marker names the container this pusher targets, and that container is
|
||||
running. A marker without a live container is a boot that died; a live
|
||||
container without a marker is some other engine on the port — on 2026-10-29
|
||||
it was restored production, which took four rounds of 10M tokens before the
|
||||
old driver admitted it had recorded nothing.
|
||||
"""
|
||||
marker = os.path.join(a.tap_dir, a.tap_marker)
|
||||
if not os.path.isfile(marker):
|
||||
qc.refuse(f"no rig marker at {marker}: this tap was not opened by "
|
||||
"run-qwen38-bf16.sh calib, so a push here would feed whatever "
|
||||
"answers the port and record nothing")
|
||||
try:
|
||||
lines = open(marker, encoding="utf-8", errors="replace").read().splitlines()
|
||||
except OSError as exc:
|
||||
qc.refuse(f"cannot read the rig marker {marker}: {exc}")
|
||||
meta = dict(line.split("=", 1) for line in lines if "=" in line)
|
||||
who = meta.get("container", "")
|
||||
if who != a.container:
|
||||
qc.refuse(f"the rig marker names container {who!r} and this pusher targets "
|
||||
f"{a.container!r} — wrong tap, or a stale marker")
|
||||
try:
|
||||
p = subprocess.run(["podman", "inspect", "-f", "{{.State.Running}}", a.container],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
except Exception as exc:
|
||||
qc.refuse(f"cannot inspect the rig container {a.container}: {exc}")
|
||||
if p.returncode != 0 or p.stdout.strip() != "true":
|
||||
qc.refuse(f"the rig container {a.container} is not running — whatever is "
|
||||
"answering the port, it is not the calibration rig")
|
||||
|
||||
|
||||
def parse_args():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=7770)
|
||||
@@ -114,6 +177,8 @@ def parse_args():
|
||||
ap.add_argument("--container", default="arbench")
|
||||
ap.add_argument("--model", default="frank-archon")
|
||||
ap.add_argument("--tap-dir", default="/home/dank/campaign/qwen38-w5nl64/calib")
|
||||
ap.add_argument("--tap-marker", default=".rig-boot",
|
||||
help="the file run-qwen38-bf16.sh calib writes into the tap dir; without it the pusher refuses, because an unmarked tap is not a rig")
|
||||
ap.add_argument("--corpus", default="/home/dank/corpora/out/calib.jsonl")
|
||||
ap.add_argument("--max-rounds", type=int, default=4)
|
||||
return ap.parse_args()
|
||||
@@ -136,6 +201,7 @@ def main():
|
||||
a = parse_args()
|
||||
url = f"http://127.0.0.1:{a.port}/v1/completions"
|
||||
prompts = load_prompts(a)
|
||||
require_rig(a)
|
||||
chars = sum(len(p) for p in prompts)
|
||||
print(f"corpus: {len(prompts)} prompts, ~{chars // 4} tokens (chars/4)",
|
||||
flush=True)
|
||||
@@ -156,11 +222,15 @@ def main():
|
||||
if tap_done(a.container):
|
||||
break
|
||||
after = tap_bytes(a.tap_dir)
|
||||
if after <= before:
|
||||
print(f"FAIL: the tap at {a.tap_dir} sealed nothing this round "
|
||||
f"({before} B before, {after} B after) — the engine is serving "
|
||||
"without recording (wrong dir, full disk, read-only mount)", flush=True)
|
||||
verdict = round_verdict(before, after, False)
|
||||
if verdict == "shrink":
|
||||
print(f"FAIL: the tap at {a.tap_dir} LOST bytes this round "
|
||||
f"({before} B before, {after} B after) — it was reset or replaced "
|
||||
"under us", flush=True)
|
||||
sys.exit(1)
|
||||
if verdict == "flat":
|
||||
print(f"round {rnd}: the tap sealed nothing yet (byte floor not reached; "
|
||||
f"{before} B) — continuing", flush=True)
|
||||
else:
|
||||
print("max rounds exhausted, tap not done", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
+114
-50
@@ -1,44 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
# kld-gate-qwen38.sh — RUNS ON DANKSERVER (or anywhere the reports are). The campaign's
|
||||
# acceptance gate: three KLD reports against ONE reference recording, one verdict.
|
||||
# acceptance gate: one verdict from the KLD reports of the containers that ride one reference
|
||||
# recording.
|
||||
#
|
||||
# The reference is the BF16 container built from the vendor checkpoint (run-qwen38-bf16.sh
|
||||
# kldref, in the same drained window that records the calibration grams). Every scored
|
||||
# container is measured against that one recording, so the KL scale runs from 0 = the
|
||||
# checkpoint itself downward in faithfulness. Three containers ride it:
|
||||
# checkpoint itself downward in faithfulness.
|
||||
#
|
||||
# fp8.json the published FP8-native release — the achievable end of the interval
|
||||
# iq4r.json the payload frank-archon serves — the bar to beat
|
||||
# cand.json the arm under test — the subject
|
||||
# TWO MODES, because the disk decides how many anchors can exist at once:
|
||||
#
|
||||
# The FP8 number is what makes the interval absolute instead of relative: "half the gap"
|
||||
# is half of the distance between the 4-bit payload and the near-lossless release, not half
|
||||
# of an invented distance to the checkpoint. That is also why the FP8 payload is FETCHED and
|
||||
# scored, never converted: it is a second opinion about what this checkpoint tolerates, not a
|
||||
# step of the campaign. It lands LAST (disk sequencing) — by then the 335 GiB bf16 container
|
||||
# is gone, so its 172.8 GiB fits.
|
||||
# gate IQ4R.json CAND.json candidate vs the served payload
|
||||
# gate FP8.json IQ4R.json CAND.json the same, inside the absolute interval
|
||||
#
|
||||
# The law, in order of severity, on the mean KL of one slice (--slice all | a corpus source):
|
||||
# 1. KL(fp8) < KL(iq4r) the anchors are ordered. If the near-lossless
|
||||
# release does NOT sit closer to the checkpoint than
|
||||
# the 4-bit payload, the corpus, the tokenizer or the
|
||||
# boot shape is wrong and no arm number means anything.
|
||||
# 2. KL(fp8) < KL(cand) < KL(iq4r) the candidate lies strictly inside the interval —
|
||||
# outside it, something is mis-measured, not better
|
||||
# 3. KL(cand) <= KL(iq4r) - R x gap it closes at least R (--recover, default 0.5) of the
|
||||
# gap = KL(iq4r) - KL(fp8): at least half the way from
|
||||
# what is served toward the release (agreed 2026-10-27)
|
||||
# 4. top-1(cand) >= top-1(iq4r) the agreement the harness keys on does not fall
|
||||
# iq4r.json is the payload frank-archon serves — the bar to beat. When the FP8-native release
|
||||
# (172.8 GiB, fetched last because of the peak-disk law) is on disk, pass its report first:
|
||||
# "half the gap" is then half of the distance between the 4-bit payload and the near-lossless
|
||||
# release, not half of an invented distance to the checkpoint. Without it the gate still
|
||||
# answers the campaign's question — does the candidate beat what is served — and says so.
|
||||
#
|
||||
# The law, on the summary of one slice (--slice all | a corpus source):
|
||||
#
|
||||
# candidate vs served (2 reports)
|
||||
# 1. mean KL(cand) < mean KL(iq4r) the served payload is beaten on the corpus
|
||||
# 2. p99.9(cand) <= p99.9(iq4r) and the tail is no worse — a mean can hide a few
|
||||
# positions taking the whole hit
|
||||
# 3. top-1(cand) >= top-1(iq4r) the agreement the harness keys on does not fall
|
||||
# 4. every source slice passes 1-3 a win bought by regressing one distribution
|
||||
# (code, prose, tools) is not a win
|
||||
#
|
||||
# inside the interval (3 reports), laws 1-4 above (1-3 renumbered to the interval and the
|
||||
# recovery fraction, as agreed 2026-10-27) plus:
|
||||
# 5. p99.9(cand) <= p99.9(iq4r) the tail law
|
||||
# 6. every source slice passes the slice law
|
||||
#
|
||||
# Booting the containers is NOT this script's job: `run-qwen38-bf16.sh score RAD OUT.json`
|
||||
# writes the reports and owns the single engine argv (same pin, --tp 2, --kv-cache-dtype fp8,
|
||||
# same step size, --deterministic, no speculation) that makes the three numbers comparable.
|
||||
# same step size, --deterministic, no speculation) that makes the numbers comparable.
|
||||
# This script is arithmetic, so it re-runs against old reports and is testable without a GPU.
|
||||
#
|
||||
# Usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]
|
||||
# Usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]
|
||||
# kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]
|
||||
set -uo pipefail
|
||||
|
||||
[ "$#" -ge 3 ] || { echo "usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]" >&2; exit 2; }
|
||||
if [ "$#" -lt 2 ]; then
|
||||
echo "usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]" >&2
|
||||
echo " kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]" >&2
|
||||
exit 2
|
||||
fi
|
||||
python3 - "$@" <<'PY'
|
||||
import json, sys
|
||||
|
||||
@@ -52,13 +61,15 @@ while i < len(a):
|
||||
else:
|
||||
files.append(a[i])
|
||||
i += 1
|
||||
if len(files) != 3:
|
||||
sys.exit("usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]")
|
||||
if len(files) not in (2, 3):
|
||||
sys.exit("usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]\n"
|
||||
" kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]")
|
||||
o = dict(zip(opts[0::2], opts[1::2]))
|
||||
slice_ = o.get("--slice", "all")
|
||||
recover = float(o.get("--recover", 0.5))
|
||||
if not 0.0 < recover <= 1.0:
|
||||
sys.exit(f"--recover must be in (0,1], got {recover}")
|
||||
interval = len(files) == 3
|
||||
|
||||
|
||||
def die(msg):
|
||||
@@ -66,11 +77,15 @@ def die(msg):
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
def read(path):
|
||||
def load(path):
|
||||
try:
|
||||
d = json.load(open(path))
|
||||
return json.load(open(path))
|
||||
except Exception as e:
|
||||
die(f"cannot read the KLD report {path}: {e}")
|
||||
|
||||
|
||||
def summary(path, d):
|
||||
"""The slice this gate grades: 'all', or one corpus source."""
|
||||
if slice_ == "all":
|
||||
g = d.get("all")
|
||||
else:
|
||||
@@ -80,23 +95,25 @@ def read(path):
|
||||
f"{', '.join(sorted(d.get('by_source') or {})) or '(none)'}")
|
||||
if g is None or "kld" not in g or "top1_agreement" not in g:
|
||||
die(f"{path} has no {slice_!r} summary — wrong report version?")
|
||||
return d, g
|
||||
return g
|
||||
|
||||
|
||||
docs = {label: load(path) for label, path in
|
||||
zip(("fp8", "iq4r", "cand") if interval else ("iq4r", "cand"), files)}
|
||||
rows = []
|
||||
for label, path in zip(("fp8", "iq4r", "cand"), files):
|
||||
d, g = read(path)
|
||||
rows.append((label, path, d.get("model", "?"), g["kld"]["mean"], g["kld"]["p99_9"],
|
||||
g["top1_agreement"], g["positions"]))
|
||||
for label, path in zip(("fp8", "iq4r", "cand") if interval else ("iq4r", "cand"), files):
|
||||
g = summary(path, docs[label])
|
||||
rows.append((label, path, docs[label].get("model", "?"), g["kld"]["mean"],
|
||||
g["kld"]["p99_9"], g["top1_agreement"], g["positions"]))
|
||||
|
||||
print(f"reference: BF16 container; slice: {slice_}")
|
||||
print(f"reference: BF16 container; slice: {slice_}; "
|
||||
+ ("inside the FP8 interval" if interval else "candidate vs served (no FP8 anchor)"))
|
||||
print(f"{'':6} {'model':38} {'KL mean':>10} {'KL p99.9':>10} {'top-1':>8} positions")
|
||||
for label, path, model, kl, p999, top1, n in rows:
|
||||
print(f"{label:6} {str(model)[:38]:38} {kl:10.6f} {p999:10.4f} {100*top1:7.2f}% {n}")
|
||||
r = {label: (kl, top1) for label, _p, _m, kl, _q, top1, _n in rows}
|
||||
kl_fp8, kl_iq4r, kl_cand = r["fp8"][0], r["iq4r"][0], r["cand"][0]
|
||||
t_iq4r, t_cand = r["iq4r"][1], r["cand"][1]
|
||||
gap = kl_iq4r - kl_fp8
|
||||
r = {label: (kl, p999, top1) for label, _p, _m, kl, p999, top1, _n in rows}
|
||||
kl_cand, p_cand, t_cand = r["cand"]
|
||||
kl_iq4r, p_iq4r, t_iq4r = r["iq4r"]
|
||||
ok = True
|
||||
|
||||
|
||||
@@ -106,17 +123,64 @@ def law(n, text, passed):
|
||||
print(f" [{'PASS' if passed else 'FAIL'}] {n}. {text}")
|
||||
|
||||
|
||||
law(1, f"anchors ordered: fp8 {kl_fp8:.6f} < iq4r {kl_iq4r:.6f} (gap {gap:.6f})",
|
||||
gap > 0)
|
||||
law(2, f"candidate inside the interval: {kl_fp8:.6f} < {kl_cand:.6f} < {kl_iq4r:.6f}",
|
||||
kl_fp8 < kl_cand < kl_iq4r)
|
||||
need = kl_iq4r - recover * gap
|
||||
law(3, f"candidate closes {100*max((kl_iq4r - kl_cand)/gap, 0):.1f}% of the "
|
||||
f"{gap:.6f} gap (needs {100*recover:.0f}%, i.e. KL <= {need:.6f})"
|
||||
if gap > 0 else "gap is not positive; cannot score recovery",
|
||||
gap > 0 and kl_cand <= need)
|
||||
law(4, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%",
|
||||
t_cand >= t_iq4r)
|
||||
def tail_law(n):
|
||||
law(n, f"candidate KL p99.9 {p_cand:.4f} <= served iq4r {p_iq4r:.4f}",
|
||||
p_cand <= p_iq4r)
|
||||
|
||||
|
||||
# --- the slice law: every corpus source must pass the metric laws -------------
|
||||
def slice_law(n):
|
||||
"""A win that regresses one source class is not a win: check them all."""
|
||||
mine = docs["cand"].get("by_source") or {}
|
||||
theirs = docs["iq4r"].get("by_source") or {}
|
||||
if not mine or not theirs:
|
||||
law(n, "no per-source summaries to check", True)
|
||||
return
|
||||
missing = sorted(set(theirs) - set(mine))
|
||||
extra = sorted(set(mine) - set(theirs))
|
||||
if missing or extra:
|
||||
law(n, "same source slices in both reports"
|
||||
+ (f" (missing from cand: {', '.join(missing)})" if missing else "")
|
||||
+ (f" (only in cand: {', '.join(extra)})" if extra else ""), False)
|
||||
return
|
||||
bad = []
|
||||
for src in sorted(theirs):
|
||||
c, q = mine[src], theirs[src]
|
||||
worse_mean = c["kld"]["mean"] >= q["kld"]["mean"]
|
||||
worse_tail = c["kld"]["p99_9"] > q["kld"]["p99_9"]
|
||||
worse_top1 = c["top1_agreement"] < q["top1_agreement"]
|
||||
if worse_mean or worse_tail or worse_top1:
|
||||
why = ",".join(m for m, w in (("mean", worse_mean), ("p99.9", worse_tail),
|
||||
("top-1", worse_top1)) if w)
|
||||
bad.append(f"{src} ({why})")
|
||||
law(n, "every source slice passes (mean, p99.9, top-1)"
|
||||
+ (f" — regressed: {'; '.join(bad)}" if bad else ""), not bad)
|
||||
|
||||
|
||||
if interval:
|
||||
kl_fp8 = r["fp8"][0]
|
||||
gap = kl_iq4r - kl_fp8
|
||||
law(1, f"anchors ordered: fp8 {kl_fp8:.6f} < iq4r {kl_iq4r:.6f} (gap {gap:.6f})",
|
||||
gap > 0)
|
||||
law(2, f"candidate inside the interval: {kl_fp8:.6f} < {kl_cand:.6f} < {kl_iq4r:.6f}",
|
||||
kl_fp8 < kl_cand < kl_iq4r)
|
||||
need = kl_iq4r - recover * gap
|
||||
law(3, f"candidate closes {100*max((kl_iq4r - kl_cand)/gap, 0):.1f}% of the "
|
||||
f"{gap:.6f} gap (needs {100*recover:.0f}%, i.e. KL <= {need:.6f})"
|
||||
if gap > 0 else "gap is not positive; cannot score recovery",
|
||||
gap > 0 and kl_cand <= need)
|
||||
law(4, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%",
|
||||
t_cand >= t_iq4r)
|
||||
tail_law(5)
|
||||
slice_law(6)
|
||||
else:
|
||||
law(1, f"candidate beats the served payload: KL {kl_cand:.6f} < iq4r {kl_iq4r:.6f}",
|
||||
kl_cand < kl_iq4r)
|
||||
tail_law(2)
|
||||
law(3, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%",
|
||||
t_cand >= t_iq4r)
|
||||
slice_law(4)
|
||||
|
||||
print("KLD GATE " + ("PASS" if ok else "FAIL"))
|
||||
sys.exit(0 if ok else 1)
|
||||
PY
|
||||
Executable
+122
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env bash
|
||||
# qwen38-calib-window.sh — RUNS ON DANKSERVER. The calibration window, and nothing else:
|
||||
# boot the BF16 rig, wait until it is actually serving, push the corpus until the engine says
|
||||
# the tap is full, stop the rig, restore frank-archon, and bank the reference container.
|
||||
#
|
||||
# The reference recording is already taken (kld-ref/kld.json exists), so this window is the
|
||||
# calibration tap alone — the one job that needs both cards after it. The kldref phase is NOT
|
||||
# re-run here: the chained launcher that used to do that moved the banked kld-ref into trash
|
||||
# first, which is how a 16-hour phase gets paid for twice.
|
||||
#
|
||||
# Every phase is gated on the previous one, because the last time this ran unguarded
|
||||
# (2026-10-29) the rig's boot died of an OOM, the health loop quietly fell through, and the
|
||||
# pusher fed four rounds of 10M tokens into RESTORED PRODUCTION with nothing recorded:
|
||||
#
|
||||
# 1. the runner must still be alive and its container must answer /health, or this script
|
||||
# kills it, restores the service and exits non-zero — no push, ever;
|
||||
# 2. the pusher refuses on its own unless the rig wrote its marker into the tap dir, the
|
||||
# marker names this container, and the container is running (scripts/calib-push.py
|
||||
# --tap-marker), so a second lock survives a mistake in this file;
|
||||
# 3. the reference container is banked into trash only when BOTH the serve and the push
|
||||
# exited 0 and the tap dir holds grams. The checkpoint that reproduces it stays.
|
||||
#
|
||||
# knobs: Q38_WINDOW_CALIB_CORPUS (calib-7168.jsonl — the re-chunked file the rig's pool can
|
||||
# hold), Q38_WINDOW_HEALTH_TIMEOUT (seconds to wait for /health), Q38_WINDOW_ROUNDS
|
||||
# (pusher rounds; one is 10M tokens), Q38_WINDOW_KEEP_RAD=1 to leave the .rad in place.
|
||||
# Usage: scripts/qwen38-calib-window.sh [--dry-run]
|
||||
set -uo pipefail
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 9
|
||||
|
||||
DRY=0
|
||||
[ "${1:-}" = "--dry-run" ] && DRY=1
|
||||
say() { printf '%s %s\n' "$(date -Is)" "$*"; }
|
||||
|
||||
MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
|
||||
CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
|
||||
RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}"
|
||||
CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}"
|
||||
KLDREF="$CAMPAIGN/kld-ref/kld.json"
|
||||
CALIB="$CAMPAIGN/calib"
|
||||
HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}"
|
||||
ROUNDS="${Q38_WINDOW_ROUNDS:-6}"
|
||||
RIG=qwen38-bf16
|
||||
SERVICE=frank-archon
|
||||
PY="${PY:-python3}"
|
||||
|
||||
# ---- preflight: everything this window needs, before a card is touched -------
|
||||
[ -f "$KLDREF" ] || { echo "REFUSED: no reference recording at $KLDREF — record it first (scripts/run-qwen38-bf16.sh kldref); this window only fills the calibration tap" >&2; exit 2; }
|
||||
[ -f "$RAD" ] || { echo "REFUSED: no BF16 reference container at $RAD — nothing to serve; scripts/build-qwen38-bf16-rad.sh" >&2; exit 2; }
|
||||
[ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; }
|
||||
[ -d "$CALIB" ] || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; }
|
||||
[ -w "$CALIB" ] || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; }
|
||||
! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; }
|
||||
! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; }
|
||||
|
||||
if [ "$DRY" = 1 ]; then
|
||||
echo "dry run: calib window — the service is left running, nothing is mounted or written"
|
||||
echo "would run: bash scripts/run-qwen38-bf16.sh calib (rig boots, writes $CALIB/.rig-boot)"
|
||||
echo "would wait: /health up within ${HEALTH_TIMEOUT}s, else kill the rig, restore $SERVICE, exit 1"
|
||||
echo "would run: $PY scripts/calib-push.py --container $RIG --corpus $CORPUS --max-rounds $ROUNDS"
|
||||
echo "would bank: $RAD -> ~/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad/ (only if both phases exit 0)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
say "calib window start pid=$$ corpus=$CORPUS rounds=$ROUNDS"
|
||||
[ -n "$(ls -A "$CALIB" 2>/dev/null)" ] && say "tap dir $CALIB is not empty; the engine's token floor is pooled, so an earlier partial tap still counts"
|
||||
|
||||
# ---- phase 1: the rig -------------------------------------------------------
|
||||
say "phase=calib serve begin"
|
||||
bash scripts/run-qwen38-bf16.sh calib & spid=$!
|
||||
|
||||
rig_up() {
|
||||
podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true \
|
||||
&& curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health
|
||||
}
|
||||
|
||||
waited=0
|
||||
until rig_up; do
|
||||
if ! kill -0 "$spid" 2>/dev/null; then
|
||||
wait "$spid"; crc=$?
|
||||
say "phase=calib serve end rc=$crc (the rig exited before it ever answered /health)"
|
||||
echo "FAIL: the rig boot died (rc=$crc) — nothing was pushed; read /home/dank/logs/qwen38_bf16_calib.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$waited" -ge "$HEALTH_TIMEOUT" ]; then
|
||||
say "phase=calib health never came after ${HEALTH_TIMEOUT}s; killing the rig"
|
||||
podman stop "$RIG" >/dev/null 2>&1 || true
|
||||
wait "$spid"
|
||||
echo "FAIL: the rig never answered /health within ${HEALTH_TIMEOUT}s — nothing was pushed" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 15
|
||||
waited=$((waited + 15))
|
||||
done
|
||||
say "phase=calib health ok waited=${waited}s"
|
||||
|
||||
# ---- phase 2: the tap -------------------------------------------------------
|
||||
say "phase=calib push begin"
|
||||
"$PY" scripts/calib-push.py --port 7770 --parallel 8 --container "$RIG" \
|
||||
--tap-dir "$CALIB" --corpus "$CORPUS" --max-rounds "$ROUNDS"
|
||||
prc=$?
|
||||
say "phase=calib push end rc=$prc"
|
||||
|
||||
podman stop "$RIG" >/dev/null 2>&1 || true
|
||||
wait "$spid"; crc=$?
|
||||
say "phase=calib serve end rc=$crc"
|
||||
|
||||
if [ "$prc" -ne 0 ] || [ "$crc" -ne 0 ]; then
|
||||
echo "FAIL: serve rc=$crc push rc=$prc — the tap is NOT banked and $RAD stays: $(basename "$RAD")" >&2
|
||||
exit 1
|
||||
fi
|
||||
[ -n "$(ls -A "$CALIB" 2>/dev/null)" ] || { echo "FAIL: both phases exited 0 but the tap dir $CALIB is empty" >&2; exit 1; }
|
||||
say "TAP BANKED: $(ls -1 "$CALIB" | wc -l) file(s) in $CALIB"
|
||||
|
||||
# ---- phase 3: bank the reference container ----------------------------------
|
||||
if [ "${Q38_WINDOW_KEEP_RAD:-0}" = "1" ]; then
|
||||
say "reference .rad kept (Q38_WINDOW_KEEP_RAD=1)"
|
||||
else
|
||||
T="$HOME/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad"
|
||||
mkdir -p "$T" || { echo "FAIL: cannot create trash dir $T" >&2; exit 1; }
|
||||
mv "$RAD" "$T/" && say "reference .rad banked in $T (the checkpoint that rebuilds it stays)"
|
||||
fi
|
||||
say "calib window end"
|
||||
@@ -92,7 +92,8 @@
|
||||
# the container reads through /models), FRANK_QWEN38_LOG_DIR, and the step
|
||||
# and slot counts inherited as FRANK_ARCHON_MAX_BATCHED_TOKENS /
|
||||
# FRANK_ARCHON_MAX_SEQS — no per-mode literals (the audit law: a mode that
|
||||
# carries its own step measures a shape nothing serves) —
|
||||
# carries its own step measures a shape nothing serves; calib is exempt, see
|
||||
# Q38_CALIB_MNBT / Q38_CALIB_SEQS below — its output is never compared) —
|
||||
# and FRANK_QWEN38_MAX_MODEL_LEN — the window every mode boots at (32,768;
|
||||
# see the corpus census above, and raise it together with the corpus).
|
||||
# kldref refuses (exit 2) when the longest record of the corpus it is about to
|
||||
@@ -185,6 +186,24 @@ if ! [[ "$MNBT" =~ ^[0-9]+$ ]]; then
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Calib's shape is NOT a measuring shape. kldref and score produce KL numbers that are only
|
||||
# comparable across boots at an identical shape, which is why they inherit production's step and
|
||||
# slots to the letter. The tap's output is layer Hessians, never a number any report compares, so
|
||||
# its step and slot count are its own — and they are the tap's dominant cost: a 7,168-token
|
||||
# document cut into four 2,048-token steps re-streams each layer's routed expert union four
|
||||
# times from the container, so one wide step is worth up to ~4x the read volume. Measured
|
||||
# 2026-10-29 at the inherited 2,048/4: prefill 71 tok/s and a 47-hour tap; the probe exists to
|
||||
# find out what the wide step is worth before the campaign pays for two days of drained serving.
|
||||
# Both default to the inherited value, so nothing moves unless a probe asks it to.
|
||||
CALIB_MNBT="${Q38_CALIB_MNBT:-$MNBT}"
|
||||
CALIB_SEQS="${Q38_CALIB_SEQS:-$MAX_SEQS}"
|
||||
for pair in "Q38_CALIB_MNBT=$CALIB_MNBT" "Q38_CALIB_SEQS=$CALIB_SEQS"; do
|
||||
if ! [[ "${pair#*=}" =~ ^[0-9]+$ ]]; then
|
||||
echo "REFUSED: ${pair%%=*} must be a number (got: ${pair#*=})" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
# The window every mode boots at. Over-length is a REFUSAL, not a truncation (the
|
||||
# env file says so for production), so this is the number the corpus must fit in.
|
||||
MAX_MODEL_LEN="${FRANK_QWEN38_MAX_MODEL_LEN:-32768}"
|
||||
@@ -193,6 +212,25 @@ if ! [[ "$MAX_MODEL_LEN" =~ ^[0-9]+$ ]]; then
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Where the n-gram table lives. On the bf16 payload it is 95.37 GiB and `ram`
|
||||
# PINS it: the calibration boot died at "startup stopped: out of memory" with
|
||||
# 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available (measured 2026-10-29),
|
||||
# while the reference recording and every score boot had already succeeded with
|
||||
# the same pool and slots. The difference that matters is the job: calib only
|
||||
# needs the routed weights prefilled, so it reads the table from the container's
|
||||
# own file tier and the pin never happens; kldref and score keep production's
|
||||
# placement, because their numbers are only comparable when every other term of
|
||||
# the shape is identical. FRANK_QWEN38_NGRAM_PLACEMENT overrides either default.
|
||||
case "$MODE" in
|
||||
calib) NGRAM_PLACEMENT="${FRANK_QWEN38_NGRAM_PLACEMENT:-disk}" ;;
|
||||
*) NGRAM_PLACEMENT="${FRANK_QWEN38_NGRAM_PLACEMENT:-ram}" ;;
|
||||
esac
|
||||
case "$NGRAM_PLACEMENT" in
|
||||
auto|vram|ram|disk) ;;
|
||||
*) echo "REFUSED: FRANK_QWEN38_NGRAM_PLACEMENT must be auto|vram|ram|disk (got: $NGRAM_PLACEMENT)" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
|
||||
# Every container this campaign boots is read through the ro /models mount, so
|
||||
# the host path must live under it; the container-side spelling is what goes
|
||||
# into the argv. Same rule for the container a score pass opens.
|
||||
@@ -230,7 +268,7 @@ ARGS=(--tp 2 --placement expert_tiered --weights-disk-tier
|
||||
--host-pool-mib "$HOST_POOL_MIB" --gpu-headroom-mib "$GPU_HEADROOM_MIB"
|
||||
--deterministic --num-speculative-tokens 0
|
||||
--kv-cache-dtype fp8 --max-model-len "$MAX_MODEL_LEN"
|
||||
--ngram-placement ram
|
||||
--ngram-placement "$NGRAM_PLACEMENT"
|
||||
--override-chat-template /models/tpl.jinja)
|
||||
MOUNTS=(-v "$MODELS:/models:ro")
|
||||
TAPENV=()
|
||||
@@ -277,7 +315,7 @@ case "$MODE" in
|
||||
# VRAM knob. The slots follow frank-archon.env's FRANK_ARCHON_MAX_SEQS
|
||||
# through MAX_SEQS, exactly as kldref and score do.
|
||||
ARGS+=(--model "$MODEL_IN" --port 7770
|
||||
--max-num-seqs "$MAX_SEQS" --max-num-batched-tokens "$MNBT")
|
||||
--max-num-seqs "$CALIB_SEQS" --max-num-batched-tokens "$CALIB_MNBT")
|
||||
# The tap is only real if this directory exists and the engine can write it:
|
||||
# without the check a typo'd CALIB_DIR auto-creates a root-owned empty dir in
|
||||
# whatever path was given, and the run burns hours serving prompts into a tap
|
||||
@@ -289,6 +327,17 @@ case "$MODE" in
|
||||
MOUNTS+=(-v "$CALIB_DIR:/data/calib")
|
||||
TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib
|
||||
-e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-12000000}")
|
||||
# The tap's own authorization. A pusher must see this marker before it sends
|
||||
# a prompt (calib-push.py --tap-marker), it names the container so a push
|
||||
# aimed anywhere else refuses, and a boot that fails removes it again below.
|
||||
# Measured 2026-10-29: with no marker, the chain pushed 4 rounds x 10M tokens
|
||||
# into RESTORED PRODUCTION after the rig's boot died, and nothing recorded.
|
||||
if [ "$DRY" = 1 ]; then
|
||||
:
|
||||
else
|
||||
printf 'container=%s\nimage=%s\nmode=calib\nstarted=%s\npid=%s\n' \
|
||||
"$C" "$IMAGE" "$(date -Is)" "$$" > "$CALIB_DIR/.rig-boot"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -377,5 +426,12 @@ podman rm -f "$C" >/dev/null 2>&1 || true
|
||||
|
||||
"${CMD[@]}" 2>&1 | tee "$LOG"
|
||||
rc=${PIPESTATUS[0]}
|
||||
# A boot that failed must not leave the tap marked as rig-authorized: the marker
|
||||
# is what calib-push.py trusts, and the point of it is that a dead rig can never
|
||||
# be pushed into again.
|
||||
if [ "$MODE" = calib ] && [ "$rc" -ne 0 ] && [ -f "$CALIB_DIR/.rig-boot" ]; then
|
||||
mv -f "$CALIB_DIR/.rig-boot" "$CALIB_DIR/.rig-boot.failed-$(date +%Y%m%d-%H%M%S)" 2>/dev/null || true
|
||||
echo "the rig boot failed (rc=$rc): the tap marker is retired, so no pusher can take this tap for a live rig"
|
||||
fi
|
||||
echo "run-qwen38-bf16 $MODE exited rc=$rc; $SERVICE restored; log: $LOG"
|
||||
exit "$rc"
|
||||
@@ -562,6 +562,29 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
|
||||
fork bomb inside the container. Kill with `podman rm -f` (the cgroup owns every generation)
|
||||
never pattern-kill an interpreter name on this host; use `TORCH_LOGS` for dynamo questions.
|
||||
|
||||
### The quantization pipeline
|
||||
|
||||
- **A calibration tap is only a tap when the rig that opened it is running.** The runner writes
|
||||
`.rig-boot` into the tap dir (container, image, pid) and retires it when its boot exits non-zero;
|
||||
`calib-push.py` refuses without that marker, refuses when it names another container, and refuses
|
||||
when the named container is not running; `qwen38-calib-window.sh` stops on a failed phase instead
|
||||
of falling through its health wait. All three exist because on 2026-10-29 the unguarded chain's
|
||||
rig boot died of an OOM, the health loop expired without failing, and the pusher fed four rounds
|
||||
of 10M tokens into **restored production** — 86 minutes of live prefill that recorded nothing.
|
||||
- **The n-gram table's pin is what starves a rig boot, and the tier is a knob.** On the BF16 payload
|
||||
the table is 95.37 GiB, `--ngram-placement ram` pins it, and the calibration boot died at
|
||||
`startup stopped: out of memory` with 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available —
|
||||
while the reference recording and every score boot succeeded at the same pool and slot count.
|
||||
Calibration only needs the routed weights prefilled, so it reads the table from the container's
|
||||
file tier (`--ngram-placement disk`); kldref and score keep production's placement, because their
|
||||
numbers are only comparable at an identical shape.
|
||||
- **Acceptance is three metrics and every slice, not a mean.** `scripts/kld-gate-qwen38.sh` grades
|
||||
mean KL, the p99.9 tail, top-1 agreement and every `by_source` slice (code, prose, tools):
|
||||
a mean can hide a few positions taking the whole hit, and an overall win can be bought by
|
||||
regressing one distribution. The FP8-native anchor is an interval, not a requirement — without
|
||||
the 172.8 GiB fetch the gate runs in two-report mode (candidate vs the served payload) and says
|
||||
so on its first line.
|
||||
|
||||
## Contract (the wire)
|
||||
|
||||
- **Live surface:** `/health` (== `/ping`), `/v1/models`, `/v1/chat/completions`
|
||||
@@ -638,14 +661,19 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
|
||||
rebuild), `scripts/download-frank-archon-swift15.sh` (the BF16 source a w4dn8 re-convert
|
||||
needs), `scripts/hfget.sh` / `scripts/hf-fetch.sh`; template
|
||||
`scripts/qwen38-froggeric-v22.5.jinja`.
|
||||
- Convert campaign (dormant, kept for a future quant): `scripts/convert-swift15-w4dn8.sh`
|
||||
`scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe`, `scripts/run-swift15-bf16.sh`
|
||||
`scripts/kld-gate-swift15.sh`, `scripts/kld-then-calib.sh`, `scripts/calib-push.py`
|
||||
- The campaign (qwen38-w5nl64, live): `scripts/run-qwen38-bf16.sh` (kldref / calib / score — the
|
||||
ONE measuring shape), `scripts/qwen38-calib-window.sh` (the gated calibration window),
|
||||
`scripts/convert-qwen38-w5nl64.sh` with
|
||||
`scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe` (the arm driver and its template),
|
||||
`scripts/kld-gate-qwen38.sh` (acceptance), `scripts/calib-push.py`,
|
||||
`scripts/make-calib-corpus.py`, `scripts/rechunk-kld-corpus.py`,
|
||||
`scripts/qwen38_common.py` (the tokenizer, JSONL and refusal plumbing those three
|
||||
corpus tools share — one `n_tokens`, so every tool counts the same way),
|
||||
`scripts/derive-protected-experts.py`
|
||||
`scripts/apply-protected-cast.sh`, `scripts/verify-swift15-ple.py`.
|
||||
`scripts/derive-protected-experts.py`, `scripts/apply-protected-cast.sh`.
|
||||
- Retired swift15-era campaign scripts, superseded by the above and kept only as reference:
|
||||
`scripts/convert-swift15-w4dn8.sh`, `scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe`,
|
||||
`scripts/run-swift15-bf16.sh`, `scripts/kld-gate-swift15.sh`, `scripts/kld-then-calib.sh`,
|
||||
`scripts/verify-swift15-ple.py`.
|
||||
- Probes: `scripts/probe-frank-archon-kv.py` (reads slots/window/dtype from the deployed env so
|
||||
it cannot drift; leg D = decode), `scripts/frank-archon-expert-activity.py`
|
||||
`scripts/autoresearch-frank-archon.sh` + `scripts/autoresearch/frank-archon-measure.py`.
|
||||
|
||||
@@ -200,6 +200,9 @@ DERIVE="$REPO_ROOT/scripts/derive-protected-experts.py"
|
||||
APPLY="$REPO_ROOT/scripts/apply-protected-cast.sh"
|
||||
CONVERT="$REPO_ROOT/scripts/convert-qwen38-w5nl64.sh"
|
||||
GATE="$REPO_ROOT/scripts/kld-gate-qwen38.sh"
|
||||
RUNNER="$REPO_ROOT/scripts/run-qwen38-bf16.sh"
|
||||
PUSHER="$REPO_ROOT/scripts/calib-push.py"
|
||||
WINDOW="$REPO_ROOT/scripts/qwen38-calib-window.sh"
|
||||
TEMPLATE="$REPO_ROOT/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe"
|
||||
|
||||
# make_gram <dir> <layer> <expert> <energy> <rows> — a RADGRAM2 down-Gram whose
|
||||
@@ -525,3 +528,206 @@ gate_case() { # gate_case <kl fp8> <kl iq4r> <kl cand> <top1 cand> [extra flags.
|
||||
[[ "$output" == *"has no slice 'code'"* ]]
|
||||
[[ "$output" == *"it has: ops"* ]]
|
||||
}
|
||||
|
||||
# --- the n-gram tier: the pin that starved the calibration boot ---------------
|
||||
|
||||
@test "the calibration boot reads the n-gram table from the file tier" {
|
||||
# 2026-10-29: 95.37 GiB pinned beside a 65.59 GiB/rank pool left 8.06 GiB of
|
||||
# RAM and the boot died "startup stopped: out of memory". calib only needs the
|
||||
# routed weights prefilled, so it reads the table off the container's file tier.
|
||||
run bash "$RUNNER" calib --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'disk'
|
||||
}
|
||||
|
||||
@test "the recording and every score pass keep production's n-gram placement" {
|
||||
# Those numbers are only comparable when the shape is identical; the tier is
|
||||
# part of the shape.
|
||||
run bash "$RUNNER" kldref --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'ram'
|
||||
}
|
||||
|
||||
@test "an unknown n-gram placement is refused by name before the boot" {
|
||||
run env FRANK_QWEN38_NGRAM_PLACEMENT=ssd bash "$RUNNER" calib --dry-run
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"auto|vram|ram|disk"* ]]
|
||||
}
|
||||
|
||||
# --- the pusher: a tap without a live rig is not a tap ------------------------
|
||||
|
||||
# stub_podman <dir> <running|absent> — a podman that answers the one question the
|
||||
# pusher asks: is the rig container running?
|
||||
stub_podman() {
|
||||
mkdir -p "$1/bin"
|
||||
if [ "$2" = running ]; then
|
||||
printf '#!/usr/bin/env bash\necho true\n' > "$1/bin/podman"
|
||||
elif [ "$2" = stopped ]; then
|
||||
printf '#!/usr/bin/env bash\necho false\n' > "$1/bin/podman"
|
||||
else
|
||||
printf '#!/usr/bin/env bash\nexit 125\n' > "$1/bin/podman"
|
||||
fi
|
||||
chmod +x "$1/bin/podman"
|
||||
}
|
||||
|
||||
pusher_fixture() { # pusher_fixture <marker-container or empty>
|
||||
D="$BATS_TEST_TMPDIR/push"
|
||||
mkdir -p "$D/tap" "$D/bin"
|
||||
printf '{"prompt":"hello"}\n' > "$D/corpus.jsonl"
|
||||
if [ -n "$1" ]; then
|
||||
printf 'container=%s\nimage=radiance@sha256:test\nmode=calib\npid=1\n' "$1" \
|
||||
> "$D/tap/.rig-boot"
|
||||
fi
|
||||
echo "$D"
|
||||
}
|
||||
|
||||
@test "the pusher refuses a tap that carries no rig marker: production is not a rig" {
|
||||
# With no marker, the old driver fed four rounds of 10M tokens into restored
|
||||
# production after the rig's boot had died, and recorded nothing.
|
||||
D="$(pusher_fixture '')"
|
||||
run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" --port 1
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"no rig marker"* ]]
|
||||
}
|
||||
|
||||
@test "the pusher refuses a marker that names another container" {
|
||||
D="$(pusher_fixture arbench)"
|
||||
run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" \
|
||||
--container qwen38-bf16 --port 1
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"names container 'arbench'"* ]]
|
||||
[[ "$output" == *"targets 'qwen38-bf16'"* ]]
|
||||
}
|
||||
|
||||
@test "the pusher refuses when the rig container is not running" {
|
||||
D="$(pusher_fixture qwen38-bf16)"
|
||||
stub_podman "$D" stopped
|
||||
run env PATH="$D/bin:$PATH" python3 "$PUSHER" --corpus "$D/corpus.jsonl" \
|
||||
--tap-dir "$D/tap" --container qwen38-bf16 --port 1
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"is not running"* ]]
|
||||
}
|
||||
|
||||
# --- the acceptance gate without an FP8 anchor -------------------------------
|
||||
|
||||
# kld_report_slices <path> <model> <kl all> <kl code> <top1 all> <top1 code>
|
||||
kld_report_slices() {
|
||||
python3 - "$@" <<'PY'
|
||||
import json, sys
|
||||
path, model, kl_all, kl_code, t_all, t_code = sys.argv[1:7]
|
||||
def g(kl, t, n=25000):
|
||||
return {"kld": {"mean": float(kl), "p99_9": float(kl) * 4},
|
||||
"top1_agreement": float(t), "positions": n}
|
||||
json.dump({"model": model, "all": g(kl_all, t_all),
|
||||
"by_source": {"code": g(kl_code, t_code), "ops": g(kl_all, t_all)}},
|
||||
open(path, "w"))
|
||||
PY
|
||||
}
|
||||
|
||||
gate2() { # gate2 <kl iq4r> <kl cand> <top1 iq4r> <top1 cand> [extra]
|
||||
D="$BATS_TEST_TMPDIR/kl2"
|
||||
mkdir -p "$D"
|
||||
kld_report_slices "$D/iq4r.json" iq4r-moe "$1" "$1" "$3" "$3"
|
||||
kld_report_slices "$D/cand.json" w5nl64 "$2" "$2" "$4" "$4"
|
||||
run bash "$GATE" "$D/iq4r.json" "$D/cand.json" "${@:5}"
|
||||
}
|
||||
|
||||
@test "a candidate that beats the served payload on mean, tail and top-1 passes without an anchor" {
|
||||
gate2 0.0190 0.0090 0.962 0.971
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"candidate vs served (no FP8 anchor)"* ]]
|
||||
[[ "$output" == *"KLD GATE PASS"* ]]
|
||||
}
|
||||
|
||||
@test "a better mean with a worse tail fails: the tail is where a mean hides its damage" {
|
||||
D="$BATS_TEST_TMPDIR/kl3"
|
||||
mkdir -p "$D"
|
||||
kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962
|
||||
python3 - "$D/cand.json" <<'PY'
|
||||
import json, sys
|
||||
# mean better than the served payload, p99.9 worse: the shape a mean-only gate waves through
|
||||
g = {"kld": {"mean": 0.0090, "p99_9": 0.4000}, "top1_agreement": 0.971, "positions": 25000}
|
||||
json.dump({"model": "w5nl64", "all": g, "by_source": {"code": dict(g), "ops": dict(g)}},
|
||||
open(sys.argv[1], "w"))
|
||||
PY
|
||||
run bash "$GATE" "$D/iq4r.json" "$D/cand.json"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"[FAIL] 2."* ]]
|
||||
[[ "$output" == *"KL p99.9"* ]]
|
||||
}
|
||||
|
||||
@test "a candidate that wins overall but regresses one source slice fails" {
|
||||
D="$BATS_TEST_TMPDIR/kl4"
|
||||
mkdir -p "$D"
|
||||
kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962
|
||||
# 'all' is better, 'code' is worse: exactly the trade a mean-only gate cannot see
|
||||
kld_report_slices "$D/cand.json" w5nl64 0.0090 0.0300 0.971 0.950
|
||||
run bash "$GATE" "$D/iq4r.json" "$D/cand.json"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" == *"[PASS] 1."* ]]
|
||||
[[ "$output" == *"[FAIL] 4."* ]]
|
||||
[[ "$output" == *"regressed: code"* ]]
|
||||
}
|
||||
|
||||
# --- the calibration window: gated, and it says so ---------------------------
|
||||
|
||||
window_fixture() {
|
||||
D="$BATS_TEST_TMPDIR/win"
|
||||
mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora"
|
||||
echo '{}' > "$D/campaign/kld-ref/kld.json"
|
||||
: > "$D/rad/qwen38-bf16.rad"
|
||||
: > "$D/corpora/calib.jsonl"
|
||||
echo "$D"
|
||||
}
|
||||
|
||||
@test "the calibration window refuses before touching a card without a reference recording" {
|
||||
D="$BATS_TEST_TMPDIR/win-less"
|
||||
mkdir -p "$D/campaign/calib"
|
||||
run env FRANK_QWEN38_CAMPAIGN="$D/campaign" bash "$WINDOW"
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"no reference recording"* ]]
|
||||
}
|
||||
|
||||
@test "the calibration window's dry run prints the gated plan, including what it would bank" {
|
||||
D="$(window_fixture)"
|
||||
run env FRANK_QWEN38_CAMPAIGN="$D/campaign" \
|
||||
FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" bash "$WINDOW" --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"run-qwen38-bf16.sh calib"* ]]
|
||||
[[ "$output" == *"else kill the rig"* ]]
|
||||
[[ "$output" == *"calib-push.py"* ]]
|
||||
[[ "$output" == *"AGENTTRASH"* ]]
|
||||
}
|
||||
|
||||
@test "a flat calibration round is not a failure: the tap's floor is pooled" {
|
||||
# round 1 of a 10.03M-token corpus against a 12M floor seals nothing, and the
|
||||
# first version of this gate aborted the run there — one round too early.
|
||||
run python3 - "$PUSHER" <<'PY'
|
||||
import importlib.util, sys
|
||||
spec = importlib.util.spec_from_file_location("cp", sys.argv[1])
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m)
|
||||
print(m.round_verdict(8192, 8192, False), # flat: continue
|
||||
m.round_verdict(8192, 4096, False), # shrink: fail
|
||||
m.round_verdict(8192, 20000, False), # grew: continue
|
||||
m.round_verdict(0, 0, True)) # done: stop
|
||||
PY
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$output" = "flat shrink grew done" ]
|
||||
}
|
||||
|
||||
@test "the calibration boot's step is its own knob; the KL modes keep production's" {
|
||||
# Calib is the only phase whose cost is read volume per document: four 2,048-token steps
|
||||
# re-stream each layer's routed experts four times. Measured 2026-10-29: 71 tok/s, 47 h tap.
|
||||
run env Q38_CALIB_MNBT=8192 bash "$RUNNER" calib --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
grep -A1 -x -- '--max-num-batched-tokens' <<<"$output" | grep -qx '8192'
|
||||
# and the same knob must NOT reach the measuring shape those numbers are compared in
|
||||
run env Q38_CALIB_MNBT=8192 bash "$RUNNER" kldref --dry-run
|
||||
[ "$status" -eq 0 ]
|
||||
! grep -qx '8192' <<<"$output"
|
||||
run env Q38_CALIB_MNBT=wide bash "$RUNNER" calib --dry-run
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"Q38_CALIB_MNBT must be a number"* ]]
|
||||
}
|
||||
Reference in new issue
Block a user