Frank
bb027265db
The swift15 gate compared an arm against the BF16 reference and passed on KL <= 0.03. That bar is now the wrong instrument: the fleet already ships the publisher's FP8-native iq4r-moe payload, so an arm that beats BF16 but loses to what is running is a downgrade. The gate takes three recordings - fp8 anchor, iq4r anchor, candidate - and requires the candidate's KL strictly inside the interval the two anchors define, at every slice, plus top-1 agreement no worse than the served payload. The anchors must be ordered (fp8 better than iq4r) or the run stops: an unordered pair means the instrument or the recording is wrong, not that the arm lost, and one number out of the report is named rather than guessed at. --recover prints the tightened numbers the same way it prints the failing ones, so the operator sees the bar they would have had to clear.
123 lines
5.5 KiB
Bash
Executable File
123 lines
5.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# kld-gate-qwen38.sh — RUNS ON DANKSERVER (or anywhere the reports are). The campaign's
|
|
# acceptance gate: three KLD reports against ONE reference recording, one verdict.
|
|
#
|
|
# The reference is the BF16 container built from the vendor checkpoint (run-qwen38-bf16.sh
|
|
# kldref, in the same drained window that records the calibration grams). Every scored
|
|
# container is measured against that one recording, so the KL scale runs from 0 = the
|
|
# checkpoint itself downward in faithfulness. Three containers ride it:
|
|
#
|
|
# fp8.json the published FP8-native release — the achievable end of the interval
|
|
# iq4r.json the payload frank-archon serves — the bar to beat
|
|
# cand.json the arm under test — the subject
|
|
#
|
|
# The FP8 number is what makes the interval absolute instead of relative: "half the gap"
|
|
# is half of the distance between the 4-bit payload and the near-lossless release, not half
|
|
# of an invented distance to the checkpoint. That is also why the FP8 payload is FETCHED and
|
|
# scored, never converted: it is a second opinion about what this checkpoint tolerates, not a
|
|
# step of the campaign. It lands LAST (disk sequencing) — by then the 335 GiB bf16 container
|
|
# is gone, so its 172.8 GiB fits.
|
|
#
|
|
# The law, in order of severity, on the mean KL of one slice (--slice all | a corpus source):
|
|
# 1. KL(fp8) < KL(iq4r) the anchors are ordered. If the near-lossless
|
|
# release does NOT sit closer to the checkpoint than
|
|
# the 4-bit payload, the corpus, the tokenizer or the
|
|
# boot shape is wrong and no arm number means anything.
|
|
# 2. KL(fp8) < KL(cand) < KL(iq4r) the candidate lies strictly inside the interval —
|
|
# outside it, something is mis-measured, not better
|
|
# 3. KL(cand) <= KL(iq4r) - R x gap it closes at least R (--recover, default 0.5) of the
|
|
# gap = KL(iq4r) - KL(fp8): at least half the way from
|
|
# what is served toward the release (agreed 2026-10-27)
|
|
# 4. top-1(cand) >= top-1(iq4r) the agreement the harness keys on does not fall
|
|
#
|
|
# Booting the containers is NOT this script's job: `run-qwen38-bf16.sh score RAD OUT.json`
|
|
# writes the reports and owns the single engine argv (same pin, --tp 2, --kv-cache-dtype fp8,
|
|
# same step size, --deterministic, no speculation) that makes the three numbers comparable.
|
|
# This script is arithmetic, so it re-runs against old reports and is testable without a GPU.
|
|
#
|
|
# Usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]
|
|
set -uo pipefail
|
|
|
|
[ "$#" -ge 3 ] || { echo "usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]" >&2; exit 2; }
|
|
python3 - "$@" <<'PY'
|
|
import json, sys
|
|
|
|
a = sys.argv[1:]
|
|
opts, files = [], []
|
|
i = 0
|
|
while i < len(a):
|
|
if a[i].startswith("--"):
|
|
opts += [a[i], a[i + 1]]
|
|
i += 2
|
|
else:
|
|
files.append(a[i])
|
|
i += 1
|
|
if len(files) != 3:
|
|
sys.exit("usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]")
|
|
o = dict(zip(opts[0::2], opts[1::2]))
|
|
slice_ = o.get("--slice", "all")
|
|
recover = float(o.get("--recover", 0.5))
|
|
if not 0.0 < recover <= 1.0:
|
|
sys.exit(f"--recover must be in (0,1], got {recover}")
|
|
|
|
|
|
def die(msg):
|
|
print(msg, file=sys.stderr)
|
|
raise SystemExit(2)
|
|
|
|
|
|
def read(path):
|
|
try:
|
|
d = json.load(open(path))
|
|
except Exception as e:
|
|
die(f"cannot read the KLD report {path}: {e}")
|
|
if slice_ == "all":
|
|
g = d.get("all")
|
|
else:
|
|
g = (d.get("by_source") or {}).get(slice_)
|
|
if g is None:
|
|
die(f"{path} has no slice {slice_!r}; it has: "
|
|
f"{', '.join(sorted(d.get('by_source') or {})) or '(none)'}")
|
|
if g is None or "kld" not in g or "top1_agreement" not in g:
|
|
die(f"{path} has no {slice_!r} summary — wrong report version?")
|
|
return d, g
|
|
|
|
|
|
rows = []
|
|
for label, path in zip(("fp8", "iq4r", "cand"), files):
|
|
d, g = read(path)
|
|
rows.append((label, path, d.get("model", "?"), g["kld"]["mean"], g["kld"]["p99_9"],
|
|
g["top1_agreement"], g["positions"]))
|
|
|
|
print(f"reference: BF16 container; slice: {slice_}")
|
|
print(f"{'':6} {'model':38} {'KL mean':>10} {'KL p99.9':>10} {'top-1':>8} positions")
|
|
for label, path, model, kl, p999, top1, n in rows:
|
|
print(f"{label:6} {str(model)[:38]:38} {kl:10.6f} {p999:10.4f} {100*top1:7.2f}% {n}")
|
|
r = {label: (kl, top1) for label, _p, _m, kl, _q, top1, _n in rows}
|
|
kl_fp8, kl_iq4r, kl_cand = r["fp8"][0], r["iq4r"][0], r["cand"][0]
|
|
t_iq4r, t_cand = r["iq4r"][1], r["cand"][1]
|
|
gap = kl_iq4r - kl_fp8
|
|
ok = True
|
|
|
|
|
|
def law(n, text, passed):
|
|
global ok
|
|
ok = ok and passed
|
|
print(f" [{'PASS' if passed else 'FAIL'}] {n}. {text}")
|
|
|
|
|
|
law(1, f"anchors ordered: fp8 {kl_fp8:.6f} < iq4r {kl_iq4r:.6f} (gap {gap:.6f})",
|
|
gap > 0)
|
|
law(2, f"candidate inside the interval: {kl_fp8:.6f} < {kl_cand:.6f} < {kl_iq4r:.6f}",
|
|
kl_fp8 < kl_cand < kl_iq4r)
|
|
need = kl_iq4r - recover * gap
|
|
law(3, f"candidate closes {100*max((kl_iq4r - kl_cand)/gap, 0):.1f}% of the "
|
|
f"{gap:.6f} gap (needs {100*recover:.0f}%, i.e. KL <= {need:.6f})"
|
|
if gap > 0 else "gap is not positive; cannot score recovery",
|
|
gap > 0 and kl_cand <= need)
|
|
law(4, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%",
|
|
t_cand >= t_iq4r)
|
|
print("KLD GATE " + ("PASS" if ok else "FAIL"))
|
|
sys.exit(0 if ok else 1)
|
|
PY
|