diff --git a/scripts/calib-push.py b/scripts/calib-push.py index beffdfb..4c43e7b 100755 --- a/scripts/calib-push.py +++ b/scripts/calib-push.py @@ -25,6 +25,11 @@ Mechanics notes: - a round that seals no tap bytes is a FAILURE, not a retry: the engine answering 200 while the tap directory is wrong, full or read-only is how a corpus gets pushed four times for nothing. +- before any prompt, the tap must carry the rig's own marker + (`--tap-marker`, written by run-qwen38-bf16.sh calib) naming THIS + container, and that container must be running. Measured 2026-10-29: + with neither check, four rounds of 10M tokens went into restored + production after the rig's boot had died, and recorded nothing. Usage: calib-push.py [--port 7770] [--parallel 64] [--container arbench] [--corpus /home/dank/corpora/out/calib.jsonl] [--max-rounds 4] @@ -107,6 +112,40 @@ def tap_bytes(path): return total +def require_rig(a): + """Refuse unless the rig that opened this tap is up and serving right now. + + Three facts, in order: the runner wrote its marker into the tap dir, the + marker names the container this pusher targets, and that container is + running. A marker without a live container is a boot that died; a live + container without a marker is some other engine on the port — on 2026-10-29 + it was restored production, which took four rounds of 10M tokens before the + old driver admitted it had recorded nothing. + """ + marker = os.path.join(a.tap_dir, a.tap_marker) + if not os.path.isfile(marker): + qc.refuse(f"no rig marker at {marker}: this tap was not opened by " + "run-qwen38-bf16.sh calib, so a push here would feed whatever " + "answers the port and record nothing") + try: + lines = open(marker, encoding="utf-8", errors="replace").read().splitlines() + except OSError as exc: + qc.refuse(f"cannot read the rig marker {marker}: {exc}") + meta = dict(line.split("=", 1) for line in lines if "=" in line) + who = meta.get("container", "") + if who != a.container: + qc.refuse(f"the rig marker names container {who!r} and this pusher targets " + f"{a.container!r} — wrong tap, or a stale marker") + try: + p = subprocess.run(["podman", "inspect", "-f", "{{.State.Running}}", a.container], + capture_output=True, text=True, timeout=30) + except Exception as exc: + qc.refuse(f"cannot inspect the rig container {a.container}: {exc}") + if p.returncode != 0 or p.stdout.strip() != "true": + qc.refuse(f"the rig container {a.container} is not running — whatever is " + "answering the port, it is not the calibration rig") + + def parse_args(): ap = argparse.ArgumentParser() ap.add_argument("--port", type=int, default=7770) @@ -114,6 +153,8 @@ def parse_args(): ap.add_argument("--container", default="arbench") ap.add_argument("--model", default="frank-archon") ap.add_argument("--tap-dir", default="/home/dank/campaign/qwen38-w5nl64/calib") + ap.add_argument("--tap-marker", default=".rig-boot", + help="the file run-qwen38-bf16.sh calib writes into the tap dir; without it the pusher refuses, because an unmarked tap is not a rig") ap.add_argument("--corpus", default="/home/dank/corpora/out/calib.jsonl") ap.add_argument("--max-rounds", type=int, default=4) return ap.parse_args() @@ -136,6 +177,7 @@ def main(): a = parse_args() url = f"http://127.0.0.1:{a.port}/v1/completions" prompts = load_prompts(a) + require_rig(a) chars = sum(len(p) for p in prompts) print(f"corpus: {len(prompts)} prompts, ~{chars // 4} tokens (chars/4)", flush=True) diff --git a/scripts/kld-gate-qwen38.sh b/scripts/kld-gate-qwen38.sh index 6ad5738..e60834f 100755 --- a/scripts/kld-gate-qwen38.sh +++ b/scripts/kld-gate-qwen38.sh @@ -1,44 +1,53 @@ #!/usr/bin/env bash # kld-gate-qwen38.sh — RUNS ON DANKSERVER (or anywhere the reports are). The campaign's -# acceptance gate: three KLD reports against ONE reference recording, one verdict. +# acceptance gate: one verdict from the KLD reports of the containers that ride one reference +# recording. # # The reference is the BF16 container built from the vendor checkpoint (run-qwen38-bf16.sh # kldref, in the same drained window that records the calibration grams). Every scored # container is measured against that one recording, so the KL scale runs from 0 = the -# checkpoint itself downward in faithfulness. Three containers ride it: +# checkpoint itself downward in faithfulness. # -# fp8.json the published FP8-native release — the achievable end of the interval -# iq4r.json the payload frank-archon serves — the bar to beat -# cand.json the arm under test — the subject +# TWO MODES, because the disk decides how many anchors can exist at once: # -# The FP8 number is what makes the interval absolute instead of relative: "half the gap" -# is half of the distance between the 4-bit payload and the near-lossless release, not half -# of an invented distance to the checkpoint. That is also why the FP8 payload is FETCHED and -# scored, never converted: it is a second opinion about what this checkpoint tolerates, not a -# step of the campaign. It lands LAST (disk sequencing) — by then the 335 GiB bf16 container -# is gone, so its 172.8 GiB fits. +# gate IQ4R.json CAND.json candidate vs the served payload +# gate FP8.json IQ4R.json CAND.json the same, inside the absolute interval # -# The law, in order of severity, on the mean KL of one slice (--slice all | a corpus source): -# 1. KL(fp8) < KL(iq4r) the anchors are ordered. If the near-lossless -# release does NOT sit closer to the checkpoint than -# the 4-bit payload, the corpus, the tokenizer or the -# boot shape is wrong and no arm number means anything. -# 2. KL(fp8) < KL(cand) < KL(iq4r) the candidate lies strictly inside the interval — -# outside it, something is mis-measured, not better -# 3. KL(cand) <= KL(iq4r) - R x gap it closes at least R (--recover, default 0.5) of the -# gap = KL(iq4r) - KL(fp8): at least half the way from -# what is served toward the release (agreed 2026-10-27) -# 4. top-1(cand) >= top-1(iq4r) the agreement the harness keys on does not fall +# iq4r.json is the payload frank-archon serves — the bar to beat. When the FP8-native release +# (172.8 GiB, fetched last because of the peak-disk law) is on disk, pass its report first: +# "half the gap" is then half of the distance between the 4-bit payload and the near-lossless +# release, not half of an invented distance to the checkpoint. Without it the gate still +# answers the campaign's question — does the candidate beat what is served — and says so. +# +# The law, on the summary of one slice (--slice all | a corpus source): +# +# candidate vs served (2 reports) +# 1. mean KL(cand) < mean KL(iq4r) the served payload is beaten on the corpus +# 2. p99.9(cand) <= p99.9(iq4r) and the tail is no worse — a mean can hide a few +# positions taking the whole hit +# 3. top-1(cand) >= top-1(iq4r) the agreement the harness keys on does not fall +# 4. every source slice passes 1-3 a win bought by regressing one distribution +# (code, prose, tools) is not a win +# +# inside the interval (3 reports), laws 1-4 above (1-3 renumbered to the interval and the +# recovery fraction, as agreed 2026-10-27) plus: +# 5. p99.9(cand) <= p99.9(iq4r) the tail law +# 6. every source slice passes the slice law # # Booting the containers is NOT this script's job: `run-qwen38-bf16.sh score RAD OUT.json` # writes the reports and owns the single engine argv (same pin, --tp 2, --kv-cache-dtype fp8, -# same step size, --deterministic, no speculation) that makes the three numbers comparable. +# same step size, --deterministic, no speculation) that makes the numbers comparable. # This script is arithmetic, so it re-runs against old reports and is testable without a GPU. # -# Usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F] +# Usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME] +# kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F] set -uo pipefail -[ "$#" -ge 3 ] || { echo "usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]" >&2; exit 2; } +if [ "$#" -lt 2 ]; then + echo "usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]" >&2 + echo " kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]" >&2 + exit 2 +fi python3 - "$@" <<'PY' import json, sys @@ -52,13 +61,15 @@ while i < len(a): else: files.append(a[i]) i += 1 -if len(files) != 3: - sys.exit("usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]") +if len(files) not in (2, 3): + sys.exit("usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]\n" + " kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]") o = dict(zip(opts[0::2], opts[1::2])) slice_ = o.get("--slice", "all") recover = float(o.get("--recover", 0.5)) if not 0.0 < recover <= 1.0: sys.exit(f"--recover must be in (0,1], got {recover}") +interval = len(files) == 3 def die(msg): @@ -66,11 +77,15 @@ def die(msg): raise SystemExit(2) -def read(path): +def load(path): try: - d = json.load(open(path)) + return json.load(open(path)) except Exception as e: die(f"cannot read the KLD report {path}: {e}") + + +def summary(path, d): + """The slice this gate grades: 'all', or one corpus source.""" if slice_ == "all": g = d.get("all") else: @@ -80,23 +95,25 @@ def read(path): f"{', '.join(sorted(d.get('by_source') or {})) or '(none)'}") if g is None or "kld" not in g or "top1_agreement" not in g: die(f"{path} has no {slice_!r} summary — wrong report version?") - return d, g + return g +docs = {label: load(path) for label, path in + zip(("fp8", "iq4r", "cand") if interval else ("iq4r", "cand"), files)} rows = [] -for label, path in zip(("fp8", "iq4r", "cand"), files): - d, g = read(path) - rows.append((label, path, d.get("model", "?"), g["kld"]["mean"], g["kld"]["p99_9"], - g["top1_agreement"], g["positions"])) +for label, path in zip(("fp8", "iq4r", "cand") if interval else ("iq4r", "cand"), files): + g = summary(path, docs[label]) + rows.append((label, path, docs[label].get("model", "?"), g["kld"]["mean"], + g["kld"]["p99_9"], g["top1_agreement"], g["positions"])) -print(f"reference: BF16 container; slice: {slice_}") +print(f"reference: BF16 container; slice: {slice_}; " + + ("inside the FP8 interval" if interval else "candidate vs served (no FP8 anchor)")) print(f"{'':6} {'model':38} {'KL mean':>10} {'KL p99.9':>10} {'top-1':>8} positions") for label, path, model, kl, p999, top1, n in rows: print(f"{label:6} {str(model)[:38]:38} {kl:10.6f} {p999:10.4f} {100*top1:7.2f}% {n}") -r = {label: (kl, top1) for label, _p, _m, kl, _q, top1, _n in rows} -kl_fp8, kl_iq4r, kl_cand = r["fp8"][0], r["iq4r"][0], r["cand"][0] -t_iq4r, t_cand = r["iq4r"][1], r["cand"][1] -gap = kl_iq4r - kl_fp8 +r = {label: (kl, p999, top1) for label, _p, _m, kl, p999, top1, _n in rows} +kl_cand, p_cand, t_cand = r["cand"] +kl_iq4r, p_iq4r, t_iq4r = r["iq4r"] ok = True @@ -106,17 +123,64 @@ def law(n, text, passed): print(f" [{'PASS' if passed else 'FAIL'}] {n}. {text}") -law(1, f"anchors ordered: fp8 {kl_fp8:.6f} < iq4r {kl_iq4r:.6f} (gap {gap:.6f})", - gap > 0) -law(2, f"candidate inside the interval: {kl_fp8:.6f} < {kl_cand:.6f} < {kl_iq4r:.6f}", - kl_fp8 < kl_cand < kl_iq4r) -need = kl_iq4r - recover * gap -law(3, f"candidate closes {100*max((kl_iq4r - kl_cand)/gap, 0):.1f}% of the " - f"{gap:.6f} gap (needs {100*recover:.0f}%, i.e. KL <= {need:.6f})" - if gap > 0 else "gap is not positive; cannot score recovery", - gap > 0 and kl_cand <= need) -law(4, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%", - t_cand >= t_iq4r) +def tail_law(n): + law(n, f"candidate KL p99.9 {p_cand:.4f} <= served iq4r {p_iq4r:.4f}", + p_cand <= p_iq4r) + + +# --- the slice law: every corpus source must pass the metric laws ------------- +def slice_law(n): + """A win that regresses one source class is not a win: check them all.""" + mine = docs["cand"].get("by_source") or {} + theirs = docs["iq4r"].get("by_source") or {} + if not mine or not theirs: + law(n, "no per-source summaries to check", True) + return + missing = sorted(set(theirs) - set(mine)) + extra = sorted(set(mine) - set(theirs)) + if missing or extra: + law(n, "same source slices in both reports" + + (f" (missing from cand: {', '.join(missing)})" if missing else "") + + (f" (only in cand: {', '.join(extra)})" if extra else ""), False) + return + bad = [] + for src in sorted(theirs): + c, q = mine[src], theirs[src] + worse_mean = c["kld"]["mean"] >= q["kld"]["mean"] + worse_tail = c["kld"]["p99_9"] > q["kld"]["p99_9"] + worse_top1 = c["top1_agreement"] < q["top1_agreement"] + if worse_mean or worse_tail or worse_top1: + why = ",".join(m for m, w in (("mean", worse_mean), ("p99.9", worse_tail), + ("top-1", worse_top1)) if w) + bad.append(f"{src} ({why})") + law(n, "every source slice passes (mean, p99.9, top-1)" + + (f" — regressed: {'; '.join(bad)}" if bad else ""), not bad) + + +if interval: + kl_fp8 = r["fp8"][0] + gap = kl_iq4r - kl_fp8 + law(1, f"anchors ordered: fp8 {kl_fp8:.6f} < iq4r {kl_iq4r:.6f} (gap {gap:.6f})", + gap > 0) + law(2, f"candidate inside the interval: {kl_fp8:.6f} < {kl_cand:.6f} < {kl_iq4r:.6f}", + kl_fp8 < kl_cand < kl_iq4r) + need = kl_iq4r - recover * gap + law(3, f"candidate closes {100*max((kl_iq4r - kl_cand)/gap, 0):.1f}% of the " + f"{gap:.6f} gap (needs {100*recover:.0f}%, i.e. KL <= {need:.6f})" + if gap > 0 else "gap is not positive; cannot score recovery", + gap > 0 and kl_cand <= need) + law(4, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%", + t_cand >= t_iq4r) + tail_law(5) + slice_law(6) +else: + law(1, f"candidate beats the served payload: KL {kl_cand:.6f} < iq4r {kl_iq4r:.6f}", + kl_cand < kl_iq4r) + tail_law(2) + law(3, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%", + t_cand >= t_iq4r) + slice_law(4) + print("KLD GATE " + ("PASS" if ok else "FAIL")) sys.exit(0 if ok else 1) PY diff --git a/scripts/qwen38-calib-window.sh b/scripts/qwen38-calib-window.sh new file mode 100755 index 0000000..15e3c51 --- /dev/null +++ b/scripts/qwen38-calib-window.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# qwen38-calib-window.sh — RUNS ON DANKSERVER. The calibration window, and nothing else: +# boot the BF16 rig, wait until it is actually serving, push the corpus until the engine says +# the tap is full, stop the rig, restore frank-archon, and bank the reference container. +# +# The reference recording is already taken (kld-ref/kld.json exists), so this window is the +# calibration tap alone — the one job that needs both cards after it. The kldref phase is NOT +# re-run here: the chained launcher that used to do that moved the banked kld-ref into trash +# first, which is how a 16-hour phase gets paid for twice. +# +# Every phase is gated on the previous one, because the last time this ran unguarded +# (2026-10-29) the rig's boot died of an OOM, the health loop quietly fell through, and the +# pusher fed four rounds of 10M tokens into RESTORED PRODUCTION with nothing recorded: +# +# 1. the runner must still be alive and its container must answer /health, or this script +# kills it, restores the service and exits non-zero — no push, ever; +# 2. the pusher refuses on its own unless the rig wrote its marker into the tap dir, the +# marker names this container, and the container is running (scripts/calib-push.py +# --tap-marker), so a second lock survives a mistake in this file; +# 3. the reference container is banked into trash only when BOTH the serve and the push +# exited 0 and the tap dir holds grams. The checkpoint that reproduces it stays. +# +# knobs: Q38_WINDOW_CALIB_CORPUS (calib-7168.jsonl — the re-chunked file the rig's pool can +# hold), Q38_WINDOW_HEALTH_TIMEOUT (seconds to wait for /health), Q38_WINDOW_ROUNDS +# (pusher rounds; one is 10M tokens), Q38_WINDOW_KEEP_RAD=1 to leave the .rad in place. +# Usage: scripts/qwen38-calib-window.sh [--dry-run] +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 9 + +DRY=0 +[ "${1:-}" = "--dry-run" ] && DRY=1 +say() { printf '%s %s\n' "$(date -Is)" "$*"; } + +MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}" +CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}" +RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}" +CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}" +KLDREF="$CAMPAIGN/kld-ref/kld.json" +CALIB="$CAMPAIGN/calib" +HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}" +ROUNDS="${Q38_WINDOW_ROUNDS:-6}" +RIG=qwen38-bf16 +SERVICE=frank-archon +PY="${PY:-python3}" + +# ---- preflight: everything this window needs, before a card is touched ------- +[ -f "$KLDREF" ] || { echo "REFUSED: no reference recording at $KLDREF — record it first (scripts/run-qwen38-bf16.sh kldref); this window only fills the calibration tap" >&2; exit 2; } +[ -f "$RAD" ] || { echo "REFUSED: no BF16 reference container at $RAD — nothing to serve; scripts/build-qwen38-bf16-rad.sh" >&2; exit 2; } +[ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; } +[ -d "$CALIB" ] || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; } +[ -w "$CALIB" ] || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; } +! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; } +! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; } + +if [ "$DRY" = 1 ]; then + echo "dry run: calib window — the service is left running, nothing is mounted or written" + echo "would run: bash scripts/run-qwen38-bf16.sh calib (rig boots, writes $CALIB/.rig-boot)" + echo "would wait: /health up within ${HEALTH_TIMEOUT}s, else kill the rig, restore $SERVICE, exit 1" + echo "would run: $PY scripts/calib-push.py --container $RIG --corpus $CORPUS --max-rounds $ROUNDS" + echo "would bank: $RAD -> ~/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad/ (only if both phases exit 0)" + exit 0 +fi + +say "calib window start pid=$$ corpus=$CORPUS rounds=$ROUNDS" +[ -n "$(ls -A "$CALIB" 2>/dev/null)" ] && say "tap dir $CALIB is not empty; the engine's token floor is pooled, so an earlier partial tap still counts" + +# ---- phase 1: the rig ------------------------------------------------------- +say "phase=calib serve begin" +bash scripts/run-qwen38-bf16.sh calib & spid=$! + +rig_up() { + podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true \ + && curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health +} + +waited=0 +until rig_up; do + if ! kill -0 "$spid" 2>/dev/null; then + wait "$spid"; crc=$? + say "phase=calib serve end rc=$crc (the rig exited before it ever answered /health)" + echo "FAIL: the rig boot died (rc=$crc) — nothing was pushed; read /home/dank/logs/qwen38_bf16_calib.log" >&2 + exit 1 + fi + if [ "$waited" -ge "$HEALTH_TIMEOUT" ]; then + say "phase=calib health never came after ${HEALTH_TIMEOUT}s; killing the rig" + podman stop "$RIG" >/dev/null 2>&1 || true + wait "$spid" + echo "FAIL: the rig never answered /health within ${HEALTH_TIMEOUT}s — nothing was pushed" >&2 + exit 1 + fi + sleep 15 + waited=$((waited + 15)) +done +say "phase=calib health ok waited=${waited}s" + +# ---- phase 2: the tap ------------------------------------------------------- +say "phase=calib push begin" +"$PY" scripts/calib-push.py --port 7770 --parallel 8 --container "$RIG" \ + --tap-dir "$CALIB" --corpus "$CORPUS" --max-rounds "$ROUNDS" +prc=$? +say "phase=calib push end rc=$prc" + +podman stop "$RIG" >/dev/null 2>&1 || true +wait "$spid"; crc=$? +say "phase=calib serve end rc=$crc" + +if [ "$prc" -ne 0 ] || [ "$crc" -ne 0 ]; then + echo "FAIL: serve rc=$crc push rc=$prc — the tap is NOT banked and $RAD stays: $(basename "$RAD")" >&2 + exit 1 +fi +[ -n "$(ls -A "$CALIB" 2>/dev/null)" ] || { echo "FAIL: both phases exited 0 but the tap dir $CALIB is empty" >&2; exit 1; } +say "TAP BANKED: $(ls -1 "$CALIB" | wc -l) file(s) in $CALIB" + +# ---- phase 3: bank the reference container ---------------------------------- +if [ "${Q38_WINDOW_KEEP_RAD:-0}" = "1" ]; then + say "reference .rad kept (Q38_WINDOW_KEEP_RAD=1)" +else + T="$HOME/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad" + mkdir -p "$T" || { echo "FAIL: cannot create trash dir $T" >&2; exit 1; } + mv "$RAD" "$T/" && say "reference .rad banked in $T (the checkpoint that rebuilds it stays)" +fi +say "calib window end" diff --git a/scripts/run-qwen38-bf16.sh b/scripts/run-qwen38-bf16.sh index fe358b6..cf7e730 100755 --- a/scripts/run-qwen38-bf16.sh +++ b/scripts/run-qwen38-bf16.sh @@ -193,6 +193,25 @@ if ! [[ "$MAX_MODEL_LEN" =~ ^[0-9]+$ ]]; then exit 2 fi +# Where the n-gram table lives. On the bf16 payload it is 95.37 GiB and `ram` +# PINS it: the calibration boot died at "startup stopped: out of memory" with +# 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available (measured 2026-10-29), +# while the reference recording and every score boot had already succeeded with +# the same pool and slots. The difference that matters is the job: calib only +# needs the routed weights prefilled, so it reads the table from the container's +# own file tier and the pin never happens; kldref and score keep production's +# placement, because their numbers are only comparable when every other term of +# the shape is identical. FRANK_QWEN38_NGRAM_PLACEMENT overrides either default. +case "$MODE" in + calib) NGRAM_PLACEMENT="${FRANK_QWEN38_NGRAM_PLACEMENT:-disk}" ;; + *) NGRAM_PLACEMENT="${FRANK_QWEN38_NGRAM_PLACEMENT:-ram}" ;; +esac +case "$NGRAM_PLACEMENT" in + auto|vram|ram|disk) ;; + *) echo "REFUSED: FRANK_QWEN38_NGRAM_PLACEMENT must be auto|vram|ram|disk (got: $NGRAM_PLACEMENT)" >&2 + exit 2 ;; +esac + # Every container this campaign boots is read through the ro /models mount, so # the host path must live under it; the container-side spelling is what goes # into the argv. Same rule for the container a score pass opens. @@ -230,7 +249,7 @@ ARGS=(--tp 2 --placement expert_tiered --weights-disk-tier --host-pool-mib "$HOST_POOL_MIB" --gpu-headroom-mib "$GPU_HEADROOM_MIB" --deterministic --num-speculative-tokens 0 --kv-cache-dtype fp8 --max-model-len "$MAX_MODEL_LEN" - --ngram-placement ram + --ngram-placement "$NGRAM_PLACEMENT" --override-chat-template /models/tpl.jinja) MOUNTS=(-v "$MODELS:/models:ro") TAPENV=() @@ -289,6 +308,17 @@ case "$MODE" in MOUNTS+=(-v "$CALIB_DIR:/data/calib") TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib -e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-12000000}") + # The tap's own authorization. A pusher must see this marker before it sends + # a prompt (calib-push.py --tap-marker), it names the container so a push + # aimed anywhere else refuses, and a boot that fails removes it again below. + # Measured 2026-10-29: with no marker, the chain pushed 4 rounds x 10M tokens + # into RESTORED PRODUCTION after the rig's boot died, and nothing recorded. + if [ "$DRY" = 1 ]; then + : + else + printf 'container=%s\nimage=%s\nmode=calib\nstarted=%s\npid=%s\n' \ + "$C" "$IMAGE" "$(date -Is)" "$$" > "$CALIB_DIR/.rig-boot" + fi ;; esac @@ -377,5 +407,12 @@ podman rm -f "$C" >/dev/null 2>&1 || true "${CMD[@]}" 2>&1 | tee "$LOG" rc=${PIPESTATUS[0]} +# A boot that failed must not leave the tap marked as rig-authorized: the marker +# is what calib-push.py trusts, and the point of it is that a dead rig can never +# be pushed into again. +if [ "$MODE" = calib ] && [ "$rc" -ne 0 ] && [ -f "$CALIB_DIR/.rig-boot" ]; then + mv -f "$CALIB_DIR/.rig-boot" "$CALIB_DIR/.rig-boot.failed-$(date +%Y%m%d-%H%M%S)" 2>/dev/null || true + echo "the rig boot failed (rc=$rc): the tap marker is retired, so no pusher can take this tap for a live rig" +fi echo "run-qwen38-bf16 $MODE exited rc=$rc; $SERVICE restored; log: $LOG" exit "$rc" diff --git a/serverAppConfigs-docs/LLM/frank-archon.md b/serverAppConfigs-docs/LLM/frank-archon.md index 89ec957..a110da3 100644 --- a/serverAppConfigs-docs/LLM/frank-archon.md +++ b/serverAppConfigs-docs/LLM/frank-archon.md @@ -562,6 +562,29 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10). fork bomb inside the container. Kill with `podman rm -f` (the cgroup owns every generation) never pattern-kill an interpreter name on this host; use `TORCH_LOGS` for dynamo questions. +### The quantization pipeline + +- **A calibration tap is only a tap when the rig that opened it is running.** The runner writes + `.rig-boot` into the tap dir (container, image, pid) and retires it when its boot exits non-zero; + `calib-push.py` refuses without that marker, refuses when it names another container, and refuses + when the named container is not running; `qwen38-calib-window.sh` stops on a failed phase instead + of falling through its health wait. All three exist because on 2026-10-29 the unguarded chain's + rig boot died of an OOM, the health loop expired without failing, and the pusher fed four rounds + of 10M tokens into **restored production** — 86 minutes of live prefill that recorded nothing. +- **The n-gram table's pin is what starves a rig boot, and the tier is a knob.** On the BF16 payload + the table is 95.37 GiB, `--ngram-placement ram` pins it, and the calibration boot died at + `startup stopped: out of memory` with 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available — + while the reference recording and every score boot succeeded at the same pool and slot count. + Calibration only needs the routed weights prefilled, so it reads the table from the container's + file tier (`--ngram-placement disk`); kldref and score keep production's placement, because their + numbers are only comparable at an identical shape. +- **Acceptance is three metrics and every slice, not a mean.** `scripts/kld-gate-qwen38.sh` grades + mean KL, the p99.9 tail, top-1 agreement and every `by_source` slice (code, prose, tools): + a mean can hide a few positions taking the whole hit, and an overall win can be bought by + regressing one distribution. The FP8-native anchor is an interval, not a requirement — without + the 172.8 GiB fetch the gate runs in two-report mode (candidate vs the served payload) and says + so on its first line. + ## Contract (the wire) - **Live surface:** `/health` (== `/ping`), `/v1/models`, `/v1/chat/completions` @@ -638,14 +661,19 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10). rebuild), `scripts/download-frank-archon-swift15.sh` (the BF16 source a w4dn8 re-convert needs), `scripts/hfget.sh` / `scripts/hf-fetch.sh`; template `scripts/qwen38-froggeric-v22.5.jinja`. -- Convert campaign (dormant, kept for a future quant): `scripts/convert-swift15-w4dn8.sh` - `scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe`, `scripts/run-swift15-bf16.sh` - `scripts/kld-gate-swift15.sh`, `scripts/kld-then-calib.sh`, `scripts/calib-push.py` +- The campaign (qwen38-w5nl64, live): `scripts/run-qwen38-bf16.sh` (kldref / calib / score — the + ONE measuring shape), `scripts/qwen38-calib-window.sh` (the gated calibration window), + `scripts/convert-qwen38-w5nl64.sh` with + `scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe` (the arm driver and its template), + `scripts/kld-gate-qwen38.sh` (acceptance), `scripts/calib-push.py`, `scripts/make-calib-corpus.py`, `scripts/rechunk-kld-corpus.py`, `scripts/qwen38_common.py` (the tokenizer, JSONL and refusal plumbing those three corpus tools share — one `n_tokens`, so every tool counts the same way), - `scripts/derive-protected-experts.py` - `scripts/apply-protected-cast.sh`, `scripts/verify-swift15-ple.py`. + `scripts/derive-protected-experts.py`, `scripts/apply-protected-cast.sh`. +- Retired swift15-era campaign scripts, superseded by the above and kept only as reference: + `scripts/convert-swift15-w4dn8.sh`, `scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe`, + `scripts/run-swift15-bf16.sh`, `scripts/kld-gate-swift15.sh`, `scripts/kld-then-calib.sh`, + `scripts/verify-swift15-ple.py`. - Probes: `scripts/probe-frank-archon-kv.py` (reads slots/window/dtype from the deployed env so it cannot drift; leg D = decode), `scripts/frank-archon-expert-activity.py` `scripts/autoresearch-frank-archon.sh` + `scripts/autoresearch/frank-archon-measure.py`. diff --git a/tests/qwen38_campaign.bats b/tests/qwen38_campaign.bats index f061a8d..afcdc12 100755 --- a/tests/qwen38_campaign.bats +++ b/tests/qwen38_campaign.bats @@ -200,6 +200,9 @@ DERIVE="$REPO_ROOT/scripts/derive-protected-experts.py" APPLY="$REPO_ROOT/scripts/apply-protected-cast.sh" CONVERT="$REPO_ROOT/scripts/convert-qwen38-w5nl64.sh" GATE="$REPO_ROOT/scripts/kld-gate-qwen38.sh" +RUNNER="$REPO_ROOT/scripts/run-qwen38-bf16.sh" +PUSHER="$REPO_ROOT/scripts/calib-push.py" +WINDOW="$REPO_ROOT/scripts/qwen38-calib-window.sh" TEMPLATE="$REPO_ROOT/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe" # make_gram — a RADGRAM2 down-Gram whose @@ -525,3 +528,174 @@ gate_case() { # gate_case [extra flags. [[ "$output" == *"has no slice 'code'"* ]] [[ "$output" == *"it has: ops"* ]] } + +# --- the n-gram tier: the pin that starved the calibration boot --------------- + +@test "the calibration boot reads the n-gram table from the file tier" { + # 2026-10-29: 95.37 GiB pinned beside a 65.59 GiB/rank pool left 8.06 GiB of + # RAM and the boot died "startup stopped: out of memory". calib only needs the + # routed weights prefilled, so it reads the table off the container's file tier. + run bash "$RUNNER" calib --dry-run + [ "$status" -eq 0 ] + grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'disk' +} + +@test "the recording and every score pass keep production's n-gram placement" { + # Those numbers are only comparable when the shape is identical; the tier is + # part of the shape. + run bash "$RUNNER" kldref --dry-run + [ "$status" -eq 0 ] + grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'ram' +} + +@test "an unknown n-gram placement is refused by name before the boot" { + run env FRANK_QWEN38_NGRAM_PLACEMENT=ssd bash "$RUNNER" calib --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"auto|vram|ram|disk"* ]] +} + +# --- the pusher: a tap without a live rig is not a tap ------------------------ + +# stub_podman — a podman that answers the one question the +# pusher asks: is the rig container running? +stub_podman() { + mkdir -p "$1/bin" + if [ "$2" = running ]; then + printf '#!/usr/bin/env bash\necho true\n' > "$1/bin/podman" + elif [ "$2" = stopped ]; then + printf '#!/usr/bin/env bash\necho false\n' > "$1/bin/podman" + else + printf '#!/usr/bin/env bash\nexit 125\n' > "$1/bin/podman" + fi + chmod +x "$1/bin/podman" +} + +pusher_fixture() { # pusher_fixture + D="$BATS_TEST_TMPDIR/push" + mkdir -p "$D/tap" "$D/bin" + printf '{"prompt":"hello"}\n' > "$D/corpus.jsonl" + if [ -n "$1" ]; then + printf 'container=%s\nimage=radiance@sha256:test\nmode=calib\npid=1\n' "$1" \ + > "$D/tap/.rig-boot" + fi + echo "$D" +} + +@test "the pusher refuses a tap that carries no rig marker: production is not a rig" { + # With no marker, the old driver fed four rounds of 10M tokens into restored + # production after the rig's boot had died, and recorded nothing. + D="$(pusher_fixture '')" + run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" --port 1 + [ "$status" -eq 2 ] + [[ "$output" == *"no rig marker"* ]] +} + +@test "the pusher refuses a marker that names another container" { + D="$(pusher_fixture arbench)" + run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" \ + --container qwen38-bf16 --port 1 + [ "$status" -eq 2 ] + [[ "$output" == *"names container 'arbench'"* ]] + [[ "$output" == *"targets 'qwen38-bf16'"* ]] +} + +@test "the pusher refuses when the rig container is not running" { + D="$(pusher_fixture qwen38-bf16)" + stub_podman "$D" stopped + run env PATH="$D/bin:$PATH" python3 "$PUSHER" --corpus "$D/corpus.jsonl" \ + --tap-dir "$D/tap" --container qwen38-bf16 --port 1 + [ "$status" -eq 2 ] + [[ "$output" == *"is not running"* ]] +} + +# --- the acceptance gate without an FP8 anchor ------------------------------- + +# kld_report_slices +kld_report_slices() { + python3 - "$@" <<'PY' +import json, sys +path, model, kl_all, kl_code, t_all, t_code = sys.argv[1:7] +def g(kl, t, n=25000): + return {"kld": {"mean": float(kl), "p99_9": float(kl) * 4}, + "top1_agreement": float(t), "positions": n} +json.dump({"model": model, "all": g(kl_all, t_all), + "by_source": {"code": g(kl_code, t_code), "ops": g(kl_all, t_all)}}, + open(path, "w")) +PY +} + +gate2() { # gate2 [extra] + D="$BATS_TEST_TMPDIR/kl2" + mkdir -p "$D" + kld_report_slices "$D/iq4r.json" iq4r-moe "$1" "$1" "$3" "$3" + kld_report_slices "$D/cand.json" w5nl64 "$2" "$2" "$4" "$4" + run bash "$GATE" "$D/iq4r.json" "$D/cand.json" "${@:5}" +} + +@test "a candidate that beats the served payload on mean, tail and top-1 passes without an anchor" { + gate2 0.0190 0.0090 0.962 0.971 + [ "$status" -eq 0 ] + [[ "$output" == *"candidate vs served (no FP8 anchor)"* ]] + [[ "$output" == *"KLD GATE PASS"* ]] +} + +@test "a better mean with a worse tail fails: the tail is where a mean hides its damage" { + D="$BATS_TEST_TMPDIR/kl3" + mkdir -p "$D" + kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962 + python3 - "$D/cand.json" <<'PY' +import json, sys +# mean better than the served payload, p99.9 worse: the shape a mean-only gate waves through +g = {"kld": {"mean": 0.0090, "p99_9": 0.4000}, "top1_agreement": 0.971, "positions": 25000} +json.dump({"model": "w5nl64", "all": g, "by_source": {"code": dict(g), "ops": dict(g)}}, + open(sys.argv[1], "w")) +PY + run bash "$GATE" "$D/iq4r.json" "$D/cand.json" + [ "$status" -eq 1 ] + [[ "$output" == *"[FAIL] 2."* ]] + [[ "$output" == *"KL p99.9"* ]] +} + +@test "a candidate that wins overall but regresses one source slice fails" { + D="$BATS_TEST_TMPDIR/kl4" + mkdir -p "$D" + kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962 + # 'all' is better, 'code' is worse: exactly the trade a mean-only gate cannot see + kld_report_slices "$D/cand.json" w5nl64 0.0090 0.0300 0.971 0.950 + run bash "$GATE" "$D/iq4r.json" "$D/cand.json" + [ "$status" -eq 1 ] + [[ "$output" == *"[PASS] 1."* ]] + [[ "$output" == *"[FAIL] 4."* ]] + [[ "$output" == *"regressed: code"* ]] +} + +# --- the calibration window: gated, and it says so --------------------------- + +window_fixture() { + D="$BATS_TEST_TMPDIR/win" + mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora" + echo '{}' > "$D/campaign/kld-ref/kld.json" + : > "$D/rad/qwen38-bf16.rad" + : > "$D/corpora/calib.jsonl" + echo "$D" +} + +@test "the calibration window refuses before touching a card without a reference recording" { + D="$BATS_TEST_TMPDIR/win-less" + mkdir -p "$D/campaign/calib" + run env FRANK_QWEN38_CAMPAIGN="$D/campaign" bash "$WINDOW" + [ "$status" -eq 2 ] + [[ "$output" == *"no reference recording"* ]] +} + +@test "the calibration window's dry run prints the gated plan, including what it would bank" { + D="$(window_fixture)" + run env FRANK_QWEN38_CAMPAIGN="$D/campaign" \ + FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \ + Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" bash "$WINDOW" --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"run-qwen38-bf16.sh calib"* ]] + [[ "$output" == *"else kill the rig"* ]] + [[ "$output" == *"calib-push.py"* ]] + [[ "$output" == *"AGENTTRASH"* ]] +}