scripts/qwen38-calib-window.sh

@@ -40,7 +40,6 @@ MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
 CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
 RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}"
 CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}"
-KLDREF="$CAMPAIGN/kld-ref/kld.json"
 CALIB="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}"
 HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}"
 ROUNDS="${Q38_WINDOW_ROUNDS:-6}"
@@ -48,22 +47,19 @@ RIG=qwen38-bf16
 SERVICE=frank-archon
 PY="${PY:-python3}"
 
-# ---- preflight: everything this window needs, before a card is touched -------
+# ---- preflight: every fact by identity, one owner (scripts/qwen38-preflight.sh) ----
-[ -f "$KLDREF" ] || { echo "REFUSED: no reference recording at $KLDREF — record it first (scripts/run-qwen38-bf16.sh kldref); this window only fills the calibration tap" >&2; exit 2; }
+# The checks that used to live here were proxies, and each lied at least once: the space check
+# asked a directory that merely LOOKED related, `/health` answered for a dying production
+# process, `podman inspect` named a container that existed but had bound nothing. The shared
+# script binds each fact to its object — cwd, the digest the tag actually resolves to, free
+# space on the tap's own filesystem, every /dev/kfd holder's HOST pid against the service's,
+# the :7770 listener's host pid, the corpus and the reference recording. A resumed window
+# keeps an existing tap, so the tap-empty fact is waived.
 [ -f "$RAD" ]    || { echo "REFUSED: no BF16 reference container at $RAD — nothing to serve; scripts/build-qwen38-bf16-rad.sh" >&2; exit 2; }
-[ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; }
+FRANK_QWEN38_CAMPAIGN="$CAMPAIGN" \
-[ -d "$CALIB" ]  || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; }
+Q38_WINDOW_CALIB_CORPUS="$CORPUS" \
-[ -w "$CALIB" ]  || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; }
+Q38_CALIB_MIN_FREE_MIB="${Q38_CALIB_MIN_FREE_MIB:-61440}" \
-# One tap seal writes ~40 GiB (measured 2026-10-30). The runner refuses too; this
+    bash scripts/qwen38-preflight.sh --allow-existing-tap || exit 2
-# names the same number before a card is touched, so a full volume never becomes
-# a 17-hour dead run.
-avail_mib=$(df --output=avail -B 1M "$CALIB" 2>/dev/null | tail -1 | tr -d ' ')
-[[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0
-need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}"
-[ "$avail_mib" -ge "$need_mib" ] || {
-    echo "REFUSED: tap dir $CALIB has ${avail_mib} MiB free; one seal writes ~40 GiB and this window needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2
-    exit 2
-}
 ! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; }
 ! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; }
 

scripts/qwen38-preflight.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+# qwen38-preflight.sh — the facts a calibration window must have BEFORE it takes the cards,
+# each checked by IDENTITY and not by proxy. Every line is PASS or FAIL with the reason.
+#
+# Why this file exists: on 2026-10-30 a window died at second zero because `/health` answered
+# 200 from the *production* process that was still draining its listener, while the freshly
+# created rig container had not bound anything. Other runs died from a port that answered for
+# the wrong process, a /dev/kfd held by an orphaned engine, a tap dir on a 35 GiB filesystem,
+# and a launcher whose cwd silently lacked the repo. None of those were math problems; all of
+# them were a check that measured something adjacent to the thing it meant.
+#
+# Usage:  bash scripts/qwen38-preflight.sh [--allow-existing-tap]
+# Exit:   0 all PASS · 2 any FAIL · 3 a fact could not be evaluated (missing tool)
+set -uo pipefail
+
+SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SELF_DIR/.." && pwd)"
+
+CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
+TAP="$CAMPAIGN/calib"
+CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}"
+NEED_MIB="${Q38_CALIB_MIN_FREE_MIB:-61440}"
+ALLOW_EXISTING=0
+[ "${1:-}" = "--allow-existing-tap" ] && ALLOW_EXISTING=1
+
+# The producer pin is the calib image; the window fails closed if the tag on this host is not it.
+WANT_DIGEST="${Q38_PREFLIGHT_IMAGE_DIGEST:-sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}"
+IMAGE_REF="${Q38_CALIB_IMAGE:-stilldeadcode/radiance@$WANT_DIGEST}"
+SERVICE="frank-archon"
+PORT=7770
+
+fails=0
+ok()   { printf 'PASS  %-14s %s\n' "$1" "$2"; }
+bad()  { printf 'FAIL  %-14s %s\n' "$1" "$2"; fails=$((fails + 1)); }
+skip() { printf 'WARN  %-14s %s\n' "$1" "$2"; }
+
+# 1. cwd — a detached launcher that lost its cwd fails here in a second, not in the log later.
+if [ "$(cd "$REPO_ROOT" && pwd)" = "$(pwd)" ] && [ -x "$REPO_ROOT/scripts/run-qwen38-bf16.sh" ]; then
+    ok cwd "running from the repo root ($REPO_ROOT)"
+else
+    bad cwd "cwd is $(pwd), not the repo root ($REPO_ROOT) — every relative script path would break"
+fi
+
+# 2. image — resolve the REF to a digest. A stale local `:latest` is a different model of the world.
+if command -v podman >/dev/null 2>&1; then
+    got="$(podman image inspect -f '{{index .RepoDigests 0}}' "$IMAGE_REF" 2>/dev/null | sed 's/.*@//')"
+    if [ -z "$got" ]; then
+        bad image "$IMAGE_REF is not present on this host (podman has no digest for it)"
+    elif [ "$got" = "$WANT_DIGEST" ]; then
+        ok image "$IMAGE_REF resolves to ${got:0:19}…"
+    else
+        bad image "$IMAGE_REF resolves to $got, expected $WANT_DIGEST"
+    fi
+else
+    skip image "podman not on PATH — cannot resolve $IMAGE_REF"
+fi
+
+# 3. space — df the FILE PATH the seal writes to, never a directory that merely looks related.
+if [ -d "$TAP" ] || mkdir -p "$TAP" 2>/dev/null; then
+    avail="$(df --output=avail -B 1M "$TAP" 2>/dev/null | tail -1 | tr -d ' ')"
+    if [[ "${avail:-}" =~ ^[0-9]+$ ]] && [ "$avail" -ge "$NEED_MIB" ]; then
+        ok space "$avail MiB free on $(df --output=target "$TAP" | tail -1) (need $NEED_MIB)"
+    else
+        bad space "${avail:-unknown} MiB free on $TAP, need $NEED_MIB — one seal writes ~43 GB"
+    fi
+    # 4. tap cleanliness — stale grams from another run must never be mixed in.
+    n=$(ls -A "$TAP" 2>/dev/null | grep -c . || true)
+    if [ "$n" -eq 0 ] || [ "$ALLOW_EXISTING" = 1 ]; then
+        ok tap-empty "$n entries in $TAP"
+    else
+        bad tap-empty "$n entries in $TAP — rotate them to .AGENTTRASH first"
+    fi
+else
+    bad space "cannot create $TAP"
+fi
+
+# 5. device — every /dev/kfd holder must be a process of the service we are about to stop. An
+#    orphaned engine from a previous run makes the next boot die with `startup stopped: device
+#    error`. Identity comes from podman's own host-pid list: `podman top <ctr> pid` prints
+#    CONTAINER-internal pids (1, 7, …), which compare against nothing, and a rootless cgroup
+#    carries the container ID, never the service name — both are proxies that lie.
+PROD_CTR="${Q38_PREFLIGHT_PROD_CONTAINER:-$SERVICE}"
+service_pids() { podman top "$PROD_CTR" hpid 2>/dev/null | tail -n +2; }
+if [ -e /dev/kfd ]; then
+    mine="$(service_pids | tr '\n' ' ')"
+    foreign=""
+    for p in $(fuser /dev/kfd 2>/dev/null); do
+        case " $mine " in
+            *" $p "*) : ;;                                          # the service we will stop
+            *) foreign="$foreign $p:$(cat /proc/$p/comm 2>/dev/null)" ;;
+        esac
+    done
+    if [ -z "$foreign" ]; then
+        ok device "no foreign /dev/kfd holders"
+    else
+        bad device "foreign holders:$foreign — kill by pid, never by pattern"
+    fi
+else
+    skip device "/dev/kfd absent — not an AMD host?"
+fi
+
+# 6. port — a listener on :7770 must be a process of the service we stop, never an unknown one.
+lpid="$(ss -H -ltnp "sport = :$PORT" 2>/dev/null | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2)"
+if [ -z "${lpid:-}" ]; then
+    ok port ":$PORT is free"
+else
+    mine="$(service_pids | tr '\n' ' ')"
+    case " $mine " in
+        *" $lpid "*) ok port ":$PORT is held by $SERVICE (host pid $lpid) — the window stops it" ;;
+        *)           bad port ":$PORT is held by host pid $lpid ($(cat /proc/$lpid/comm 2>/dev/null)), not $SERVICE" ;;
+    esac
+fi
+
+# 7. corpus — the push source must exist and be the re-chunked file the window documents.
+if [ -s "$CORPUS" ]; then
+    ok corpus "$(wc -l < "$CORPUS") prompts in $CORPUS"
+else
+    bad corpus "$CORPUS is missing or empty"
+fi
+
+# 8. reference — the KL gate has nothing to compare against without it.
+if [ -s "$CAMPAIGN/kld-ref/kld.json" ]; then
+    ok reference "$(du -h "$CAMPAIGN/kld-ref/kld.json" | cut -f1) kld.json present"
+else
+    bad reference "$CAMPAIGN/kld-ref/kld.json is missing"
+fi
+
+echo
+if [ "$fails" -gt 0 ]; then
+    echo "PREFLIGHT FAILED: $fails fact(s). The window must not take the cards."
+    exit 2
+fi
+echo "PREFLIGHT PASSED: every fact checked by identity."

serverAppConfigs-docs/LLM/frank-archon.md

@@ -571,6 +571,19 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
   of falling through its health wait. All three exist because on 2026-10-29 the unguarded chain's
   rig boot died of an OOM, the health loop expired without failing, and the pusher fed four rounds
   of 10M tokens into **restored production** — 86 minutes of live prefill that recorded nothing.
+- **A container that exists, a port that answers, and a `/health` 200 are three different claims —
+  bind identity before believing any of them.** Measured 2026-10-30: `podman inspect` named the rig
+  while the port still belonged to the *dying* production process (`health ok waited=0s`, window dead
+  at second zero), and the identity check written to catch that compared two different number spaces:
+  `podman top <ctr> pid` prints **container-internal** pids (`1`, `7`), never host pids, and a rootless
+  cgroup carries the container ID rather than the service name — so both "matches" can only ever fail.
+  The binding is `podman top <ctr> hpid` against the listener from
+  `ss -H -ltnp 'sport = :7770'`, and `fuser /dev/kfd` likewise resolved through podman.
+  `scripts/qwen38-preflight.sh` owns those facts now (cwd, the digest a tag actually resolves to,
+  free space on the tap's **own** filesystem, stale grams, every GPU holder and the listener by host
+  pid), prints `PASS`/`FAIL` per fact, and `qwen38-calib-window.sh` calls it before a card is touched.
+  Corollary for every ad-hoc driver: a detached launcher that loses its cwd fails with
+  `No such file or directory` in a log nobody reads — assert cwd, and never bypass the orchestrator.
 - **The n-gram table's pin is what starves a rig boot, and the tier is a knob.** On the BF16 payload
   the table is 95.37 GiB, `--ngram-placement ram` pins it, and the calibration boot died at
   `startup stopped: out of memory` with 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available —

tests/qwen38_campaign.bats

@@ -673,24 +673,41 @@ PY
 
 window_fixture() {
     D="$BATS_TEST_TMPDIR/win"
-    mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora"
+    mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora" "$D/bin"
     echo '{}' > "$D/campaign/kld-ref/kld.json"
     : > "$D/rad/qwen38-bf16.rad"
-    : > "$D/corpora/calib.jsonl"
+    echo "{\"p\":1}" > "$D/corpora/calib.jsonl"
+    # the window's preflight resolves the image digest through podman and asks about the
+    # port/device by host pid; these stubs answer for a quiet host
+    cat > "$D/bin/podman" <<'EOS'
+#!/usr/bin/env bash
+case "${1:-}" in
+    image) echo "sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093" ;;
+esac
+EOS
+    printf '#!/usr/bin/env bash\nexit 0\n' > "$D/bin/ss"
+    printf '#!/usr/bin/env bash\nexit 0\n' > "$D/bin/fuser"
+    chmod +x "$D/bin/podman" "$D/bin/ss" "$D/bin/fuser"
     echo "$D"
 }
 
 @test "the calibration window refuses before touching a card without a reference recording" {
-    D="$BATS_TEST_TMPDIR/win-less"
+    D="$(window_fixture)"
-    mkdir -p "$D/campaign/calib"
+    mv "$D/campaign/kld-ref/kld.json" "$D/campaign/kld-ref/kld.json.absent"
-    run env FRANK_QWEN38_CAMPAIGN="$D/campaign" bash "$WINDOW"
+    run env PATH="$D/bin:$PATH" \
+        FRANK_QWEN38_CAMPAIGN="$D/campaign" \
+        FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \
+        Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" \
+        Q38_CALIB_MIN_FREE_MIB=1 bash "$WINDOW"
     [ "$status" -eq 2 ]
-    [[ "$output" == *"no reference recording"* ]]
+    [[ "$output" == *"FAIL  reference"* ]]
+    [[ "$output" == *"must not take the cards"* ]]
 }
 
 @test "the calibration window's dry run prints the gated plan, including what it would bank" {
     D="$(window_fixture)"
-    run env FRANK_QWEN38_CAMPAIGN="$D/campaign" \
+    run env PATH="$D/bin:$PATH" \
+        FRANK_QWEN38_CAMPAIGN="$D/campaign" \
         FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \
         Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" \
         Q38_CALIB_MIN_FREE_MIB=1 bash "$WINDOW" --dry-run
@@ -743,9 +760,10 @@ tap_window_fixture() {
     export W="$BATS_TEST_TMPDIR/window"
     mkdir -p "$W/scripts" "$W/campaign/calib" "$W/campaign/kld-ref" "$W/models/rad" "$W/corpora" "$W/bin"
     cp "$BATS_TEST_DIRNAME/../scripts/qwen38-calib-window.sh" "$W/scripts/"
-    : > "$W/campaign/kld-ref/kld.json"
+    cp "$BATS_TEST_DIRNAME/../scripts/qwen38-preflight.sh" "$W/scripts/"
+    echo "{}" > "$W/campaign/kld-ref/kld.json"
     : > "$W/models/rad/qwen38-bf16.rad"
-    : > "$W/corpora/calib.jsonl"
+    echo "{\"p\":1}" > "$W/corpora/calib.jsonl"
     # a gram so the empty-tap check passes on the positive path
     : > "$W/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin"
 
@@ -764,10 +782,20 @@ EOS
 #!/usr/bin/env bash
 case "${1:-}" in
     inspect) echo true ;;
-    top)     echo PID; echo "${RIG_PID:-424242}" ;;
+    # the PREFLIGHT asks who owns :7770 (production, per LISTEN_PID); the WINDOW asks which
+    # pids belong to the rig (RIG_PID). Both stubs are the same binary, so keep them apart.
+    image)   echo "sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093" ;;
+    top)     echo PID
+             # the WINDOW asks "which pids are the rig's", the PREFLIGHT asks "which are
+             # production's". One stub, two answers: that separation IS the identity lesson.
+             case "${2:-}" in
+                 qwen38-bf16) echo "${RIG_PID:-424242}" ;;
+                 *)           echo "${LISTEN_PID:-424242}" ;;
+             esac ;;
     *)       : ;;
 esac
 EOS
+    printf '#!/usr/bin/env bash\nexit 0\n' > "$W/bin/fuser"
     cat > "$W/bin/ss" <<'EOS'
 #!/usr/bin/env bash
 echo "LISTEN 0 4096 *:7770 *:* users:((\"radiance\",pid=${LISTEN_PID:-424242},fd=9))"
@@ -776,7 +804,7 @@ EOS
 #!/usr/bin/env bash
 exit 0
 EOS
-    chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl"
+    chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl" "$W/bin/fuser"
 }
 
 tap_window_env() {
@@ -811,3 +839,98 @@ tap_window_env() {
     [ -e "$W/push.ran" ]
     [[ "$output" == *"TAP BANKED"* ]]
 }
+
+# ---- the preflight: every fact by IDENTITY, and it must fail loudly -------------------------
+# Each case here is a lie that cost a real window on 2026-10-30.
+
+preflight_fixture() {
+    export PF="$BATS_TEST_TMPDIR/pf"
+    mkdir -p "$PF/campaign/calib" "$PF/campaign/kld-ref" "$PF/bin"
+    echo '{}' > "$PF/campaign/kld-ref/kld.json"
+    echo '{"p":1}' > "$PF/corpus.jsonl"
+    cat > "$PF/bin/podman" <<'EOS'
+#!/usr/bin/env bash
+case "$1" in
+  image) echo "${STUB_IMAGE_DIGEST:-sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}" ;;
+  top)   echo HPID; echo "${STUB_PROD_PIDS:-4242}" ;;
+esac
+EOS
+    cat > "$PF/bin/ss" <<'EOS'
+#!/usr/bin/env bash
+[ -n "${STUB_LISTENER_PID:-}" ] && echo "LISTEN 0 4096 *:7770 *:* users:((\"x\",pid=${STUB_LISTENER_PID},fd=9))"
+exit 0
+EOS
+    cat > "$PF/bin/fuser" <<'EOS'
+#!/usr/bin/env bash
+[ -n "${STUB_KFD_PID:-}" ] && echo "${STUB_KFD_PID}"
+exit 0
+EOS
+    chmod +x "$PF/bin/podman" "$PF/bin/ss" "$PF/bin/fuser"
+}
+
+run_preflight() {
+    ( cd "$BATS_TEST_DIRNAME/.." && env PATH="$PF/bin:$PATH" \
+        FRANK_QWEN38_CAMPAIGN="$PF/campaign" \
+        Q38_WINDOW_CALIB_CORPUS="$PF/corpus.jsonl" \
+        Q38_CALIB_MIN_FREE_MIB=1 \
+        bash scripts/qwen38-preflight.sh "$@" )
+}
+
+@test "preflight passes when every fact is true, and says so" {
+    preflight_fixture
+    run run_preflight
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"PREFLIGHT PASSED"* ]]
+    [[ "$output" == *"resolves to sha256:69add58f"* ]]
+}
+
+@test "preflight accepts the service's own process on the port, not just any answer" {
+    preflight_fixture
+    export STUB_PROD_PIDS=4242 STUB_LISTENER_PID=4242 STUB_KFD_PID=4242
+    run run_preflight
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"held by frank-archon (host pid 4242)"* ]]
+}
+
+@test "preflight fails when the tag resolves to a different digest" {
+    preflight_fixture
+    export STUB_IMAGE_DIGEST="sha256:deadbeefdeadbeef"
+    run run_preflight
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"expected sha256:69add58f"* ]]
+}
+
+@test "preflight fails when :7770 belongs to a host pid the service does not own" {
+    preflight_fixture
+    export STUB_LISTENER_PID=999999
+    run run_preflight
+    [ "$status" -eq 2 ]
+    [[ "$output" == *":7770 is held by host pid 999999"* ]]
+}
+
+@test "preflight fails when another process holds the GPU" {
+    preflight_fixture
+    export STUB_KFD_PID=999998
+    run run_preflight
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"foreign holders:"* ]]
+}
+
+@test "preflight fails when the tap dir already holds grams from another run" {
+    preflight_fixture
+    : > "$PF/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin"
+    run run_preflight
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"rotate them to .AGENTTRASH"* ]]
+    run run_preflight --allow-existing-tap
+    [ "$status" -eq 0 ]
+}
+
+@test "preflight fails when the launcher is not in the repo root" {
+    preflight_fixture
+    run env PATH="$PF/bin:$PATH" FRANK_QWEN38_CAMPAIGN="$PF/campaign" \
+        Q38_WINDOW_CALIB_CORPUS="$PF/corpus.jsonl" Q38_CALIB_MIN_FREE_MIB=1 \
+        bash -c "cd /tmp && bash '$BATS_TEST_DIRNAME/../scripts/qwen38-preflight.sh'"
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"not the repo root"* ]]
+}