diff --git a/scripts/qwen38-calib-window.sh b/scripts/qwen38-calib-window.sh index e1d9993..3e9769e 100755 --- a/scripts/qwen38-calib-window.sh +++ b/scripts/qwen38-calib-window.sh @@ -40,7 +40,6 @@ MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}" CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}" RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}" CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}" -KLDREF="$CAMPAIGN/kld-ref/kld.json" CALIB="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}" HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}" ROUNDS="${Q38_WINDOW_ROUNDS:-6}" @@ -48,22 +47,19 @@ RIG=qwen38-bf16 SERVICE=frank-archon PY="${PY:-python3}" -# ---- preflight: everything this window needs, before a card is touched ------- -[ -f "$KLDREF" ] || { echo "REFUSED: no reference recording at $KLDREF — record it first (scripts/run-qwen38-bf16.sh kldref); this window only fills the calibration tap" >&2; exit 2; } +# ---- preflight: every fact by identity, one owner (scripts/qwen38-preflight.sh) ---- +# The checks that used to live here were proxies, and each lied at least once: the space check +# asked a directory that merely LOOKED related, `/health` answered for a dying production +# process, `podman inspect` named a container that existed but had bound nothing. The shared +# script binds each fact to its object — cwd, the digest the tag actually resolves to, free +# space on the tap's own filesystem, every /dev/kfd holder's HOST pid against the service's, +# the :7770 listener's host pid, the corpus and the reference recording. A resumed window +# keeps an existing tap, so the tap-empty fact is waived. [ -f "$RAD" ] || { echo "REFUSED: no BF16 reference container at $RAD — nothing to serve; scripts/build-qwen38-bf16-rad.sh" >&2; exit 2; } -[ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; } -[ -d "$CALIB" ] || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; } -[ -w "$CALIB" ] || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; } -# One tap seal writes ~40 GiB (measured 2026-10-30). The runner refuses too; this -# names the same number before a card is touched, so a full volume never becomes -# a 17-hour dead run. -avail_mib=$(df --output=avail -B 1M "$CALIB" 2>/dev/null | tail -1 | tr -d ' ') -[[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0 -need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}" -[ "$avail_mib" -ge "$need_mib" ] || { - echo "REFUSED: tap dir $CALIB has ${avail_mib} MiB free; one seal writes ~40 GiB and this window needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2 - exit 2 -} +FRANK_QWEN38_CAMPAIGN="$CAMPAIGN" \ +Q38_WINDOW_CALIB_CORPUS="$CORPUS" \ +Q38_CALIB_MIN_FREE_MIB="${Q38_CALIB_MIN_FREE_MIB:-61440}" \ + bash scripts/qwen38-preflight.sh --allow-existing-tap || exit 2 ! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; } ! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; } diff --git a/scripts/qwen38-preflight.sh b/scripts/qwen38-preflight.sh new file mode 100755 index 0000000..9de2b99 --- /dev/null +++ b/scripts/qwen38-preflight.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# qwen38-preflight.sh — the facts a calibration window must have BEFORE it takes the cards, +# each checked by IDENTITY and not by proxy. Every line is PASS or FAIL with the reason. +# +# Why this file exists: on 2026-10-30 a window died at second zero because `/health` answered +# 200 from the *production* process that was still draining its listener, while the freshly +# created rig container had not bound anything. Other runs died from a port that answered for +# the wrong process, a /dev/kfd held by an orphaned engine, a tap dir on a 35 GiB filesystem, +# and a launcher whose cwd silently lacked the repo. None of those were math problems; all of +# them were a check that measured something adjacent to the thing it meant. +# +# Usage: bash scripts/qwen38-preflight.sh [--allow-existing-tap] +# Exit: 0 all PASS · 2 any FAIL · 3 a fact could not be evaluated (missing tool) +set -uo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SELF_DIR/.." && pwd)" + +CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}" +TAP="$CAMPAIGN/calib" +CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}" +NEED_MIB="${Q38_CALIB_MIN_FREE_MIB:-61440}" +ALLOW_EXISTING=0 +[ "${1:-}" = "--allow-existing-tap" ] && ALLOW_EXISTING=1 + +# The producer pin is the calib image; the window fails closed if the tag on this host is not it. +WANT_DIGEST="${Q38_PREFLIGHT_IMAGE_DIGEST:-sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}" +IMAGE_REF="${Q38_CALIB_IMAGE:-stilldeadcode/radiance@$WANT_DIGEST}" +SERVICE="frank-archon" +PORT=7770 + +fails=0 +ok() { printf 'PASS %-14s %s\n' "$1" "$2"; } +bad() { printf 'FAIL %-14s %s\n' "$1" "$2"; fails=$((fails + 1)); } +skip() { printf 'WARN %-14s %s\n' "$1" "$2"; } + +# 1. cwd — a detached launcher that lost its cwd fails here in a second, not in the log later. +if [ "$(cd "$REPO_ROOT" && pwd)" = "$(pwd)" ] && [ -x "$REPO_ROOT/scripts/run-qwen38-bf16.sh" ]; then + ok cwd "running from the repo root ($REPO_ROOT)" +else + bad cwd "cwd is $(pwd), not the repo root ($REPO_ROOT) — every relative script path would break" +fi + +# 2. image — resolve the REF to a digest. A stale local `:latest` is a different model of the world. +if command -v podman >/dev/null 2>&1; then + got="$(podman image inspect -f '{{index .RepoDigests 0}}' "$IMAGE_REF" 2>/dev/null | sed 's/.*@//')" + if [ -z "$got" ]; then + bad image "$IMAGE_REF is not present on this host (podman has no digest for it)" + elif [ "$got" = "$WANT_DIGEST" ]; then + ok image "$IMAGE_REF resolves to ${got:0:19}…" + else + bad image "$IMAGE_REF resolves to $got, expected $WANT_DIGEST" + fi +else + skip image "podman not on PATH — cannot resolve $IMAGE_REF" +fi + +# 3. space — df the FILE PATH the seal writes to, never a directory that merely looks related. +if [ -d "$TAP" ] || mkdir -p "$TAP" 2>/dev/null; then + avail="$(df --output=avail -B 1M "$TAP" 2>/dev/null | tail -1 | tr -d ' ')" + if [[ "${avail:-}" =~ ^[0-9]+$ ]] && [ "$avail" -ge "$NEED_MIB" ]; then + ok space "$avail MiB free on $(df --output=target "$TAP" | tail -1) (need $NEED_MIB)" + else + bad space "${avail:-unknown} MiB free on $TAP, need $NEED_MIB — one seal writes ~43 GB" + fi + # 4. tap cleanliness — stale grams from another run must never be mixed in. + n=$(ls -A "$TAP" 2>/dev/null | grep -c . || true) + if [ "$n" -eq 0 ] || [ "$ALLOW_EXISTING" = 1 ]; then + ok tap-empty "$n entries in $TAP" + else + bad tap-empty "$n entries in $TAP — rotate them to .AGENTTRASH first" + fi +else + bad space "cannot create $TAP" +fi + +# 5. device — every /dev/kfd holder must be a process of the service we are about to stop. An +# orphaned engine from a previous run makes the next boot die with `startup stopped: device +# error`. Identity comes from podman's own host-pid list: `podman top pid` prints +# CONTAINER-internal pids (1, 7, …), which compare against nothing, and a rootless cgroup +# carries the container ID, never the service name — both are proxies that lie. +PROD_CTR="${Q38_PREFLIGHT_PROD_CONTAINER:-$SERVICE}" +service_pids() { podman top "$PROD_CTR" hpid 2>/dev/null | tail -n +2; } +if [ -e /dev/kfd ]; then + mine="$(service_pids | tr '\n' ' ')" + foreign="" + for p in $(fuser /dev/kfd 2>/dev/null); do + case " $mine " in + *" $p "*) : ;; # the service we will stop + *) foreign="$foreign $p:$(cat /proc/$p/comm 2>/dev/null)" ;; + esac + done + if [ -z "$foreign" ]; then + ok device "no foreign /dev/kfd holders" + else + bad device "foreign holders:$foreign — kill by pid, never by pattern" + fi +else + skip device "/dev/kfd absent — not an AMD host?" +fi + +# 6. port — a listener on :7770 must be a process of the service we stop, never an unknown one. +lpid="$(ss -H -ltnp "sport = :$PORT" 2>/dev/null | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2)" +if [ -z "${lpid:-}" ]; then + ok port ":$PORT is free" +else + mine="$(service_pids | tr '\n' ' ')" + case " $mine " in + *" $lpid "*) ok port ":$PORT is held by $SERVICE (host pid $lpid) — the window stops it" ;; + *) bad port ":$PORT is held by host pid $lpid ($(cat /proc/$lpid/comm 2>/dev/null)), not $SERVICE" ;; + esac +fi + +# 7. corpus — the push source must exist and be the re-chunked file the window documents. +if [ -s "$CORPUS" ]; then + ok corpus "$(wc -l < "$CORPUS") prompts in $CORPUS" +else + bad corpus "$CORPUS is missing or empty" +fi + +# 8. reference — the KL gate has nothing to compare against without it. +if [ -s "$CAMPAIGN/kld-ref/kld.json" ]; then + ok reference "$(du -h "$CAMPAIGN/kld-ref/kld.json" | cut -f1) kld.json present" +else + bad reference "$CAMPAIGN/kld-ref/kld.json is missing" +fi + +echo +if [ "$fails" -gt 0 ]; then + echo "PREFLIGHT FAILED: $fails fact(s). The window must not take the cards." + exit 2 +fi +echo "PREFLIGHT PASSED: every fact checked by identity." diff --git a/serverAppConfigs-docs/LLM/frank-archon.md b/serverAppConfigs-docs/LLM/frank-archon.md index c1442a5..53d2b69 100644 --- a/serverAppConfigs-docs/LLM/frank-archon.md +++ b/serverAppConfigs-docs/LLM/frank-archon.md @@ -571,6 +571,19 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10). of falling through its health wait. All three exist because on 2026-10-29 the unguarded chain's rig boot died of an OOM, the health loop expired without failing, and the pusher fed four rounds of 10M tokens into **restored production** — 86 minutes of live prefill that recorded nothing. +- **A container that exists, a port that answers, and a `/health` 200 are three different claims — + bind identity before believing any of them.** Measured 2026-10-30: `podman inspect` named the rig + while the port still belonged to the *dying* production process (`health ok waited=0s`, window dead + at second zero), and the identity check written to catch that compared two different number spaces: + `podman top pid` prints **container-internal** pids (`1`, `7`), never host pids, and a rootless + cgroup carries the container ID rather than the service name — so both "matches" can only ever fail. + The binding is `podman top hpid` against the listener from + `ss -H -ltnp 'sport = :7770'`, and `fuser /dev/kfd` likewise resolved through podman. + `scripts/qwen38-preflight.sh` owns those facts now (cwd, the digest a tag actually resolves to, + free space on the tap's **own** filesystem, stale grams, every GPU holder and the listener by host + pid), prints `PASS`/`FAIL` per fact, and `qwen38-calib-window.sh` calls it before a card is touched. + Corollary for every ad-hoc driver: a detached launcher that loses its cwd fails with + `No such file or directory` in a log nobody reads — assert cwd, and never bypass the orchestrator. - **The n-gram table's pin is what starves a rig boot, and the tier is a knob.** On the BF16 payload the table is 95.37 GiB, `--ngram-placement ram` pins it, and the calibration boot died at `startup stopped: out of memory` with 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available — diff --git a/tests/qwen38_campaign.bats b/tests/qwen38_campaign.bats index 348ad24..c6dc3e5 100755 --- a/tests/qwen38_campaign.bats +++ b/tests/qwen38_campaign.bats @@ -673,24 +673,41 @@ PY window_fixture() { D="$BATS_TEST_TMPDIR/win" - mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora" + mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora" "$D/bin" echo '{}' > "$D/campaign/kld-ref/kld.json" : > "$D/rad/qwen38-bf16.rad" - : > "$D/corpora/calib.jsonl" + echo "{\"p\":1}" > "$D/corpora/calib.jsonl" + # the window's preflight resolves the image digest through podman and asks about the + # port/device by host pid; these stubs answer for a quiet host + cat > "$D/bin/podman" <<'EOS' +#!/usr/bin/env bash +case "${1:-}" in + image) echo "sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093" ;; +esac +EOS + printf '#!/usr/bin/env bash\nexit 0\n' > "$D/bin/ss" + printf '#!/usr/bin/env bash\nexit 0\n' > "$D/bin/fuser" + chmod +x "$D/bin/podman" "$D/bin/ss" "$D/bin/fuser" echo "$D" } @test "the calibration window refuses before touching a card without a reference recording" { - D="$BATS_TEST_TMPDIR/win-less" - mkdir -p "$D/campaign/calib" - run env FRANK_QWEN38_CAMPAIGN="$D/campaign" bash "$WINDOW" + D="$(window_fixture)" + mv "$D/campaign/kld-ref/kld.json" "$D/campaign/kld-ref/kld.json.absent" + run env PATH="$D/bin:$PATH" \ + FRANK_QWEN38_CAMPAIGN="$D/campaign" \ + FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \ + Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" \ + Q38_CALIB_MIN_FREE_MIB=1 bash "$WINDOW" [ "$status" -eq 2 ] - [[ "$output" == *"no reference recording"* ]] + [[ "$output" == *"FAIL reference"* ]] + [[ "$output" == *"must not take the cards"* ]] } @test "the calibration window's dry run prints the gated plan, including what it would bank" { D="$(window_fixture)" - run env FRANK_QWEN38_CAMPAIGN="$D/campaign" \ + run env PATH="$D/bin:$PATH" \ + FRANK_QWEN38_CAMPAIGN="$D/campaign" \ FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \ Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" \ Q38_CALIB_MIN_FREE_MIB=1 bash "$WINDOW" --dry-run @@ -743,9 +760,10 @@ tap_window_fixture() { export W="$BATS_TEST_TMPDIR/window" mkdir -p "$W/scripts" "$W/campaign/calib" "$W/campaign/kld-ref" "$W/models/rad" "$W/corpora" "$W/bin" cp "$BATS_TEST_DIRNAME/../scripts/qwen38-calib-window.sh" "$W/scripts/" - : > "$W/campaign/kld-ref/kld.json" + cp "$BATS_TEST_DIRNAME/../scripts/qwen38-preflight.sh" "$W/scripts/" + echo "{}" > "$W/campaign/kld-ref/kld.json" : > "$W/models/rad/qwen38-bf16.rad" - : > "$W/corpora/calib.jsonl" + echo "{\"p\":1}" > "$W/corpora/calib.jsonl" # a gram so the empty-tap check passes on the positive path : > "$W/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin" @@ -764,10 +782,20 @@ EOS #!/usr/bin/env bash case "${1:-}" in inspect) echo true ;; - top) echo PID; echo "${RIG_PID:-424242}" ;; + # the PREFLIGHT asks who owns :7770 (production, per LISTEN_PID); the WINDOW asks which + # pids belong to the rig (RIG_PID). Both stubs are the same binary, so keep them apart. + image) echo "sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093" ;; + top) echo PID + # the WINDOW asks "which pids are the rig's", the PREFLIGHT asks "which are + # production's". One stub, two answers: that separation IS the identity lesson. + case "${2:-}" in + qwen38-bf16) echo "${RIG_PID:-424242}" ;; + *) echo "${LISTEN_PID:-424242}" ;; + esac ;; *) : ;; esac EOS + printf '#!/usr/bin/env bash\nexit 0\n' > "$W/bin/fuser" cat > "$W/bin/ss" <<'EOS' #!/usr/bin/env bash echo "LISTEN 0 4096 *:7770 *:* users:((\"radiance\",pid=${LISTEN_PID:-424242},fd=9))" @@ -776,7 +804,7 @@ EOS #!/usr/bin/env bash exit 0 EOS - chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl" + chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl" "$W/bin/fuser" } tap_window_env() { @@ -811,3 +839,98 @@ tap_window_env() { [ -e "$W/push.ran" ] [[ "$output" == *"TAP BANKED"* ]] } + +# ---- the preflight: every fact by IDENTITY, and it must fail loudly ------------------------- +# Each case here is a lie that cost a real window on 2026-10-30. + +preflight_fixture() { + export PF="$BATS_TEST_TMPDIR/pf" + mkdir -p "$PF/campaign/calib" "$PF/campaign/kld-ref" "$PF/bin" + echo '{}' > "$PF/campaign/kld-ref/kld.json" + echo '{"p":1}' > "$PF/corpus.jsonl" + cat > "$PF/bin/podman" <<'EOS' +#!/usr/bin/env bash +case "$1" in + image) echo "${STUB_IMAGE_DIGEST:-sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}" ;; + top) echo HPID; echo "${STUB_PROD_PIDS:-4242}" ;; +esac +EOS + cat > "$PF/bin/ss" <<'EOS' +#!/usr/bin/env bash +[ -n "${STUB_LISTENER_PID:-}" ] && echo "LISTEN 0 4096 *:7770 *:* users:((\"x\",pid=${STUB_LISTENER_PID},fd=9))" +exit 0 +EOS + cat > "$PF/bin/fuser" <<'EOS' +#!/usr/bin/env bash +[ -n "${STUB_KFD_PID:-}" ] && echo "${STUB_KFD_PID}" +exit 0 +EOS + chmod +x "$PF/bin/podman" "$PF/bin/ss" "$PF/bin/fuser" +} + +run_preflight() { + ( cd "$BATS_TEST_DIRNAME/.." && env PATH="$PF/bin:$PATH" \ + FRANK_QWEN38_CAMPAIGN="$PF/campaign" \ + Q38_WINDOW_CALIB_CORPUS="$PF/corpus.jsonl" \ + Q38_CALIB_MIN_FREE_MIB=1 \ + bash scripts/qwen38-preflight.sh "$@" ) +} + +@test "preflight passes when every fact is true, and says so" { + preflight_fixture + run run_preflight + [ "$status" -eq 0 ] + [[ "$output" == *"PREFLIGHT PASSED"* ]] + [[ "$output" == *"resolves to sha256:69add58f"* ]] +} + +@test "preflight accepts the service's own process on the port, not just any answer" { + preflight_fixture + export STUB_PROD_PIDS=4242 STUB_LISTENER_PID=4242 STUB_KFD_PID=4242 + run run_preflight + [ "$status" -eq 0 ] + [[ "$output" == *"held by frank-archon (host pid 4242)"* ]] +} + +@test "preflight fails when the tag resolves to a different digest" { + preflight_fixture + export STUB_IMAGE_DIGEST="sha256:deadbeefdeadbeef" + run run_preflight + [ "$status" -eq 2 ] + [[ "$output" == *"expected sha256:69add58f"* ]] +} + +@test "preflight fails when :7770 belongs to a host pid the service does not own" { + preflight_fixture + export STUB_LISTENER_PID=999999 + run run_preflight + [ "$status" -eq 2 ] + [[ "$output" == *":7770 is held by host pid 999999"* ]] +} + +@test "preflight fails when another process holds the GPU" { + preflight_fixture + export STUB_KFD_PID=999998 + run run_preflight + [ "$status" -eq 2 ] + [[ "$output" == *"foreign holders:"* ]] +} + +@test "preflight fails when the tap dir already holds grams from another run" { + preflight_fixture + : > "$PF/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin" + run run_preflight + [ "$status" -eq 2 ] + [[ "$output" == *"rotate them to .AGENTTRASH"* ]] + run run_preflight --allow-existing-tap + [ "$status" -eq 0 ] +} + +@test "preflight fails when the launcher is not in the repo root" { + preflight_fixture + run env PATH="$PF/bin:$PATH" FRANK_QWEN38_CAMPAIGN="$PF/campaign" \ + Q38_WINDOW_CALIB_CORPUS="$PF/corpus.jsonl" Q38_CALIB_MIN_FREE_MIB=1 \ + bash -c "cd /tmp && bash '$BATS_TEST_DIRNAME/../scripts/qwen38-preflight.sh'" + [ "$status" -eq 2 ] + [[ "$output" == *"not the repo root"* ]] +}