fix(qwen38): one identity preflight owns every fact a calibration window needs
Each gate the window used was a proxy, and each lied at least once: - /health answered 200 from the dying production process (window dead at second zero) - podman inspect named a rig container that had bound nothing - the space check asked a directory that merely looked related - the identity check compared podman's CONTAINER-internal pids (1, 7) to an ss host pid scripts/qwen38-preflight.sh now checks cwd, the digest a tag actually resolves to, free space on the tap's own filesystem, stale grams, every /dev/kfd holder's host pid (podman top <ctr> hpid) and the :7770 listener's host pid against the service's, the corpus and the reference recording; it prints PASS/FAIL per fact and exits 2. The window calls it before a card is touched, and its duplicated checks are gone.
This commit is contained in:
Frank committed
1 parent
167e6d6de3
commit
5c678b804c
4 files changed
+292
-27
No files matched your search
Executable
+133
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env bash
|
||||
# qwen38-preflight.sh — the facts a calibration window must have BEFORE it takes the cards,
|
||||
# each checked by IDENTITY and not by proxy. Every line is PASS or FAIL with the reason.
|
||||
#
|
||||
# Why this file exists: on 2026-10-30 a window died at second zero because `/health` answered
|
||||
# 200 from the *production* process that was still draining its listener, while the freshly
|
||||
# created rig container had not bound anything. Other runs died from a port that answered for
|
||||
# the wrong process, a /dev/kfd held by an orphaned engine, a tap dir on a 35 GiB filesystem,
|
||||
# and a launcher whose cwd silently lacked the repo. None of those were math problems; all of
|
||||
# them were a check that measured something adjacent to the thing it meant.
|
||||
#
|
||||
# Usage: bash scripts/qwen38-preflight.sh [--allow-existing-tap]
|
||||
# Exit: 0 all PASS · 2 any FAIL · 3 a fact could not be evaluated (missing tool)
|
||||
set -uo pipefail
|
||||
|
||||
SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SELF_DIR/.." && pwd)"
|
||||
|
||||
CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
|
||||
TAP="$CAMPAIGN/calib"
|
||||
CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}"
|
||||
NEED_MIB="${Q38_CALIB_MIN_FREE_MIB:-61440}"
|
||||
ALLOW_EXISTING=0
|
||||
[ "${1:-}" = "--allow-existing-tap" ] && ALLOW_EXISTING=1
|
||||
|
||||
# The producer pin is the calib image; the window fails closed if the tag on this host is not it.
|
||||
WANT_DIGEST="${Q38_PREFLIGHT_IMAGE_DIGEST:-sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}"
|
||||
IMAGE_REF="${Q38_CALIB_IMAGE:-stilldeadcode/radiance@$WANT_DIGEST}"
|
||||
SERVICE="frank-archon"
|
||||
PORT=7770
|
||||
|
||||
fails=0
|
||||
ok() { printf 'PASS %-14s %s\n' "$1" "$2"; }
|
||||
bad() { printf 'FAIL %-14s %s\n' "$1" "$2"; fails=$((fails + 1)); }
|
||||
skip() { printf 'WARN %-14s %s\n' "$1" "$2"; }
|
||||
|
||||
# 1. cwd — a detached launcher that lost its cwd fails here in a second, not in the log later.
|
||||
if [ "$(cd "$REPO_ROOT" && pwd)" = "$(pwd)" ] && [ -x "$REPO_ROOT/scripts/run-qwen38-bf16.sh" ]; then
|
||||
ok cwd "running from the repo root ($REPO_ROOT)"
|
||||
else
|
||||
bad cwd "cwd is $(pwd), not the repo root ($REPO_ROOT) — every relative script path would break"
|
||||
fi
|
||||
|
||||
# 2. image — resolve the REF to a digest. A stale local `:latest` is a different model of the world.
|
||||
if command -v podman >/dev/null 2>&1; then
|
||||
got="$(podman image inspect -f '{{index .RepoDigests 0}}' "$IMAGE_REF" 2>/dev/null | sed 's/.*@//')"
|
||||
if [ -z "$got" ]; then
|
||||
bad image "$IMAGE_REF is not present on this host (podman has no digest for it)"
|
||||
elif [ "$got" = "$WANT_DIGEST" ]; then
|
||||
ok image "$IMAGE_REF resolves to ${got:0:19}…"
|
||||
else
|
||||
bad image "$IMAGE_REF resolves to $got, expected $WANT_DIGEST"
|
||||
fi
|
||||
else
|
||||
skip image "podman not on PATH — cannot resolve $IMAGE_REF"
|
||||
fi
|
||||
|
||||
# 3. space — df the FILE PATH the seal writes to, never a directory that merely looks related.
|
||||
if [ -d "$TAP" ] || mkdir -p "$TAP" 2>/dev/null; then
|
||||
avail="$(df --output=avail -B 1M "$TAP" 2>/dev/null | tail -1 | tr -d ' ')"
|
||||
if [[ "${avail:-}" =~ ^[0-9]+$ ]] && [ "$avail" -ge "$NEED_MIB" ]; then
|
||||
ok space "$avail MiB free on $(df --output=target "$TAP" | tail -1) (need $NEED_MIB)"
|
||||
else
|
||||
bad space "${avail:-unknown} MiB free on $TAP, need $NEED_MIB — one seal writes ~43 GB"
|
||||
fi
|
||||
# 4. tap cleanliness — stale grams from another run must never be mixed in.
|
||||
n=$(ls -A "$TAP" 2>/dev/null | grep -c . || true)
|
||||
if [ "$n" -eq 0 ] || [ "$ALLOW_EXISTING" = 1 ]; then
|
||||
ok tap-empty "$n entries in $TAP"
|
||||
else
|
||||
bad tap-empty "$n entries in $TAP — rotate them to .AGENTTRASH first"
|
||||
fi
|
||||
else
|
||||
bad space "cannot create $TAP"
|
||||
fi
|
||||
|
||||
# 5. device — every /dev/kfd holder must be a process of the service we are about to stop. An
|
||||
# orphaned engine from a previous run makes the next boot die with `startup stopped: device
|
||||
# error`. Identity comes from podman's own host-pid list: `podman top <ctr> pid` prints
|
||||
# CONTAINER-internal pids (1, 7, …), which compare against nothing, and a rootless cgroup
|
||||
# carries the container ID, never the service name — both are proxies that lie.
|
||||
PROD_CTR="${Q38_PREFLIGHT_PROD_CONTAINER:-$SERVICE}"
|
||||
service_pids() { podman top "$PROD_CTR" hpid 2>/dev/null | tail -n +2; }
|
||||
if [ -e /dev/kfd ]; then
|
||||
mine="$(service_pids | tr '\n' ' ')"
|
||||
foreign=""
|
||||
for p in $(fuser /dev/kfd 2>/dev/null); do
|
||||
case " $mine " in
|
||||
*" $p "*) : ;; # the service we will stop
|
||||
*) foreign="$foreign $p:$(cat /proc/$p/comm 2>/dev/null)" ;;
|
||||
esac
|
||||
done
|
||||
if [ -z "$foreign" ]; then
|
||||
ok device "no foreign /dev/kfd holders"
|
||||
else
|
||||
bad device "foreign holders:$foreign — kill by pid, never by pattern"
|
||||
fi
|
||||
else
|
||||
skip device "/dev/kfd absent — not an AMD host?"
|
||||
fi
|
||||
|
||||
# 6. port — a listener on :7770 must be a process of the service we stop, never an unknown one.
|
||||
lpid="$(ss -H -ltnp "sport = :$PORT" 2>/dev/null | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2)"
|
||||
if [ -z "${lpid:-}" ]; then
|
||||
ok port ":$PORT is free"
|
||||
else
|
||||
mine="$(service_pids | tr '\n' ' ')"
|
||||
case " $mine " in
|
||||
*" $lpid "*) ok port ":$PORT is held by $SERVICE (host pid $lpid) — the window stops it" ;;
|
||||
*) bad port ":$PORT is held by host pid $lpid ($(cat /proc/$lpid/comm 2>/dev/null)), not $SERVICE" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# 7. corpus — the push source must exist and be the re-chunked file the window documents.
|
||||
if [ -s "$CORPUS" ]; then
|
||||
ok corpus "$(wc -l < "$CORPUS") prompts in $CORPUS"
|
||||
else
|
||||
bad corpus "$CORPUS is missing or empty"
|
||||
fi
|
||||
|
||||
# 8. reference — the KL gate has nothing to compare against without it.
|
||||
if [ -s "$CAMPAIGN/kld-ref/kld.json" ]; then
|
||||
ok reference "$(du -h "$CAMPAIGN/kld-ref/kld.json" | cut -f1) kld.json present"
|
||||
else
|
||||
bad reference "$CAMPAIGN/kld-ref/kld.json is missing"
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "PREFLIGHT FAILED: $fails fact(s). The window must not take the cards."
|
||||
exit 2
|
||||
fi
|
||||
echo "PREFLIGHT PASSED: every fact checked by identity."
|
||||
Reference in new issue
Block a user