fix(qwen38): one identity preflight owns every fact a calibration window needs
Each gate the window used was a proxy, and each lied at least once: - /health answered 200 from the dying production process (window dead at second zero) - podman inspect named a rig container that had bound nothing - the space check asked a directory that merely looked related - the identity check compared podman's CONTAINER-internal pids (1, 7) to an ss host pid scripts/qwen38-preflight.sh now checks cwd, the digest a tag actually resolves to, free space on the tap's own filesystem, stale grams, every /dev/kfd holder's host pid (podman top <ctr> hpid) and the :7770 listener's host pid against the service's, the corpus and the reference recording; it prints PASS/FAIL per fact and exits 2. The window calls it before a card is touched, and its duplicated checks are gone.
This commit is contained in:
Frank committed
1 parent
167e6d6de3
commit
5c678b804c
4 files changed
+292
-27
No files matched your search
@@ -40,7 +40,6 @@ MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
|
||||
CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
|
||||
RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}"
|
||||
CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}"
|
||||
KLDREF="$CAMPAIGN/kld-ref/kld.json"
|
||||
CALIB="${FRANK_QWEN38_CALIB_DIR:-$CAMPAIGN/calib}"
|
||||
HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}"
|
||||
ROUNDS="${Q38_WINDOW_ROUNDS:-6}"
|
||||
@@ -48,22 +47,19 @@ RIG=qwen38-bf16
|
||||
SERVICE=frank-archon
|
||||
PY="${PY:-python3}"
|
||||
|
||||
# ---- preflight: everything this window needs, before a card is touched -------
|
||||
[ -f "$KLDREF" ] || { echo "REFUSED: no reference recording at $KLDREF — record it first (scripts/run-qwen38-bf16.sh kldref); this window only fills the calibration tap" >&2; exit 2; }
|
||||
# ---- preflight: every fact by identity, one owner (scripts/qwen38-preflight.sh) ----
|
||||
# The checks that used to live here were proxies, and each lied at least once: the space check
|
||||
# asked a directory that merely LOOKED related, `/health` answered for a dying production
|
||||
# process, `podman inspect` named a container that existed but had bound nothing. The shared
|
||||
# script binds each fact to its object — cwd, the digest the tag actually resolves to, free
|
||||
# space on the tap's own filesystem, every /dev/kfd holder's HOST pid against the service's,
|
||||
# the :7770 listener's host pid, the corpus and the reference recording. A resumed window
|
||||
# keeps an existing tap, so the tap-empty fact is waived.
|
||||
[ -f "$RAD" ] || { echo "REFUSED: no BF16 reference container at $RAD — nothing to serve; scripts/build-qwen38-bf16-rad.sh" >&2; exit 2; }
|
||||
[ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; }
|
||||
[ -d "$CALIB" ] || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; }
|
||||
[ -w "$CALIB" ] || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; }
|
||||
# One tap seal writes ~40 GiB (measured 2026-10-30). The runner refuses too; this
|
||||
# names the same number before a card is touched, so a full volume never becomes
|
||||
# a 17-hour dead run.
|
||||
avail_mib=$(df --output=avail -B 1M "$CALIB" 2>/dev/null | tail -1 | tr -d ' ')
|
||||
[[ "$avail_mib" =~ ^[0-9]+$ ]] || avail_mib=0
|
||||
need_mib="${Q38_CALIB_MIN_FREE_MIB:-61440}"
|
||||
[ "$avail_mib" -ge "$need_mib" ] || {
|
||||
echo "REFUSED: tap dir $CALIB has ${avail_mib} MiB free; one seal writes ~40 GiB and this window needs ${need_mib} MiB (Q38_CALIB_MIN_FREE_MIB overrides)" >&2
|
||||
exit 2
|
||||
}
|
||||
FRANK_QWEN38_CAMPAIGN="$CAMPAIGN" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$CORPUS" \
|
||||
Q38_CALIB_MIN_FREE_MIB="${Q38_CALIB_MIN_FREE_MIB:-61440}" \
|
||||
bash scripts/qwen38-preflight.sh --allow-existing-tap || exit 2
|
||||
! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; }
|
||||
! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; }
|
||||
|
||||
|
||||
Executable
+133
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env bash
|
||||
# qwen38-preflight.sh — the facts a calibration window must have BEFORE it takes the cards,
|
||||
# each checked by IDENTITY and not by proxy. Every line is PASS or FAIL with the reason.
|
||||
#
|
||||
# Why this file exists: on 2026-10-30 a window died at second zero because `/health` answered
|
||||
# 200 from the *production* process that was still draining its listener, while the freshly
|
||||
# created rig container had not bound anything. Other runs died from a port that answered for
|
||||
# the wrong process, a /dev/kfd held by an orphaned engine, a tap dir on a 35 GiB filesystem,
|
||||
# and a launcher whose cwd silently lacked the repo. None of those were math problems; all of
|
||||
# them were a check that measured something adjacent to the thing it meant.
|
||||
#
|
||||
# Usage: bash scripts/qwen38-preflight.sh [--allow-existing-tap]
|
||||
# Exit: 0 all PASS · 2 any FAIL · 3 a fact could not be evaluated (missing tool)
|
||||
set -uo pipefail
|
||||
|
||||
SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SELF_DIR/.." && pwd)"
|
||||
|
||||
CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}"
|
||||
TAP="$CAMPAIGN/calib"
|
||||
CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}"
|
||||
NEED_MIB="${Q38_CALIB_MIN_FREE_MIB:-61440}"
|
||||
ALLOW_EXISTING=0
|
||||
[ "${1:-}" = "--allow-existing-tap" ] && ALLOW_EXISTING=1
|
||||
|
||||
# The producer pin is the calib image; the window fails closed if the tag on this host is not it.
|
||||
WANT_DIGEST="${Q38_PREFLIGHT_IMAGE_DIGEST:-sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}"
|
||||
IMAGE_REF="${Q38_CALIB_IMAGE:-stilldeadcode/radiance@$WANT_DIGEST}"
|
||||
SERVICE="frank-archon"
|
||||
PORT=7770
|
||||
|
||||
fails=0
|
||||
ok() { printf 'PASS %-14s %s\n' "$1" "$2"; }
|
||||
bad() { printf 'FAIL %-14s %s\n' "$1" "$2"; fails=$((fails + 1)); }
|
||||
skip() { printf 'WARN %-14s %s\n' "$1" "$2"; }
|
||||
|
||||
# 1. cwd — a detached launcher that lost its cwd fails here in a second, not in the log later.
|
||||
if [ "$(cd "$REPO_ROOT" && pwd)" = "$(pwd)" ] && [ -x "$REPO_ROOT/scripts/run-qwen38-bf16.sh" ]; then
|
||||
ok cwd "running from the repo root ($REPO_ROOT)"
|
||||
else
|
||||
bad cwd "cwd is $(pwd), not the repo root ($REPO_ROOT) — every relative script path would break"
|
||||
fi
|
||||
|
||||
# 2. image — resolve the REF to a digest. A stale local `:latest` is a different model of the world.
|
||||
if command -v podman >/dev/null 2>&1; then
|
||||
got="$(podman image inspect -f '{{index .RepoDigests 0}}' "$IMAGE_REF" 2>/dev/null | sed 's/.*@//')"
|
||||
if [ -z "$got" ]; then
|
||||
bad image "$IMAGE_REF is not present on this host (podman has no digest for it)"
|
||||
elif [ "$got" = "$WANT_DIGEST" ]; then
|
||||
ok image "$IMAGE_REF resolves to ${got:0:19}…"
|
||||
else
|
||||
bad image "$IMAGE_REF resolves to $got, expected $WANT_DIGEST"
|
||||
fi
|
||||
else
|
||||
skip image "podman not on PATH — cannot resolve $IMAGE_REF"
|
||||
fi
|
||||
|
||||
# 3. space — df the FILE PATH the seal writes to, never a directory that merely looks related.
|
||||
if [ -d "$TAP" ] || mkdir -p "$TAP" 2>/dev/null; then
|
||||
avail="$(df --output=avail -B 1M "$TAP" 2>/dev/null | tail -1 | tr -d ' ')"
|
||||
if [[ "${avail:-}" =~ ^[0-9]+$ ]] && [ "$avail" -ge "$NEED_MIB" ]; then
|
||||
ok space "$avail MiB free on $(df --output=target "$TAP" | tail -1) (need $NEED_MIB)"
|
||||
else
|
||||
bad space "${avail:-unknown} MiB free on $TAP, need $NEED_MIB — one seal writes ~43 GB"
|
||||
fi
|
||||
# 4. tap cleanliness — stale grams from another run must never be mixed in.
|
||||
n=$(ls -A "$TAP" 2>/dev/null | grep -c . || true)
|
||||
if [ "$n" -eq 0 ] || [ "$ALLOW_EXISTING" = 1 ]; then
|
||||
ok tap-empty "$n entries in $TAP"
|
||||
else
|
||||
bad tap-empty "$n entries in $TAP — rotate them to .AGENTTRASH first"
|
||||
fi
|
||||
else
|
||||
bad space "cannot create $TAP"
|
||||
fi
|
||||
|
||||
# 5. device — every /dev/kfd holder must be a process of the service we are about to stop. An
|
||||
# orphaned engine from a previous run makes the next boot die with `startup stopped: device
|
||||
# error`. Identity comes from podman's own host-pid list: `podman top <ctr> pid` prints
|
||||
# CONTAINER-internal pids (1, 7, …), which compare against nothing, and a rootless cgroup
|
||||
# carries the container ID, never the service name — both are proxies that lie.
|
||||
PROD_CTR="${Q38_PREFLIGHT_PROD_CONTAINER:-$SERVICE}"
|
||||
service_pids() { podman top "$PROD_CTR" hpid 2>/dev/null | tail -n +2; }
|
||||
if [ -e /dev/kfd ]; then
|
||||
mine="$(service_pids | tr '\n' ' ')"
|
||||
foreign=""
|
||||
for p in $(fuser /dev/kfd 2>/dev/null); do
|
||||
case " $mine " in
|
||||
*" $p "*) : ;; # the service we will stop
|
||||
*) foreign="$foreign $p:$(cat /proc/$p/comm 2>/dev/null)" ;;
|
||||
esac
|
||||
done
|
||||
if [ -z "$foreign" ]; then
|
||||
ok device "no foreign /dev/kfd holders"
|
||||
else
|
||||
bad device "foreign holders:$foreign — kill by pid, never by pattern"
|
||||
fi
|
||||
else
|
||||
skip device "/dev/kfd absent — not an AMD host?"
|
||||
fi
|
||||
|
||||
# 6. port — a listener on :7770 must be a process of the service we stop, never an unknown one.
|
||||
lpid="$(ss -H -ltnp "sport = :$PORT" 2>/dev/null | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2)"
|
||||
if [ -z "${lpid:-}" ]; then
|
||||
ok port ":$PORT is free"
|
||||
else
|
||||
mine="$(service_pids | tr '\n' ' ')"
|
||||
case " $mine " in
|
||||
*" $lpid "*) ok port ":$PORT is held by $SERVICE (host pid $lpid) — the window stops it" ;;
|
||||
*) bad port ":$PORT is held by host pid $lpid ($(cat /proc/$lpid/comm 2>/dev/null)), not $SERVICE" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# 7. corpus — the push source must exist and be the re-chunked file the window documents.
|
||||
if [ -s "$CORPUS" ]; then
|
||||
ok corpus "$(wc -l < "$CORPUS") prompts in $CORPUS"
|
||||
else
|
||||
bad corpus "$CORPUS is missing or empty"
|
||||
fi
|
||||
|
||||
# 8. reference — the KL gate has nothing to compare against without it.
|
||||
if [ -s "$CAMPAIGN/kld-ref/kld.json" ]; then
|
||||
ok reference "$(du -h "$CAMPAIGN/kld-ref/kld.json" | cut -f1) kld.json present"
|
||||
else
|
||||
bad reference "$CAMPAIGN/kld-ref/kld.json is missing"
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "PREFLIGHT FAILED: $fails fact(s). The window must not take the cards."
|
||||
exit 2
|
||||
fi
|
||||
echo "PREFLIGHT PASSED: every fact checked by identity."
|
||||
@@ -571,6 +571,19 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
|
||||
of falling through its health wait. All three exist because on 2026-10-29 the unguarded chain's
|
||||
rig boot died of an OOM, the health loop expired without failing, and the pusher fed four rounds
|
||||
of 10M tokens into **restored production** — 86 minutes of live prefill that recorded nothing.
|
||||
- **A container that exists, a port that answers, and a `/health` 200 are three different claims —
|
||||
bind identity before believing any of them.** Measured 2026-10-30: `podman inspect` named the rig
|
||||
while the port still belonged to the *dying* production process (`health ok waited=0s`, window dead
|
||||
at second zero), and the identity check written to catch that compared two different number spaces:
|
||||
`podman top <ctr> pid` prints **container-internal** pids (`1`, `7`), never host pids, and a rootless
|
||||
cgroup carries the container ID rather than the service name — so both "matches" can only ever fail.
|
||||
The binding is `podman top <ctr> hpid` against the listener from
|
||||
`ss -H -ltnp 'sport = :7770'`, and `fuser /dev/kfd` likewise resolved through podman.
|
||||
`scripts/qwen38-preflight.sh` owns those facts now (cwd, the digest a tag actually resolves to,
|
||||
free space on the tap's **own** filesystem, stale grams, every GPU holder and the listener by host
|
||||
pid), prints `PASS`/`FAIL` per fact, and `qwen38-calib-window.sh` calls it before a card is touched.
|
||||
Corollary for every ad-hoc driver: a detached launcher that loses its cwd fails with
|
||||
`No such file or directory` in a log nobody reads — assert cwd, and never bypass the orchestrator.
|
||||
- **The n-gram table's pin is what starves a rig boot, and the tier is a knob.** On the BF16 payload
|
||||
the table is 95.37 GiB, `--ngram-placement ram` pins it, and the calibration boot died at
|
||||
`startup stopped: out of memory` with 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available —
|
||||
|
||||
+134
-11
@@ -673,24 +673,41 @@ PY
|
||||
|
||||
window_fixture() {
|
||||
D="$BATS_TEST_TMPDIR/win"
|
||||
mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora"
|
||||
mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora" "$D/bin"
|
||||
echo '{}' > "$D/campaign/kld-ref/kld.json"
|
||||
: > "$D/rad/qwen38-bf16.rad"
|
||||
: > "$D/corpora/calib.jsonl"
|
||||
echo "{\"p\":1}" > "$D/corpora/calib.jsonl"
|
||||
# the window's preflight resolves the image digest through podman and asks about the
|
||||
# port/device by host pid; these stubs answer for a quiet host
|
||||
cat > "$D/bin/podman" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
case "${1:-}" in
|
||||
image) echo "sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093" ;;
|
||||
esac
|
||||
EOS
|
||||
printf '#!/usr/bin/env bash\nexit 0\n' > "$D/bin/ss"
|
||||
printf '#!/usr/bin/env bash\nexit 0\n' > "$D/bin/fuser"
|
||||
chmod +x "$D/bin/podman" "$D/bin/ss" "$D/bin/fuser"
|
||||
echo "$D"
|
||||
}
|
||||
|
||||
@test "the calibration window refuses before touching a card without a reference recording" {
|
||||
D="$BATS_TEST_TMPDIR/win-less"
|
||||
mkdir -p "$D/campaign/calib"
|
||||
run env FRANK_QWEN38_CAMPAIGN="$D/campaign" bash "$WINDOW"
|
||||
D="$(window_fixture)"
|
||||
mv "$D/campaign/kld-ref/kld.json" "$D/campaign/kld-ref/kld.json.absent"
|
||||
run env PATH="$D/bin:$PATH" \
|
||||
FRANK_QWEN38_CAMPAIGN="$D/campaign" \
|
||||
FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" \
|
||||
Q38_CALIB_MIN_FREE_MIB=1 bash "$WINDOW"
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"no reference recording"* ]]
|
||||
[[ "$output" == *"FAIL reference"* ]]
|
||||
[[ "$output" == *"must not take the cards"* ]]
|
||||
}
|
||||
|
||||
@test "the calibration window's dry run prints the gated plan, including what it would bank" {
|
||||
D="$(window_fixture)"
|
||||
run env FRANK_QWEN38_CAMPAIGN="$D/campaign" \
|
||||
run env PATH="$D/bin:$PATH" \
|
||||
FRANK_QWEN38_CAMPAIGN="$D/campaign" \
|
||||
FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" \
|
||||
Q38_CALIB_MIN_FREE_MIB=1 bash "$WINDOW" --dry-run
|
||||
@@ -743,9 +760,10 @@ tap_window_fixture() {
|
||||
export W="$BATS_TEST_TMPDIR/window"
|
||||
mkdir -p "$W/scripts" "$W/campaign/calib" "$W/campaign/kld-ref" "$W/models/rad" "$W/corpora" "$W/bin"
|
||||
cp "$BATS_TEST_DIRNAME/../scripts/qwen38-calib-window.sh" "$W/scripts/"
|
||||
: > "$W/campaign/kld-ref/kld.json"
|
||||
cp "$BATS_TEST_DIRNAME/../scripts/qwen38-preflight.sh" "$W/scripts/"
|
||||
echo "{}" > "$W/campaign/kld-ref/kld.json"
|
||||
: > "$W/models/rad/qwen38-bf16.rad"
|
||||
: > "$W/corpora/calib.jsonl"
|
||||
echo "{\"p\":1}" > "$W/corpora/calib.jsonl"
|
||||
# a gram so the empty-tap check passes on the positive path
|
||||
: > "$W/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin"
|
||||
|
||||
@@ -764,10 +782,20 @@ EOS
|
||||
#!/usr/bin/env bash
|
||||
case "${1:-}" in
|
||||
inspect) echo true ;;
|
||||
top) echo PID; echo "${RIG_PID:-424242}" ;;
|
||||
# the PREFLIGHT asks who owns :7770 (production, per LISTEN_PID); the WINDOW asks which
|
||||
# pids belong to the rig (RIG_PID). Both stubs are the same binary, so keep them apart.
|
||||
image) echo "sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093" ;;
|
||||
top) echo PID
|
||||
# the WINDOW asks "which pids are the rig's", the PREFLIGHT asks "which are
|
||||
# production's". One stub, two answers: that separation IS the identity lesson.
|
||||
case "${2:-}" in
|
||||
qwen38-bf16) echo "${RIG_PID:-424242}" ;;
|
||||
*) echo "${LISTEN_PID:-424242}" ;;
|
||||
esac ;;
|
||||
*) : ;;
|
||||
esac
|
||||
EOS
|
||||
printf '#!/usr/bin/env bash\nexit 0\n' > "$W/bin/fuser"
|
||||
cat > "$W/bin/ss" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
echo "LISTEN 0 4096 *:7770 *:* users:((\"radiance\",pid=${LISTEN_PID:-424242},fd=9))"
|
||||
@@ -776,7 +804,7 @@ EOS
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
EOS
|
||||
chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl"
|
||||
chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl" "$W/bin/fuser"
|
||||
}
|
||||
|
||||
tap_window_env() {
|
||||
@@ -811,3 +839,98 @@ tap_window_env() {
|
||||
[ -e "$W/push.ran" ]
|
||||
[[ "$output" == *"TAP BANKED"* ]]
|
||||
}
|
||||
|
||||
# ---- the preflight: every fact by IDENTITY, and it must fail loudly -------------------------
|
||||
# Each case here is a lie that cost a real window on 2026-10-30.
|
||||
|
||||
preflight_fixture() {
|
||||
export PF="$BATS_TEST_TMPDIR/pf"
|
||||
mkdir -p "$PF/campaign/calib" "$PF/campaign/kld-ref" "$PF/bin"
|
||||
echo '{}' > "$PF/campaign/kld-ref/kld.json"
|
||||
echo '{"p":1}' > "$PF/corpus.jsonl"
|
||||
cat > "$PF/bin/podman" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
case "$1" in
|
||||
image) echo "${STUB_IMAGE_DIGEST:-sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}" ;;
|
||||
top) echo HPID; echo "${STUB_PROD_PIDS:-4242}" ;;
|
||||
esac
|
||||
EOS
|
||||
cat > "$PF/bin/ss" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
[ -n "${STUB_LISTENER_PID:-}" ] && echo "LISTEN 0 4096 *:7770 *:* users:((\"x\",pid=${STUB_LISTENER_PID},fd=9))"
|
||||
exit 0
|
||||
EOS
|
||||
cat > "$PF/bin/fuser" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
[ -n "${STUB_KFD_PID:-}" ] && echo "${STUB_KFD_PID}"
|
||||
exit 0
|
||||
EOS
|
||||
chmod +x "$PF/bin/podman" "$PF/bin/ss" "$PF/bin/fuser"
|
||||
}
|
||||
|
||||
run_preflight() {
|
||||
( cd "$BATS_TEST_DIRNAME/.." && env PATH="$PF/bin:$PATH" \
|
||||
FRANK_QWEN38_CAMPAIGN="$PF/campaign" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$PF/corpus.jsonl" \
|
||||
Q38_CALIB_MIN_FREE_MIB=1 \
|
||||
bash scripts/qwen38-preflight.sh "$@" )
|
||||
}
|
||||
|
||||
@test "preflight passes when every fact is true, and says so" {
|
||||
preflight_fixture
|
||||
run run_preflight
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"PREFLIGHT PASSED"* ]]
|
||||
[[ "$output" == *"resolves to sha256:69add58f"* ]]
|
||||
}
|
||||
|
||||
@test "preflight accepts the service's own process on the port, not just any answer" {
|
||||
preflight_fixture
|
||||
export STUB_PROD_PIDS=4242 STUB_LISTENER_PID=4242 STUB_KFD_PID=4242
|
||||
run run_preflight
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"held by frank-archon (host pid 4242)"* ]]
|
||||
}
|
||||
|
||||
@test "preflight fails when the tag resolves to a different digest" {
|
||||
preflight_fixture
|
||||
export STUB_IMAGE_DIGEST="sha256:deadbeefdeadbeef"
|
||||
run run_preflight
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"expected sha256:69add58f"* ]]
|
||||
}
|
||||
|
||||
@test "preflight fails when :7770 belongs to a host pid the service does not own" {
|
||||
preflight_fixture
|
||||
export STUB_LISTENER_PID=999999
|
||||
run run_preflight
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *":7770 is held by host pid 999999"* ]]
|
||||
}
|
||||
|
||||
@test "preflight fails when another process holds the GPU" {
|
||||
preflight_fixture
|
||||
export STUB_KFD_PID=999998
|
||||
run run_preflight
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"foreign holders:"* ]]
|
||||
}
|
||||
|
||||
@test "preflight fails when the tap dir already holds grams from another run" {
|
||||
preflight_fixture
|
||||
: > "$PF/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin"
|
||||
run run_preflight
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"rotate them to .AGENTTRASH"* ]]
|
||||
run run_preflight --allow-existing-tap
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "preflight fails when the launcher is not in the repo root" {
|
||||
preflight_fixture
|
||||
run env PATH="$PF/bin:$PATH" FRANK_QWEN38_CAMPAIGN="$PF/campaign" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$PF/corpus.jsonl" Q38_CALIB_MIN_FREE_MIB=1 \
|
||||
bash -c "cd /tmp && bash '$BATS_TEST_DIRNAME/../scripts/qwen38-preflight.sh'"
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"not the repo root"* ]]
|
||||
}
|
||||
Reference in new issue
Block a user