fix(qwen38): one identity preflight owns every fact a calibration window needs
Each gate the window used was a proxy, and each lied at least once: - /health answered 200 from the dying production process (window dead at second zero) - podman inspect named a rig container that had bound nothing - the space check asked a directory that merely looked related - the identity check compared podman's CONTAINER-internal pids (1, 7) to an ss host pid scripts/qwen38-preflight.sh now checks cwd, the digest a tag actually resolves to, free space on the tap's own filesystem, stale grams, every /dev/kfd holder's host pid (podman top <ctr> hpid) and the :7770 listener's host pid against the service's, the corpus and the reference recording; it prints PASS/FAIL per fact and exits 2. The window calls it before a card is touched, and its duplicated checks are gone.
This commit is contained in:
Frank committed
1 parent
167e6d6de3
commit
5c678b804c
4 files changed
+292
-27
No files matched your search
+134
-11
@@ -673,24 +673,41 @@ PY
|
||||
|
||||
window_fixture() {
|
||||
D="$BATS_TEST_TMPDIR/win"
|
||||
mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora"
|
||||
mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora" "$D/bin"
|
||||
echo '{}' > "$D/campaign/kld-ref/kld.json"
|
||||
: > "$D/rad/qwen38-bf16.rad"
|
||||
: > "$D/corpora/calib.jsonl"
|
||||
echo "{\"p\":1}" > "$D/corpora/calib.jsonl"
|
||||
# the window's preflight resolves the image digest through podman and asks about the
|
||||
# port/device by host pid; these stubs answer for a quiet host
|
||||
cat > "$D/bin/podman" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
case "${1:-}" in
|
||||
image) echo "sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093" ;;
|
||||
esac
|
||||
EOS
|
||||
printf '#!/usr/bin/env bash\nexit 0\n' > "$D/bin/ss"
|
||||
printf '#!/usr/bin/env bash\nexit 0\n' > "$D/bin/fuser"
|
||||
chmod +x "$D/bin/podman" "$D/bin/ss" "$D/bin/fuser"
|
||||
echo "$D"
|
||||
}
|
||||
|
||||
@test "the calibration window refuses before touching a card without a reference recording" {
|
||||
D="$BATS_TEST_TMPDIR/win-less"
|
||||
mkdir -p "$D/campaign/calib"
|
||||
run env FRANK_QWEN38_CAMPAIGN="$D/campaign" bash "$WINDOW"
|
||||
D="$(window_fixture)"
|
||||
mv "$D/campaign/kld-ref/kld.json" "$D/campaign/kld-ref/kld.json.absent"
|
||||
run env PATH="$D/bin:$PATH" \
|
||||
FRANK_QWEN38_CAMPAIGN="$D/campaign" \
|
||||
FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" \
|
||||
Q38_CALIB_MIN_FREE_MIB=1 bash "$WINDOW"
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"no reference recording"* ]]
|
||||
[[ "$output" == *"FAIL reference"* ]]
|
||||
[[ "$output" == *"must not take the cards"* ]]
|
||||
}
|
||||
|
||||
@test "the calibration window's dry run prints the gated plan, including what it would bank" {
|
||||
D="$(window_fixture)"
|
||||
run env FRANK_QWEN38_CAMPAIGN="$D/campaign" \
|
||||
run env PATH="$D/bin:$PATH" \
|
||||
FRANK_QWEN38_CAMPAIGN="$D/campaign" \
|
||||
FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" \
|
||||
Q38_CALIB_MIN_FREE_MIB=1 bash "$WINDOW" --dry-run
|
||||
@@ -743,9 +760,10 @@ tap_window_fixture() {
|
||||
export W="$BATS_TEST_TMPDIR/window"
|
||||
mkdir -p "$W/scripts" "$W/campaign/calib" "$W/campaign/kld-ref" "$W/models/rad" "$W/corpora" "$W/bin"
|
||||
cp "$BATS_TEST_DIRNAME/../scripts/qwen38-calib-window.sh" "$W/scripts/"
|
||||
: > "$W/campaign/kld-ref/kld.json"
|
||||
cp "$BATS_TEST_DIRNAME/../scripts/qwen38-preflight.sh" "$W/scripts/"
|
||||
echo "{}" > "$W/campaign/kld-ref/kld.json"
|
||||
: > "$W/models/rad/qwen38-bf16.rad"
|
||||
: > "$W/corpora/calib.jsonl"
|
||||
echo "{\"p\":1}" > "$W/corpora/calib.jsonl"
|
||||
# a gram so the empty-tap check passes on the positive path
|
||||
: > "$W/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin"
|
||||
|
||||
@@ -764,10 +782,20 @@ EOS
|
||||
#!/usr/bin/env bash
|
||||
case "${1:-}" in
|
||||
inspect) echo true ;;
|
||||
top) echo PID; echo "${RIG_PID:-424242}" ;;
|
||||
# the PREFLIGHT asks who owns :7770 (production, per LISTEN_PID); the WINDOW asks which
|
||||
# pids belong to the rig (RIG_PID). Both stubs are the same binary, so keep them apart.
|
||||
image) echo "sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093" ;;
|
||||
top) echo PID
|
||||
# the WINDOW asks "which pids are the rig's", the PREFLIGHT asks "which are
|
||||
# production's". One stub, two answers: that separation IS the identity lesson.
|
||||
case "${2:-}" in
|
||||
qwen38-bf16) echo "${RIG_PID:-424242}" ;;
|
||||
*) echo "${LISTEN_PID:-424242}" ;;
|
||||
esac ;;
|
||||
*) : ;;
|
||||
esac
|
||||
EOS
|
||||
printf '#!/usr/bin/env bash\nexit 0\n' > "$W/bin/fuser"
|
||||
cat > "$W/bin/ss" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
echo "LISTEN 0 4096 *:7770 *:* users:((\"radiance\",pid=${LISTEN_PID:-424242},fd=9))"
|
||||
@@ -776,7 +804,7 @@ EOS
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
EOS
|
||||
chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl"
|
||||
chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl" "$W/bin/fuser"
|
||||
}
|
||||
|
||||
tap_window_env() {
|
||||
@@ -811,3 +839,98 @@ tap_window_env() {
|
||||
[ -e "$W/push.ran" ]
|
||||
[[ "$output" == *"TAP BANKED"* ]]
|
||||
}
|
||||
|
||||
# ---- the preflight: every fact by IDENTITY, and it must fail loudly -------------------------
|
||||
# Each case here is a lie that cost a real window on 2026-10-30.
|
||||
|
||||
preflight_fixture() {
|
||||
export PF="$BATS_TEST_TMPDIR/pf"
|
||||
mkdir -p "$PF/campaign/calib" "$PF/campaign/kld-ref" "$PF/bin"
|
||||
echo '{}' > "$PF/campaign/kld-ref/kld.json"
|
||||
echo '{"p":1}' > "$PF/corpus.jsonl"
|
||||
cat > "$PF/bin/podman" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
case "$1" in
|
||||
image) echo "${STUB_IMAGE_DIGEST:-sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}" ;;
|
||||
top) echo HPID; echo "${STUB_PROD_PIDS:-4242}" ;;
|
||||
esac
|
||||
EOS
|
||||
cat > "$PF/bin/ss" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
[ -n "${STUB_LISTENER_PID:-}" ] && echo "LISTEN 0 4096 *:7770 *:* users:((\"x\",pid=${STUB_LISTENER_PID},fd=9))"
|
||||
exit 0
|
||||
EOS
|
||||
cat > "$PF/bin/fuser" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
[ -n "${STUB_KFD_PID:-}" ] && echo "${STUB_KFD_PID}"
|
||||
exit 0
|
||||
EOS
|
||||
chmod +x "$PF/bin/podman" "$PF/bin/ss" "$PF/bin/fuser"
|
||||
}
|
||||
|
||||
run_preflight() {
|
||||
( cd "$BATS_TEST_DIRNAME/.." && env PATH="$PF/bin:$PATH" \
|
||||
FRANK_QWEN38_CAMPAIGN="$PF/campaign" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$PF/corpus.jsonl" \
|
||||
Q38_CALIB_MIN_FREE_MIB=1 \
|
||||
bash scripts/qwen38-preflight.sh "$@" )
|
||||
}
|
||||
|
||||
@test "preflight passes when every fact is true, and says so" {
|
||||
preflight_fixture
|
||||
run run_preflight
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"PREFLIGHT PASSED"* ]]
|
||||
[[ "$output" == *"resolves to sha256:69add58f"* ]]
|
||||
}
|
||||
|
||||
@test "preflight accepts the service's own process on the port, not just any answer" {
|
||||
preflight_fixture
|
||||
export STUB_PROD_PIDS=4242 STUB_LISTENER_PID=4242 STUB_KFD_PID=4242
|
||||
run run_preflight
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"held by frank-archon (host pid 4242)"* ]]
|
||||
}
|
||||
|
||||
@test "preflight fails when the tag resolves to a different digest" {
|
||||
preflight_fixture
|
||||
export STUB_IMAGE_DIGEST="sha256:deadbeefdeadbeef"
|
||||
run run_preflight
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"expected sha256:69add58f"* ]]
|
||||
}
|
||||
|
||||
@test "preflight fails when :7770 belongs to a host pid the service does not own" {
|
||||
preflight_fixture
|
||||
export STUB_LISTENER_PID=999999
|
||||
run run_preflight
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *":7770 is held by host pid 999999"* ]]
|
||||
}
|
||||
|
||||
@test "preflight fails when another process holds the GPU" {
|
||||
preflight_fixture
|
||||
export STUB_KFD_PID=999998
|
||||
run run_preflight
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"foreign holders:"* ]]
|
||||
}
|
||||
|
||||
@test "preflight fails when the tap dir already holds grams from another run" {
|
||||
preflight_fixture
|
||||
: > "$PF/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin"
|
||||
run run_preflight
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"rotate them to .AGENTTRASH"* ]]
|
||||
run run_preflight --allow-existing-tap
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "preflight fails when the launcher is not in the repo root" {
|
||||
preflight_fixture
|
||||
run env PATH="$PF/bin:$PATH" FRANK_QWEN38_CAMPAIGN="$PF/campaign" \
|
||||
Q38_WINDOW_CALIB_CORPUS="$PF/corpus.jsonl" Q38_CALIB_MIN_FREE_MIB=1 \
|
||||
bash -c "cd /tmp && bash '$BATS_TEST_DIRNAME/../scripts/qwen38-preflight.sh'"
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"not the repo root"* ]]
|
||||
}
|
||||
Reference in new issue
Block a user