fix(qwen38): the calibration window must prove the rig owns the socket

The runner stops frank-archon first, and the dying production process keeps its
listener for a moment while the freshly created rig container is still loading.
'podman inspect says running' plus '/health answers' therefore BOTH hold for the
wrong process: measured 2026-10-30, the window logged 'health ok waited=0s',
the pusher refused on the container name, and a 12M window died at second zero.

The gate now asks the kernel who owns :7770 (ss -ltnp) and requires that pid to
be one of the rig container's processes (podman top). Two tests, written red
first against the old rig_up: a foreign listener must fail the window without a
push, and an in-container listener must proceed and bank.

Also renames the new test fixture to tap_window_fixture: it had shadowed the
existing window_fixture helper, which broke the window dry-run test when the
whole file ran.
This commit is contained in:
Frank committed 2026-10-30 15:54:57 -07:00
1 parent 4043798053
commit 167e6d6de3
2 files changed
+98 -4

No files matched your search

+19 -4
View File
@@ -12,8 +12,13 @@
# (2026-10-29) the rig's boot died of an OOM, the health loop quietly fell through, and the # (2026-10-29) the rig's boot died of an OOM, the health loop quietly fell through, and the
# pusher fed four rounds of 10M tokens into RESTORED PRODUCTION with nothing recorded: # pusher fed four rounds of 10M tokens into RESTORED PRODUCTION with nothing recorded:
# #
# 1. the runner must still be alive and its container must answer /health, or this script # 1. the runner must still be alive and its container must answer /health AND own the
# kills it, restores the service and exits non-zero — no push, ever; # listening socket, or this script kills it, restores the service and exits non-zero —
# no push, ever. The socket clause is not decoration: the runner stops frank-archon
# first, the dying process keeps its listener for a moment, and a rig container that
# is still loading has not bound the port yet, so `podman inspect` + `/health` both
# report success for the WRONG process (measured 2026-10-30: `health ok waited=0s`,
# then the pusher refused on the container name and the window died at second zero);
# 2. the pusher refuses on its own unless the rig wrote its marker into the tap dir, the # 2. the pusher refuses on its own unless the rig wrote its marker into the tap dir, the
# marker names this container, and the container is running (scripts/calib-push.py # marker names this container, and the container is running (scripts/calib-push.py
# --tap-marker), so a second lock survives a mistake in this file; # --tap-marker), so a second lock survives a mistake in this file;
@@ -79,8 +84,18 @@ say "phase=calib serve begin"
bash scripts/run-qwen38-bf16.sh calib & spid=$! bash scripts/run-qwen38-bf16.sh calib & spid=$!
rig_up() { rig_up() {
podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true \ podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true || return 1
&& curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health || return 1
# The answer must come from the CONTAINER. The runner stops frank-archon first, and the
# dying production process keeps its listener for a moment while the freshly created
# rig container is still loading — so `container running` + `/health` 200 both hold
# before the rig has bound anything. Measured 2026-10-30: `health ok waited=0s`, then the
# pusher refused on the container name and the window died at second zero. Ask the kernel
# who owns the listening socket and require it to be a rig process.
local lpid
lpid=$(ss -H -ltnp 'sport = :7770' 2>/dev/null | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2)
[ -n "${lpid:-}" ] || return 1
podman top "$RIG" pid 2>/dev/null | grep -qw "$lpid"
} }
waited=0 waited=0
+79
View File
@@ -732,3 +732,82 @@ PY
[ "$status" -eq 2 ] [ "$status" -eq 2 ]
[[ "$output" == *"Q38_CALIB_MNBT must be a number"* ]] [[ "$output" == *"Q38_CALIB_MNBT must be a number"* ]]
} }
# ---- the window must not mistake a dying production socket for the rig -------
# 2026-10-30: the runner stopped frank-archon, the rig container was still loading,
# and /health answered 200 from the production process draining its listener. The
# window logged `health ok waited=0s`, the pusher refused on the container name, and
# a 12M window died at second zero. The gate must prove the LISTENER is in the rig.
tap_window_fixture() {
export W="$BATS_TEST_TMPDIR/window"
mkdir -p "$W/scripts" "$W/campaign/calib" "$W/campaign/kld-ref" "$W/models/rad" "$W/corpora" "$W/bin"
cp "$BATS_TEST_DIRNAME/../scripts/qwen38-calib-window.sh" "$W/scripts/"
: > "$W/campaign/kld-ref/kld.json"
: > "$W/models/rad/qwen38-bf16.rad"
: > "$W/corpora/calib.jsonl"
# a gram so the empty-tap check passes on the positive path
: > "$W/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin"
cat > "$W/scripts/run-qwen38-bf16.sh" <<'EOS'
#!/usr/bin/env bash
[ -n "${STUB_RIG_MARKER:-}" ] && : > "$STUB_RIG_MARKER"
sleep "${STUB_RIG_SLEEP:-3}"
exit 0
EOS
cat > "$W/bin/py" <<'EOS'
#!/usr/bin/env bash
[ -n "${STUB_PUSH_SENTINEL:-}" ] && : > "$STUB_PUSH_SENTINEL"
exit 0
EOS
cat > "$W/bin/podman" <<'EOS'
#!/usr/bin/env bash
case "${1:-}" in
inspect) echo true ;;
top) echo PID; echo "${RIG_PID:-424242}" ;;
*) : ;;
esac
EOS
cat > "$W/bin/ss" <<'EOS'
#!/usr/bin/env bash
echo "LISTEN 0 4096 *:7770 *:* users:((\"radiance\",pid=${LISTEN_PID:-424242},fd=9))"
EOS
cat > "$W/bin/curl" <<'EOS'
#!/usr/bin/env bash
exit 0
EOS
chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl"
}
tap_window_env() {
env PATH="$W/bin:$PATH" \
FRANK_QWEN38_CAMPAIGN="$W/campaign" \
FRANK_QWEN38_RAD="$W/models/rad/qwen38-bf16.rad" \
Q38_WINDOW_CALIB_CORPUS="$W/corpora/calib.jsonl" \
Q38_CALIB_MIN_FREE_MIB=1 \
Q38_WINDOW_HEALTH_TIMEOUT=1 \
PY="$W/bin/py" \
STUB_RIG_MARKER="$W/rig.marker" \
STUB_RIG_SLEEP=0 \
STUB_PUSH_SENTINEL="$W/push.ran" \
bash "$W/scripts/qwen38-calib-window.sh"
}
@test "a health answer from a listener outside the rig is not the rig: no push, window fails" {
tap_window_fixture
# health 200, container "running", but the listener pid belongs to production
export LISTEN_PID=999999 RIG_PID=424242
run tap_window_env
[ "$status" -ne 0 ]
[ -z "$(ls "$W/push.ran" 2>/dev/null)" ]
[[ "$output" == *"before it ever answered /health"* ]]
}
@test "when the listener is inside the rig the window proceeds and banks the tap" {
tap_window_fixture
export LISTEN_PID=424242 RIG_PID=424242
run tap_window_env
[ "$status" -eq 0 ]
[ -e "$W/push.ran" ]
[[ "$output" == *"TAP BANKED"* ]]
}