diff --git a/scripts/qwen38-calib-window.sh b/scripts/qwen38-calib-window.sh index 5399fc8..e1d9993 100755 --- a/scripts/qwen38-calib-window.sh +++ b/scripts/qwen38-calib-window.sh @@ -12,8 +12,13 @@ # (2026-10-29) the rig's boot died of an OOM, the health loop quietly fell through, and the # pusher fed four rounds of 10M tokens into RESTORED PRODUCTION with nothing recorded: # -# 1. the runner must still be alive and its container must answer /health, or this script -# kills it, restores the service and exits non-zero — no push, ever; +# 1. the runner must still be alive and its container must answer /health AND own the +# listening socket, or this script kills it, restores the service and exits non-zero — +# no push, ever. The socket clause is not decoration: the runner stops frank-archon +# first, the dying process keeps its listener for a moment, and a rig container that +# is still loading has not bound the port yet, so `podman inspect` + `/health` both +# report success for the WRONG process (measured 2026-10-30: `health ok waited=0s`, +# then the pusher refused on the container name and the window died at second zero); # 2. the pusher refuses on its own unless the rig wrote its marker into the tap dir, the # marker names this container, and the container is running (scripts/calib-push.py # --tap-marker), so a second lock survives a mistake in this file; @@ -79,8 +84,18 @@ say "phase=calib serve begin" bash scripts/run-qwen38-bf16.sh calib & spid=$! rig_up() { - podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true \ - && curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health + podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true || return 1 + curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health || return 1 + # The answer must come from the CONTAINER. The runner stops frank-archon first, and the + # dying production process keeps its listener for a moment while the freshly created + # rig container is still loading — so `container running` + `/health` 200 both hold + # before the rig has bound anything. Measured 2026-10-30: `health ok waited=0s`, then the + # pusher refused on the container name and the window died at second zero. Ask the kernel + # who owns the listening socket and require it to be a rig process. + local lpid + lpid=$(ss -H -ltnp 'sport = :7770' 2>/dev/null | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2) + [ -n "${lpid:-}" ] || return 1 + podman top "$RIG" pid 2>/dev/null | grep -qw "$lpid" } waited=0 diff --git a/tests/qwen38_campaign.bats b/tests/qwen38_campaign.bats index f932c5b..348ad24 100755 --- a/tests/qwen38_campaign.bats +++ b/tests/qwen38_campaign.bats @@ -732,3 +732,82 @@ PY [ "$status" -eq 2 ] [[ "$output" == *"Q38_CALIB_MNBT must be a number"* ]] } + +# ---- the window must not mistake a dying production socket for the rig ------- +# 2026-10-30: the runner stopped frank-archon, the rig container was still loading, +# and /health answered 200 from the production process draining its listener. The +# window logged `health ok waited=0s`, the pusher refused on the container name, and +# a 12M window died at second zero. The gate must prove the LISTENER is in the rig. + +tap_window_fixture() { + export W="$BATS_TEST_TMPDIR/window" + mkdir -p "$W/scripts" "$W/campaign/calib" "$W/campaign/kld-ref" "$W/models/rad" "$W/corpora" "$W/bin" + cp "$BATS_TEST_DIRNAME/../scripts/qwen38-calib-window.sh" "$W/scripts/" + : > "$W/campaign/kld-ref/kld.json" + : > "$W/models/rad/qwen38-bf16.rad" + : > "$W/corpora/calib.jsonl" + # a gram so the empty-tap check passes on the positive path + : > "$W/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin" + + cat > "$W/scripts/run-qwen38-bf16.sh" <<'EOS' +#!/usr/bin/env bash +[ -n "${STUB_RIG_MARKER:-}" ] && : > "$STUB_RIG_MARKER" +sleep "${STUB_RIG_SLEEP:-3}" +exit 0 +EOS + cat > "$W/bin/py" <<'EOS' +#!/usr/bin/env bash +[ -n "${STUB_PUSH_SENTINEL:-}" ] && : > "$STUB_PUSH_SENTINEL" +exit 0 +EOS + cat > "$W/bin/podman" <<'EOS' +#!/usr/bin/env bash +case "${1:-}" in + inspect) echo true ;; + top) echo PID; echo "${RIG_PID:-424242}" ;; + *) : ;; +esac +EOS + cat > "$W/bin/ss" <<'EOS' +#!/usr/bin/env bash +echo "LISTEN 0 4096 *:7770 *:* users:((\"radiance\",pid=${LISTEN_PID:-424242},fd=9))" +EOS + cat > "$W/bin/curl" <<'EOS' +#!/usr/bin/env bash +exit 0 +EOS + chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl" +} + +tap_window_env() { + env PATH="$W/bin:$PATH" \ + FRANK_QWEN38_CAMPAIGN="$W/campaign" \ + FRANK_QWEN38_RAD="$W/models/rad/qwen38-bf16.rad" \ + Q38_WINDOW_CALIB_CORPUS="$W/corpora/calib.jsonl" \ + Q38_CALIB_MIN_FREE_MIB=1 \ + Q38_WINDOW_HEALTH_TIMEOUT=1 \ + PY="$W/bin/py" \ + STUB_RIG_MARKER="$W/rig.marker" \ + STUB_RIG_SLEEP=0 \ + STUB_PUSH_SENTINEL="$W/push.ran" \ + bash "$W/scripts/qwen38-calib-window.sh" +} + +@test "a health answer from a listener outside the rig is not the rig: no push, window fails" { + tap_window_fixture + # health 200, container "running", but the listener pid belongs to production + export LISTEN_PID=999999 RIG_PID=424242 + run tap_window_env + [ "$status" -ne 0 ] + [ -z "$(ls "$W/push.ran" 2>/dev/null)" ] + [[ "$output" == *"before it ever answered /health"* ]] +} + +@test "when the listener is inside the rig the window proceeds and banks the tap" { + tap_window_fixture + export LISTEN_PID=424242 RIG_PID=424242 + run tap_window_env + [ "$status" -eq 0 ] + [ -e "$W/push.ran" ] + [[ "$output" == *"TAP BANKED"* ]] +}