Merge pull request 'fix(qwen38): the calibration window must prove the rig owns the socket' (#90) from fix/quant-pipeline-hardening into main

scripts/qwen38-calib-window.sh

@@ -12,8 +12,13 @@
 # (2026-10-29) the rig's boot died of an OOM, the health loop quietly fell through, and the
 # pusher fed four rounds of 10M tokens into RESTORED PRODUCTION with nothing recorded:
 #
-#   1. the runner must still be alive and its container must answer /health, or this script
-#      kills it, restores the service and exits non-zero — no push, ever;
+#   1. the runner must still be alive and its container must answer /health AND own the
+#      listening socket, or this script kills it, restores the service and exits non-zero —
+#      no push, ever. The socket clause is not decoration: the runner stops frank-archon
+#      first, the dying process keeps its listener for a moment, and a rig container that
+#      is still loading has not bound the port yet, so `podman inspect` + `/health` both
+#      report success for the WRONG process (measured 2026-10-30: `health ok waited=0s`,
+#      then the pusher refused on the container name and the window died at second zero);
 #   2. the pusher refuses on its own unless the rig wrote its marker into the tap dir, the
 #      marker names this container, and the container is running (scripts/calib-push.py
 #      --tap-marker), so a second lock survives a mistake in this file;
@@ -79,8 +84,18 @@ say "phase=calib serve begin"
 bash scripts/run-qwen38-bf16.sh calib & spid=$!
 
 rig_up() {
-    podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true \
-        && curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health
+    podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true || return 1
+    curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health || return 1
+    # The answer must come from the CONTAINER. The runner stops frank-archon first, and the
+    # dying production process keeps its listener for a moment while the freshly created
+    # rig container is still loading — so `container running` + `/health` 200 both hold
+    # before the rig has bound anything. Measured 2026-10-30: `health ok waited=0s`, then the
+    # pusher refused on the container name and the window died at second zero. Ask the kernel
+    # who owns the listening socket and require it to be a rig process.
+    local lpid
+    lpid=$(ss -H -ltnp 'sport = :7770' 2>/dev/null | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2)
+    [ -n "${lpid:-}" ] || return 1
+    podman top "$RIG" pid 2>/dev/null | grep -qw "$lpid"
 }
 
 waited=0

tests/qwen38_campaign.bats

@@ -732,3 +732,82 @@ PY
     [ "$status" -eq 2 ]
     [[ "$output" == *"Q38_CALIB_MNBT must be a number"* ]]
 }
+
+# ---- the window must not mistake a dying production socket for the rig -------
+# 2026-10-30: the runner stopped frank-archon, the rig container was still loading,
+# and /health answered 200 from the production process draining its listener. The
+# window logged `health ok waited=0s`, the pusher refused on the container name, and
+# a 12M window died at second zero. The gate must prove the LISTENER is in the rig.
+
+tap_window_fixture() {
+    export W="$BATS_TEST_TMPDIR/window"
+    mkdir -p "$W/scripts" "$W/campaign/calib" "$W/campaign/kld-ref" "$W/models/rad" "$W/corpora" "$W/bin"
+    cp "$BATS_TEST_DIRNAME/../scripts/qwen38-calib-window.sh" "$W/scripts/"
+    : > "$W/campaign/kld-ref/kld.json"
+    : > "$W/models/rad/qwen38-bf16.rad"
+    : > "$W/corpora/calib.jsonl"
+    # a gram so the empty-tap check passes on the positive path
+    : > "$W/campaign/calib/gram.r0.blk.0.ffn_down_exps.0.bin"
+
+    cat > "$W/scripts/run-qwen38-bf16.sh" <<'EOS'
+#!/usr/bin/env bash
+[ -n "${STUB_RIG_MARKER:-}" ] && : > "$STUB_RIG_MARKER"
+sleep "${STUB_RIG_SLEEP:-3}"
+exit 0
+EOS
+    cat > "$W/bin/py" <<'EOS'
+#!/usr/bin/env bash
+[ -n "${STUB_PUSH_SENTINEL:-}" ] && : > "$STUB_PUSH_SENTINEL"
+exit 0
+EOS
+    cat > "$W/bin/podman" <<'EOS'
+#!/usr/bin/env bash
+case "${1:-}" in
+    inspect) echo true ;;
+    top)     echo PID; echo "${RIG_PID:-424242}" ;;
+    *)       : ;;
+esac
+EOS
+    cat > "$W/bin/ss" <<'EOS'
+#!/usr/bin/env bash
+echo "LISTEN 0 4096 *:7770 *:* users:((\"radiance\",pid=${LISTEN_PID:-424242},fd=9))"
+EOS
+    cat > "$W/bin/curl" <<'EOS'
+#!/usr/bin/env bash
+exit 0
+EOS
+    chmod +x "$W/scripts/run-qwen38-bf16.sh" "$W/bin/py" "$W/bin/podman" "$W/bin/ss" "$W/bin/curl"
+}
+
+tap_window_env() {
+    env PATH="$W/bin:$PATH" \
+        FRANK_QWEN38_CAMPAIGN="$W/campaign" \
+        FRANK_QWEN38_RAD="$W/models/rad/qwen38-bf16.rad" \
+        Q38_WINDOW_CALIB_CORPUS="$W/corpora/calib.jsonl" \
+        Q38_CALIB_MIN_FREE_MIB=1 \
+        Q38_WINDOW_HEALTH_TIMEOUT=1 \
+        PY="$W/bin/py" \
+        STUB_RIG_MARKER="$W/rig.marker" \
+        STUB_RIG_SLEEP=0 \
+        STUB_PUSH_SENTINEL="$W/push.ran" \
+        bash "$W/scripts/qwen38-calib-window.sh"
+}
+
+@test "a health answer from a listener outside the rig is not the rig: no push, window fails" {
+    tap_window_fixture
+    # health 200, container "running", but the listener pid belongs to production
+    export LISTEN_PID=999999 RIG_PID=424242
+    run tap_window_env
+    [ "$status" -ne 0 ]
+    [ -z "$(ls "$W/push.ran" 2>/dev/null)" ]
+    [[ "$output" == *"before it ever answered /health"* ]]
+}
+
+@test "when the listener is inside the rig the window proceeds and banks the tap" {
+    tap_window_fixture
+    export LISTEN_PID=424242 RIG_PID=424242
+    run tap_window_env
+    [ "$status" -eq 0 ]
+    [ -e "$W/push.ran" ]
+    [[ "$output" == *"TAP BANKED"* ]]
+}