Merge pull request 'fix(deploy): relay firewall DROP must be appended, not -I-inserted above its ACCEPTs' (#75) from fix/signal-relay-fw-order into main

scripts/deploy-remote.sh

@@ -252,16 +252,24 @@ install_signal_relay() {
     # The relay binds the host's LAN IP (the one address a bridge container
     # can reach as host.containers.internal), so keep the unauthenticated
     # endpoint off the rest of the LAN: admit loopback, the host itself
-    # (its own health probe) and the podman bridge subnet only. Idempotent:
-    # -C before -I.
+    # (its own health probe) and the podman bridge subnet only.
+    # ORDER IS THE GATE: `iptables -I` inserts at position 1, so the old
+    # shape (-C/-I x3 ACCEPT, then -C/-I the DROP) landed the blanket DROP
+    # ABOVE the ACCEPTs and every probe timed out beside a LISTENing socket
+    # (live 2026-10-21: 159 packets on the DROP, 0 on each ACCEPT; the whole
+    # dankclaw drain aborted at the alertmanager gate twice before the cause
+    # was read off `nft list ruleset`). Rebuild the tail instead of inserting
+    # it: delete any blanket DROP wherever it sits, keep the three ACCEPTs
+    # at the head (-C before -I, unchanged), then APPEND the DROP so it is
+    # last and the ACCEPTs decide first. Converges on poisoned hosts.
+    run_privileged iptables -D INPUT -p tcp --dport 18093 -j DROP 2>/dev/null || true
     run_privileged iptables -C INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT 2>/dev/null \
         || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT
     run_privileged iptables -C INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT 2>/dev/null \
         || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT
     run_privileged iptables -C INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT 2>/dev/null \
         || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT
-    run_privileged iptables -C INPUT -p tcp --dport 18093 -j DROP 2>/dev/null \
-        || run_privileged iptables -I INPUT -p tcp --dport 18093 -j DROP
+    run_privileged iptables -A INPUT -p tcp --dport 18093 -j DROP
     # The interpreter needs a moment to bind after restart; a one-shot curl
     # raced the socket on 2026-09-17 and failed a healthy deploy.
     local i

serverAppConfigs-docs/Infrastructure/Observability.md

@@ -74,11 +74,19 @@ published ports — node `:9100`, cadvisor `:8080`, gpu `:9401` all answer there
 — while the host's own LAN IP is refused (`192.168.0.143:8080`, `:9401`), so
 every same-host Prometheus target uses the gateway and only cross-host targets
 (dankserver) use plain LAN. Whether LAN works at all is network-specific: the
-relay at `192.168.0.143:18093` IS reached from alertmanager's network, with
-iptables INPUT admitting loopback, the host itself and the podman bridge subnet
-`10.89.0.0/16` before the DROP (`scripts/deploy-remote.sh:255-262`). Never
-point a scrape target at a host LAN IP from its own host on the strength of
-one working example; dial the gateway and probe.
+relay at `192.168.0.143:18093` is reached from alertmanager's network only
+while `install_signal_relay`'s INPUT rules are in force: loopback, the host
+itself and the podman bridge subnet `10.89.0.0/16` accepted **above** a
+trailing blanket DROP. The law the poisoned host taught (2026-10-21):
+`iptables -I` prepends, so inserting the DROP last lands it first — the old
+install order dropped every packet (159 on the DROP, 0 on the ACCEPTs), the
+deployer's health gate failed beside a LISTENing socket, and the drain
+aborted twice before `nft list ruleset` named the cause. The install now
+rebuilds the tail (delete the DROP wherever it sits, ACCEPTs at the head,
+DROP appended); verify any change to it with the sandboxed bats case, not
+with `-C` greps — `-C` answers existence, never order. Never point a scrape
+target at a host LAN IP from its own host on the strength of one working
+example; dial the gateway and probe.
 
 ## Deployment
 

tests/observability_quadlet.bats

@@ -559,6 +559,64 @@ assert any('ALERTS' in e for e in exprs), exprs
     grep -q -- '--dport 18093 -j DROP' "$handler"
 }
 
+@test "the deployer installs the relay firewall with the ACCEPTs deciding before the DROP" {
+    # Behavioral, run for real against a sandbox that plays the kernel: the
+    # rule set is a LIST and iptables -I prepends, so the shape this function
+    # had until 2026-10-21 (-C/-I ACCEPT x3, then -C/-I the blanket DROP)
+    # landed the DROP at position 1 and every packet died there — the drain
+    # failed its health gate beside a LISTENing socket twice. Seeding the
+    # sandbox with that poisoned order also pins the convergent repair: a
+    # re-deploy must rewrite the tail, not skip because -C finds every rule.
+    local sandbox="$BATS_TEST_TMPDIR/fw"
+    mkdir -p "$sandbox/bin"
+    cat > "$sandbox/bin/iptables" <<'EOF'
+#!/usr/bin/env bash
+# ordered rule list in $FW_STATE: -C lookup, -I at head, -A at tail, -D delete.
+state="${FW_STATE:?}"
+touch "$state"
+op="$1"; shift
+case "$op" in
+    -C) grep -Fxq -- "$*" "$state" ;;
+    -I) grep -Fxq -- "$*" "$state" || { { echo "$*"; cat "$state"; } > "$state.new"; mv "$state.new" "$state"; } ;;
+    -A) grep -Fxq -- "$*" "$state" || echo "$*" >> "$state" ;;
+    -D) if grep -Fxq -- "$*" "$state"; then
+            grep -Fxv -- "$*" "$state" > "$state.new"; mv "$state.new" "$state"
+        else exit 1; fi ;;
+    *) exit 0 ;;
+esac
+EOF
+    chmod +x "$sandbox/bin/iptables"
+    # Poisoned state exactly as the pre-fix deployer left Dankclaw.
+    printf '%s\n' \
+        'INPUT -p tcp --dport 18093 -j DROP' \
+        'INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT' \
+        'INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT' \
+        'INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT' \
+        > "$sandbox/rules"
+    (
+        cd "$REPO_ROOT"
+        export PATH="$sandbox/bin:$PATH" FW_STATE="$sandbox/rules" HOME="$sandbox/home"
+        mkdir -p "$HOME"
+        eval "$(sed -n '/^install_signal_relay()/,/^}/p' scripts/deploy-remote.sh)"
+        run_privileged() { "$@"; }
+        systemctl() { :; }
+        curl() { return 0; }
+        sleep() { :; }
+        install_signal_relay
+    )
+    # Exactly one blanket DROP, and it sits after every ACCEPT.
+    [ "$(grep -cF -- 'INPUT -p tcp --dport 18093 -j DROP' "$sandbox/rules")" -eq 1 ]
+    local drop_at total
+    drop_at="$(grep -nF -- 'INPUT -p tcp --dport 18093 -j DROP' "$sandbox/rules" | cut -d: -f1)"
+    total="$(wc -l < "$sandbox/rules")"
+    [ "$drop_at" -eq "$total" ]
+    local s seen
+    for s in 127.0.0.1 192.168.0.143 10.89.0.0/16; do
+        seen="$(grep -nF -- "-s $s -j ACCEPT" "$sandbox/rules" | cut -d: -f1)"
+        [ -n "$seen" ] && [ "$seen" -lt "$drop_at" ]
+    done
+}
+
 @test "the planner selects alertmanager for its config and relay files" {
     run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
         --paths shared/prometheus/alertmanager.yml