diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 777ec0a..e1e5444 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -252,16 +252,24 @@ install_signal_relay() { # The relay binds the host's LAN IP (the one address a bridge container # can reach as host.containers.internal), so keep the unauthenticated # endpoint off the rest of the LAN: admit loopback, the host itself - # (its own health probe) and the podman bridge subnet only. Idempotent: - # -C before -I. + # (its own health probe) and the podman bridge subnet only. + # ORDER IS THE GATE: `iptables -I` inserts at position 1, so the old + # shape (-C/-I x3 ACCEPT, then -C/-I the DROP) landed the blanket DROP + # ABOVE the ACCEPTs and every probe timed out beside a LISTENing socket + # (live 2026-10-21: 159 packets on the DROP, 0 on each ACCEPT; the whole + # dankclaw drain aborted at the alertmanager gate twice before the cause + # was read off `nft list ruleset`). Rebuild the tail instead of inserting + # it: delete any blanket DROP wherever it sits, keep the three ACCEPTs + # at the head (-C before -I, unchanged), then APPEND the DROP so it is + # last and the ACCEPTs decide first. Converges on poisoned hosts. + run_privileged iptables -D INPUT -p tcp --dport 18093 -j DROP 2>/dev/null || true run_privileged iptables -C INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT 2>/dev/null \ || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT run_privileged iptables -C INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT 2>/dev/null \ || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT run_privileged iptables -C INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT 2>/dev/null \ || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT - run_privileged iptables -C INPUT -p tcp --dport 18093 -j DROP 2>/dev/null \ - || run_privileged iptables -I INPUT -p tcp --dport 18093 -j DROP + run_privileged iptables -A INPUT -p tcp --dport 18093 -j DROP # The interpreter needs a moment to bind after restart; a one-shot curl # raced the socket on 2026-09-17 and failed a healthy deploy. local i diff --git a/serverAppConfigs-docs/Infrastructure/Observability.md b/serverAppConfigs-docs/Infrastructure/Observability.md index 8205bb6..39a19f9 100644 --- a/serverAppConfigs-docs/Infrastructure/Observability.md +++ b/serverAppConfigs-docs/Infrastructure/Observability.md @@ -74,11 +74,19 @@ published ports — node `:9100`, cadvisor `:8080`, gpu `:9401` all answer there — while the host's own LAN IP is refused (`192.168.0.143:8080`, `:9401`), so every same-host Prometheus target uses the gateway and only cross-host targets (dankserver) use plain LAN. Whether LAN works at all is network-specific: the -relay at `192.168.0.143:18093` IS reached from alertmanager's network, with -iptables INPUT admitting loopback, the host itself and the podman bridge subnet -`10.89.0.0/16` before the DROP (`scripts/deploy-remote.sh:255-262`). Never -point a scrape target at a host LAN IP from its own host on the strength of -one working example; dial the gateway and probe. +relay at `192.168.0.143:18093` is reached from alertmanager's network only +while `install_signal_relay`'s INPUT rules are in force: loopback, the host +itself and the podman bridge subnet `10.89.0.0/16` accepted **above** a +trailing blanket DROP. The law the poisoned host taught (2026-10-21): +`iptables -I` prepends, so inserting the DROP last lands it first — the old +install order dropped every packet (159 on the DROP, 0 on the ACCEPTs), the +deployer's health gate failed beside a LISTENing socket, and the drain +aborted twice before `nft list ruleset` named the cause. The install now +rebuilds the tail (delete the DROP wherever it sits, ACCEPTs at the head, +DROP appended); verify any change to it with the sandboxed bats case, not +with `-C` greps — `-C` answers existence, never order. Never point a scrape +target at a host LAN IP from its own host on the strength of one working +example; dial the gateway and probe. ## Deployment diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index 36ba95e..f9a07b4 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -559,6 +559,64 @@ assert any('ALERTS' in e for e in exprs), exprs grep -q -- '--dport 18093 -j DROP' "$handler" } +@test "the deployer installs the relay firewall with the ACCEPTs deciding before the DROP" { + # Behavioral, run for real against a sandbox that plays the kernel: the + # rule set is a LIST and iptables -I prepends, so the shape this function + # had until 2026-10-21 (-C/-I ACCEPT x3, then -C/-I the blanket DROP) + # landed the DROP at position 1 and every packet died there — the drain + # failed its health gate beside a LISTENing socket twice. Seeding the + # sandbox with that poisoned order also pins the convergent repair: a + # re-deploy must rewrite the tail, not skip because -C finds every rule. + local sandbox="$BATS_TEST_TMPDIR/fw" + mkdir -p "$sandbox/bin" + cat > "$sandbox/bin/iptables" <<'EOF' +#!/usr/bin/env bash +# ordered rule list in $FW_STATE: -C lookup, -I at head, -A at tail, -D delete. +state="${FW_STATE:?}" +touch "$state" +op="$1"; shift +case "$op" in + -C) grep -Fxq -- "$*" "$state" ;; + -I) grep -Fxq -- "$*" "$state" || { { echo "$*"; cat "$state"; } > "$state.new"; mv "$state.new" "$state"; } ;; + -A) grep -Fxq -- "$*" "$state" || echo "$*" >> "$state" ;; + -D) if grep -Fxq -- "$*" "$state"; then + grep -Fxv -- "$*" "$state" > "$state.new"; mv "$state.new" "$state" + else exit 1; fi ;; + *) exit 0 ;; +esac +EOF + chmod +x "$sandbox/bin/iptables" + # Poisoned state exactly as the pre-fix deployer left Dankclaw. + printf '%s\n' \ + 'INPUT -p tcp --dport 18093 -j DROP' \ + 'INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT' \ + 'INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT' \ + 'INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT' \ + > "$sandbox/rules" + ( + cd "$REPO_ROOT" + export PATH="$sandbox/bin:$PATH" FW_STATE="$sandbox/rules" HOME="$sandbox/home" + mkdir -p "$HOME" + eval "$(sed -n '/^install_signal_relay()/,/^}/p' scripts/deploy-remote.sh)" + run_privileged() { "$@"; } + systemctl() { :; } + curl() { return 0; } + sleep() { :; } + install_signal_relay + ) + # Exactly one blanket DROP, and it sits after every ACCEPT. + [ "$(grep -cF -- 'INPUT -p tcp --dport 18093 -j DROP' "$sandbox/rules")" -eq 1 ] + local drop_at total + drop_at="$(grep -nF -- 'INPUT -p tcp --dport 18093 -j DROP' "$sandbox/rules" | cut -d: -f1)" + total="$(wc -l < "$sandbox/rules")" + [ "$drop_at" -eq "$total" ] + local s seen + for s in 127.0.0.1 192.168.0.143 10.89.0.0/16; do + seen="$(grep -nF -- "-s $s -j ACCEPT" "$sandbox/rules" | cut -d: -f1)" + [ -n "$seen" ] && [ "$seen" -lt "$drop_at" ] + done +} + @test "the planner selects alertmanager for its config and relay files" { run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \ --paths shared/prometheus/alertmanager.yml