From f73fd4302707c52483f727ca026e3eb27ccab8a3 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 17 Sep 2026 13:52:20 -0700 Subject: [PATCH] relay: bind the host LAN IP, admit the podman bridge subnet only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 169.254.1.2 is not a local address on this host (netavark runs its bridges in aardvark-dns' netns), so the relay cannot bind it — and a loopback bind refuses container traffic. Probe-socket proof: a host socket on 192.168.0.143:18094 answers the container's 169.254.1.2:18094 dial while the LAN IP refuses from the container. The relay binds 192.168.0.143:18093 and the deployer installs an iptables INPUT pair (ACCEPT 10.89.0.0/16, DROP rest) so the unauthenticated endpoint stays off the LAN. --- AGENTS.md | 5 ++-- .../quadlet/prometheus.container | 4 +-- scripts/deploy-remote.sh | 15 +++++++--- scripts/signal-relay.py | 28 +++++++++++-------- .../Infrastructure/Alertmanager.md | 4 +-- .../Infrastructure/Observability.md | 2 +- shared/prometheus/alertmanager.yml | 5 ++-- shared/prometheus/prometheus.yml | 4 +-- tests/observability_quadlet.bats | 18 +++++++----- 9 files changed, 52 insertions(+), 33 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index dce376b..a3328b5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -128,8 +128,9 @@ provisioned from `shared/grafana/provisioning/`, `shared/prometheus/rules/ llm-fleet.yml` loads 5 alert rules + 1 recording rule, and firing rules are delivered: Prometheus → Alertmanager (`dankclaw/observability/quadlet/ alertmanager.container`, loopback :9093) → host-side Signal relay -(`scripts/signal-relay.py`, user unit, 169.254.1.2:18093 — the -host.containers.internal endpoint) → signal-cli account 318311. Five +(`scripts/signal-relay.py`, user unit, 192.168.0.143:18093 — the host LAN +IP, the one bind a container can reach as `host.containers.internal`; +iptables admits the podman bridge subnet only) → signal-cli account 318311. Prometheus targets read down for structural reasons (HTML-scraped "probes", Gitea metrics off); the remedy is a blackbox exporter. See `serverAppConfigs-docs/Infrastructure/Observability.md`. diff --git a/dankclaw/observability/quadlet/prometheus.container b/dankclaw/observability/quadlet/prometheus.container index 0dd55d2..2fb30a7 100644 --- a/dankclaw/observability/quadlet/prometheus.container +++ b/dankclaw/observability/quadlet/prometheus.container @@ -29,8 +29,8 @@ # /etc/prometheus/rules/ (installed from shared/prometheus/rules/ by the # deployer). Delivery went live 2026-09-17: the alerting block in # prometheus.yml points at alertmanager:9093, which webhooks every firing -# group to the host-side Signal relay (169.254.1.2:18093, the -# host.containers.internal endpoint), which sends through signal-cli account 318311. +# group to the host-side Signal relay (192.168.0.143:18093, reached from +# the container as host.containers.internal), which sends through signal-cli account 318311. # The Grafana dashboard's ALERTS panel remains the second surface. # # The config volume is also why rules belong in git: editing them in the UI is diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 6f934d9..a708544 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -231,17 +231,24 @@ install_signal_relay() { systemctl --user daemon-reload systemctl --user enable signal-relay.service systemctl --user restart signal-relay.service + # The relay binds the host's LAN IP (the one address a bridge container + # can reach as host.containers.internal), so keep the unauthenticated + # endpoint off the rest of the LAN: admit the podman bridge subnet + # only. Idempotent: -C before -I. + run_privileged iptables -C INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT 2>/dev/null \ + || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT + run_privileged iptables -C INPUT -p tcp --dport 18093 -j DROP 2>/dev/null \ + || run_privileged iptables -I INPUT -p tcp --dport 18093 -j DROP # The interpreter needs a moment to bind after restart; a one-shot curl - # raced the socket on 2026-09-17 and failed a healthy deploy. The relay - # binds 169.254.1.2 (host.containers.internal), not loopback. + # raced the socket on 2026-09-17 and failed a healthy deploy. local i for i in $(seq 1 15); do - if curl -fsS -m 5 http://169.254.1.2:18093/health >/dev/null; then + if curl -fsS -m 5 http://192.168.0.143:18093/health >/dev/null; then return 0 fi sleep 1 done - echo "signal-relay is not answering on 169.254.1.2:18093" >&2 + echo "signal-relay is not answering on 192.168.0.143:18093" >&2 return 1 } diff --git a/scripts/signal-relay.py b/scripts/signal-relay.py index 10e7cd9..f8b5bf2 100755 --- a/scripts/signal-relay.py +++ b/scripts/signal-relay.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 """Alertmanager-to-Signal relay for the homelab (Dankclaw). -Listens on 169.254.1.2:18093 — the host side of host.containers.internal, -the only address a bridge container can use to reach a host-local process — -receives Alertmanager's webhook payload, formats -existing account on this host. Runs as a systemd user unit +Listens on 192.168.0.143:18093 (the host's LAN IP) and receives +Alertmanager's webhook payload from the container as +host.containers.internal:18093 — netavark delivers that address to the +host's LAN IP, which is the one bind a bridge container can reach (a +loopback bind refuses it, and 169.254.1.2 is not a local address here). +The group is formatted into a short message and handed to signal-cli under +the existing account on this host. Runs as a systemd user unit (scripts/signal-relay.service); logs go to the journal. The phone number and account name are numbers, not credentials — the @@ -20,13 +23,16 @@ import sys import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -# 169.254.1.2 is the host endpoint of host.containers.internal: a socket -# bound to 127.0.0.1 refuses container traffic that arrives via the podman -# bridge (measured 2026-09-17: "dial tcp 169.254.1.2:18093: connection -# refused"). The address is link-local to the podman bridge — not on any -# LAN interface, not on loopback, reachable only from this host's -# containers. -LISTEN_ADDR = ("169.254.1.2", 18093) +# The bind address is the one detail this design gets wrong twice +# (2026-09-17): a loopback bind refuses container traffic ("dial tcp +# 169.254.1.2:18093: connection refused"), and 169.254.1.2 itself is not a +# local address on this host ("Cannot assign requested address") — netavark +# runs the bridges in aardvark-dns' netns, not the host's. Traffic that +# arrives as 169.254.1.2 lands on the host's LAN IP, so bind THAT. The +# deployer installs an iptables INPUT rule that admits 18093/tcp from the +# podman bridge subnet (10.89.0.0/16) only, keeping the unauthenticated +# endpoint off the rest of the LAN. +LISTEN_ADDR = ("192.168.0.143", 18093) SIGNAL_CLI = "/usr/local/bin/signal-cli" SIGNAL_ACCOUNT = "318311" SIGNAL_TO = "+12062476336" diff --git a/serverAppConfigs-docs/Infrastructure/Alertmanager.md b/serverAppConfigs-docs/Infrastructure/Alertmanager.md index 80bfe65..68751d9 100644 --- a/serverAppConfigs-docs/Infrastructure/Alertmanager.md +++ b/serverAppConfigs-docs/Infrastructure/Alertmanager.md @@ -19,7 +19,7 @@ Alertmanager receives firing rules from Prometheus, groups and dedupes them, and ## Why the webhook address is `host.containers.internal` -The Signal relay is a host-side process, and the address it must bind is the one detail this design gets wrong once: **127.0.0.1 does not work**. From a bridge container, loopback is the container's own, and netavark DNATs *published* ports only — the LAN IP and the podman gateway are both refused for an unpublished host bind. `host.containers.internal` (169.254.1.2) is the one address that reaches a host-local listener from a bridge container, so the relay binds *that* address (measured 2026-09-17: a loopback bind refused the first webhook with `dial tcp 169.254.1.2:18093: connection refused`). 169.254.1.2 is link-local to the podman bridge — not on loopback, not on any LAN interface, reachable only from this host's containers — so the unauthenticated webhook endpoint stays off the wire. +The Signal relay is a host-side process, and the address it must bind is the one detail this design gets wrong twice (2026-09-17). From a bridge container, loopback is the container's own, and netavark DNATs *published* ports only — so a relay bound to 127.0.0.1 refused the first webhook with `dial tcp 169.254.1.2:18093: connection refused`. The obvious fix — bind `host.containers.internal` itself — fails too: on this host netavark runs its bridges in aardvark-dns' netns, so 169.254.1.2 is not a local address in the host's namespace (`OSError: [Errno 99] Cannot assign requested address`). What works: bind the **host's LAN IP** (192.168.0.143) — netavark delivers the container's `169.254.1.2` dial to that address (proven with a probe socket: container → `169.254.1.2:18094` → host socket bound to `192.168.0.143:18094`, while the LAN IP refused from the container, the usual netavark quirk). Since that puts an unauthenticated endpoint on a host interface, the deployer installs an iptables INPUT pair admitting 18093/tcp from the podman bridge subnet (10.89.0.0/16) only — the rest of the LAN gets a DROP. ## The Signal relay @@ -36,7 +36,7 @@ The Signal relay is a host-side process, and the address it must bind is the one ./scripts/deploy.sh dankclaw --service alertmanager ``` -`deploy_observability` installs the quadlet unit, `copy_alertmanager_config` installs the config, and `install_signal_relay` copies the user unit, enables and restarts it, then waits on `169.254.1.2:18093/health` before the container's own `/-/healthy` wait. The planner selects `alertmanager` for its quadlet file, `shared/prometheus/alertmanager.yml`, and the two relay files; `prometheus` ships the `alerting:` block that points at `alertmanager:9093`. +`deploy_observability` installs the quadlet unit, `copy_alertmanager_config` installs the config, and `install_signal_relay` copies the user unit, installs the iptables pair, enables and restarts the relay, then waits on `192.168.0.143:18093/health` before the container's own `/-/healthy` wait. The planner selects `alertmanager` for its quadlet file, `shared/prometheus/alertmanager.yml`, and the two relay files; `prometheus` ships the `alerting:` block that points at `alertmanager:9093`. ## Gotchas diff --git a/serverAppConfigs-docs/Infrastructure/Observability.md b/serverAppConfigs-docs/Infrastructure/Observability.md index c964ab5..c57cf55 100644 --- a/serverAppConfigs-docs/Infrastructure/Observability.md +++ b/serverAppConfigs-docs/Infrastructure/Observability.md @@ -40,7 +40,7 @@ Recorded 2026-09-11, on the day the stack moved to Quadlet units. Everything in - **Grafana has a datasource and one dashboard as of 2026-09-16; before that it had neither.** `shared/grafana/provisioning/` is installed to `/opt/grafana/provisioning` by the deployer and read by the container: the Prometheus datasource (`http://prometheus:9090`, fixed uid `prometheus`) and the **LLM fleet — capacity and reuse** dashboard (speculative acceptance, queue depth, prefix-cache reuse, decode step rate, throughput) covering the llama.cpp endpoints. **Re-cut on 2026-09-16**: the panels were keyed to `vllm:` series, which stopped existing when frank-medium moved to llama.cpp, and two of them (KV-pool fill, request-latency p95) have no llama.cpp equivalent at all — those two are retired rather than restated, and the acceptance panel replaces the KV one. A renamed query against a series that does not exist is a panel that draws nothing forever. Verified through the API rather than by eye — the datasource answers `llamacpp:` counters for `frank-medium`, and `GET /api/search?type=dash-db` returns uid `llm-fleet`. One trap worth knowing: a provisioned dashboard does **not** appear in the legacy `dashboard` table on this Grafana version (it lives in unified storage … - **Grafana's dashboard now also surfaces firing alerts.** The **Alerts firing (ALERTS)** panel graphs `ALERTS{alertstate="firing"}` for the LLM rules, because this stack has no other place where an alert becomes visible without opening Prometheus. -- **Prometheus now evaluates rules, and they are delivered (2026-09-17).** `shared/prometheus/rules/llm-fleet.yml` ships with the config (the deployer's `copy_prometheus_config` installs both — a rule file nothing loads evaluates nothing, and Prometheus reports no error for that) and holds five rules for the LLM fleet: request queueing, prefix-cache collapse, speculative decoding off, endpoint down, plus the `job:llm_prefix_cache_reuse:rate5m` recording rule the alert and the dashboard share. **Two rules from the vLLM era were retired, not renamed**, because the series they watched no longer exist: KV-pool saturation (llama.cpp exports no KV-fill gauge; its consequence *is* queueing, which `LlmRequestsQueueing` already covers) and p95 request latency (no latency histogram). `LlmSpecDecodeOff` takes the second slot — the failure a client cannot see, since a lost drafter just halves the speed. Verified with `promtool check rules` and `/api/v1/rules` (`health=ok` for all five). **Delivery went live 2026-09-17**: the `alerting:` block in `prometheus.yml` points at `alertmanager:9093` (container name on `dankclaw_default` — the published port is loopback-only and would be wrong from inside Prometheus). Alertmanager groups by `alertname` (30 s group wait, 5 m interval, 4 h repeat) and webhooks the host-side **Signal relay** at `host.containers.internal:18093/alertmanager` — the relay binds 169.254.1.2, the host-gateway endpoint: a loopback bind refuses container traffic (measured on the first deploy), and a host-interface bind would put an unauthenticated endpoint on the LAN. The relay (`scripts/signal-relay.py`, a systemd *user* unit, not a quadlet — it runs python3 from the checkout and can be enabled) dedupes per group for 15 minutes and sends through **signal-cli account 318311** (`/usr/local/bin/signal-cli`, state in `~/.local/share/signal-cli`), Frank's existing account. The Grafana dashboard's ALERTS panel and Prometheus' `/alerts` remain the second and third surfaces. +- **Prometheus now evaluates rules, and they are delivered (2026-09-17).** `shared/prometheus/rules/llm-fleet.yml` ships with the config (the deployer's `copy_prometheus_config` installs both — a rule file nothing loads evaluates nothing, and Prometheus reports no error for that) and holds five rules for the LLM fleet: request queueing, prefix-cache collapse, speculative decoding off, endpoint down, plus the `job:llm_prefix_cache_reuse:rate5m` recording rule the alert and the dashboard share. **Two rules from the vLLM era were retired, not renamed**, because the series they watched no longer exist: KV-pool saturation (llama.cpp exports no KV-fill gauge; its consequence *is* queueing, which `LlmRequestsQueueing` already covers) and p95 request latency (no latency histogram). `LlmSpecDecodeOff` takes the second slot — the failure a client cannot see, since a lost drafter just halves the speed. Verified with `promtool check rules` and `/api/v1/rules` (`health=ok` for all five). **Delivery went live 2026-09-17**: the `alerting:` block in `prometheus.yml` points at `alertmanager:9093` (container name on `dankclaw_default` — the published port is loopback-only and would be wrong from inside Prometheus). Alertmanager groups by `alertname` (30 s group wait, 5 m interval, 4 h repeat) and webhooks the host-side **Signal relay** at `host.containers.internal:18093/alertmanager` — the relay binds the host's LAN IP (192.168.0.143:18093, the one address netavark delivers that dial to), with an iptables pair admitting the podman bridge subnet only. The relay (`scripts/signal-relay.py`, a systemd *user* unit, not a quadlet — it runs python3 from the checkout and can be enabled) dedupes per group for 15 minutes and sends through **signal-cli account 318311** (`/usr/local/bin/signal-cli`, state in `~/.local/share/signal-cli`), Frank's existing account. The Grafana dashboard's ALERTS panel and Prometheus' `/alerts` remain the second and third surfaces. - **Loki is configured but unread.** Storage is filesystem with 30-day retention, `auth_enabled: false`, and the push path works; Grafana can now reach it in Explore, but no dashboard or alert consumes it. - **node-exporter runs its default collector set** — no collector tuning, no textfile collector for custom metrics. - **Alloy is the one configured piece.** It ships the host journal and podman container logs to Loki with `job` labels, and that pipeline is live. diff --git a/shared/prometheus/alertmanager.yml b/shared/prometheus/alertmanager.yml index c9dc6e9..18a4dfa 100644 --- a/shared/prometheus/alertmanager.yml +++ b/shared/prometheus/alertmanager.yml @@ -2,8 +2,9 @@ # Deployed to /opt/alertmanager/config/alertmanager.yml on Dankclaw. # # The one delivery path is Signal. The webhook lands on the host-side -# signal-relay (a systemd user unit listening on 169.254.1.2:18093, the -# host.containers.internal endpoint), which +# signal-relay (a systemd user unit listening on 192.168.0.143:18093, the +# host LAN IP — the address the container's host.containers.internal dial +# lands on), which # formats the group and hands it to signal-cli under Frank's existing # account. There is deliberately no second receiver: a duplicate channel is # how an alert starts arriving twice and getting muted everywhere. diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index 6dc88c5..675df33 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -14,8 +14,8 @@ rule_files: # The delivery half. Alertmanager runs on the same dankclaw_default network, # so it is addressed by container name, like grafana and loki below. Its -# single receiver webhooks the host-side signal-relay (169.254.1.2:18093, -# the host.containers.internal endpoint). +# single receiver webhooks the host-side signal-relay (192.168.0.143:18093, +# reached from the container as host.containers.internal). alerting: alertmanagers: - static_configs: diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index a718c0b..271939c 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -461,18 +461,22 @@ assert any('ALERTS' in e for e in exprs), exprs grep -q 'install_signal_relay' <(sed -n '/^deploy_observability()/,/^}/p' "$handler") } -@test "the relay sends through the existing signal-cli account, podman-bridge only" { +@test "the relay sends through the existing signal-cli account, bridge-subnet only" { # The credential is the account state under ~/.local/share/signal-cli; - # the number and account name are numbers, not secrets. The listener must - # bind 169.254.1.2 (host.containers.internal): a loopback bind refuses - # container traffic (measured 2026-09-17), and a host-interface bind - # would put an unauthenticated message-sending endpoint on the LAN — - # the failure this whole unit family exists to avoid. + # the number and account name are numbers, not secrets. The listener + # binds the host's LAN IP — the one address a bridge container can + # reach as host.containers.internal (measured 2026-09-17: loopback + # refuses, 169.254.1.2 is not local) — and the deployer admits the + # podman bridge subnet only, so the unauthenticated endpoint stays off + # the rest of the LAN. local relay="$REPO_ROOT/scripts/signal-relay.py" + local handler="$REPO_ROOT/scripts/deploy-remote.sh" grep -q 'signal-cli' "$relay" grep -q 'SIGNAL_CLI = "/usr/local/bin/signal-cli"' "$relay" grep -q 'SIGNAL_ACCOUNT = "318311"' "$relay" - grep -q 'LISTEN_ADDR = ("169.254.1.2", 18093)' "$relay" + grep -q 'LISTEN_ADDR = ("192.168.0.143", 18093)' "$relay" + grep -q -- '--dport 18093 -s 10.89.0.0/16 -j ACCEPT' "$handler" + grep -q -- '--dport 18093 -j DROP' "$handler" } @test "the planner selects alertmanager for its config and relay files" {