From 5c1006dda69804b931ce791f872dd159246a0b69 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 21 Sep 2026 11:09:03 -0700 Subject: [PATCH] fix(monitoring): scrape node-dankclaw via host.containers.internal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit node-exporter is net=host, so its 192.168.0.143:9100 bind is a host bind, not a published port — the bridge-networked Prometheus container cannot reach its own host's LAN IP there (netavark DNATs published ports only). The node-dankclaw target has been red by construction since the observability cutover while the container reported healthy. Point it at host.containers.internal:9100 (netavark delivers that to the host LAN IP, where node-exporter listens) — the same rule frank-large and frank-small already use. node-dankserver is a different machine and keeps its LAN address. Update the cross-file contract test and the Targets table. --- .../Infrastructure/Prometheus.md | 6 +++-- shared/prometheus/prometheus.yml | 10 ++++++-- tests/observability_quadlet.bats | 23 +++++++++++++++---- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/serverAppConfigs-docs/Infrastructure/Prometheus.md b/serverAppConfigs-docs/Infrastructure/Prometheus.md index f46785f..ea93bbc 100644 --- a/serverAppConfigs-docs/Infrastructure/Prometheus.md +++ b/serverAppConfigs-docs/Infrastructure/Prometheus.md @@ -25,9 +25,11 @@ Authentik; post-auth the Caddy vhost routes straight to `prometheus:9090`) — | Job | Target | Why | |-----|--------|-----| | `prometheus` | `localhost:9090` | itself | -| `node-dankclaw`, `node-dankserver` | `192.168.0.143:9100`, `192.168.0.235:9100` | host metrics from each machine's [[Node-Exporter]] | +| `node-dankclaw` | `host.containers.internal:9100` | host metrics from [[Node-Exporter]] — a net=host bind, so via the gateway (the LAN address is unreachable from this bridge container) | +| `node-dankserver` | `192.168.0.235:9100` | host metrics from [[Node-Exporter]] — a different machine, so the LAN address works | | `grafana`, `loki` | `grafana:3000`, `loki:3100` | container names on the `dankclaw_default` network; both serve real `/metrics` | -| `frank-medium`, `frank-large` | `192.168.0.235:7770`, `192.168.0.143:7770` | the LLM fleet — `vllm:` and `llamacpp:` series (LAN addresses, not the guarded vhosts) | +| `frank-medium` | `192.168.0.235:7770` | the LLM fleet — `vllm:` series; a different machine, so the LAN address works | +| `frank-large`, `frank-small` | `host.containers.internal:7770`, `:7771` | the LLM fleet — `llamacpp:` series; same-host net=host binds, so via the gateway | **Three jobs read down, and they are not outages** (audited 2026-09-16): diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index 3e647d8..a2a828f 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -27,10 +27,16 @@ scrape_configs: static_configs: - targets: ["localhost:9090"] - # Host metrics — both hosts + # Host metrics — both hosts. node-dankclaw is scraped over + # host.containers.internal, not its own LAN address: this Prometheus is a + # bridge container, and on this netavark host a container can only reach the + # host's LAN IP at a PUBLISHED port — node-exporter's net=host bind on + # 192.168.0.143:9100 is refused by construction (the same rule that sends + # frank-large and frank-small to the gateway below). node-dankserver is a + # different machine, so its LAN address works. - job_name: "node-dankclaw" static_configs: - - targets: ["192.168.0.143:9100"] + - targets: ["host.containers.internal:9100"] - job_name: "node-dankserver" static_configs: diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index 300f646..829d03d 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -108,12 +108,27 @@ service_section() { @test "prometheus scrapes node-exporter where node-exporter actually listens" { # Cross-file contract: if either side's address changes alone, host metrics # silently stop with no error anywhere. + # + # This Prometheus is a bridge container (Network=dankclaw_default), and on + # this netavark host a container can only reach the host's LAN IP at a + # PUBLISHED port. node-exporter is net=host, so its 192.168.0.143:9100 bind + # is a host bind, not a published port — unreachable at its own LAN address. + # The node-dankclaw job therefore scrapes host.containers.internal:9100, + # which netavark delivers to the host's LAN IP, where node-exporter listens. + # (node-dankserver is a different machine, so its LAN address is reached + # over the real network and is scraped directly.) local unit="$UNIT_DIR/node-exporter.container" local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" - local addr - addr=$(grep -oE '192\.168\.0\.[0-9]+:9100' "$unit" | head -1) - [ -n "$addr" ] || { echo "node-exporter has no LAN listen address"; return 1; } - grep -q "\"$addr\"" "$cfg" || { echo "prometheus does not scrape $addr"; return 1; } + local lan + lan=$(grep -oE '192\.168\.0\.[0-9]+:9100' "$unit" | head -1) + [ -n "$lan" ] || { echo "node-exporter has no LAN listen address"; return 1; } + # The exporter must still bind that LAN address — it is where the + # host.containers.internal delivery lands. + grep -q -- "--web.listen-address=$lan" "$unit" \ + || { echo "node-exporter no longer binds $lan"; return 1; } + # The local job scrapes via the gateway, not the (unreachable) LAN address. + grep -q '"host.containers.internal:9100"' "$cfg" \ + || { echo "node-dankclaw does not scrape host.containers.internal:9100"; return 1; } } @test "container names still match what Caddy and Prometheus address by name" {