From 650b6e02813c8da4d870197012bbf60257095e8b Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 00:34:17 -0700 Subject: [PATCH] fix(observability): serve every exporter bind; scrape same-host cadvisor via gateway Two shipped-2026-10-20 bugs that left gpu-dankclaw, gpu-dankserver and cadvisor-dankclaw DOWN beside a green deploy: - gpu-exporter.py built one ThreadingHTTPServer per bind but ran serve_forever() on servers[0] only: the LAN-class socket sat listen()ing but never accept()ed, loopback deploy gate passed, every scrape that reached the other bind timed out on the SYN backlog. Now one thread per extra bind. Regression test spins --serve on two loopback binds and requires BOTH to answer /metrics. - cadvisor-dankclaw scraped its own host's LAN IP. Measured from inside the Prometheus container: a bridge container cannot dial its own host's LAN address at all (hairpin refuses published ports too); the podman gateway answers. Target moved to host.containers.internal:8080; the seam that pinned the LAN address (asserting the wrong doctrine) is re-pinned to the gateway plus a negative pin against the own-host LAN dial. Observability page records the measured reachability law replacing the inferred one. --- scripts/gpu-exporter.py | 6 ++++ .../Infrastructure/Observability.md | 16 +++++++--- shared/prometheus/prometheus.yml | 11 ++++--- tests/gpu_exporter.bats | 30 +++++++++++++++++++ tests/observability_quadlet.bats | 20 ++++++++----- 5 files changed, 68 insertions(+), 15 deletions(-) diff --git a/scripts/gpu-exporter.py b/scripts/gpu-exporter.py index b315319..39dcafd 100755 --- a/scripts/gpu-exporter.py +++ b/scripts/gpu-exporter.py @@ -32,6 +32,7 @@ import glob import os import subprocess import sys +import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer FAMILIES = [ @@ -198,6 +199,11 @@ def main(argv): return 2 servers.append(ThreadingHTTPServer((b, args.port), Handler)) sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds))) + # EVERY socket must be served: a listen()ed-but-never-accept()ed bind looks + # healthy in `ss` yet times out every scrape once its SYN backlog fills — + # and the loopback deploy gate cannot see it. One thread per extra bind. + for s in servers[1:]: + threading.Thread(target=s.serve_forever, daemon=True).start() servers[0].serve_forever() return 0 diff --git a/serverAppConfigs-docs/Infrastructure/Observability.md b/serverAppConfigs-docs/Infrastructure/Observability.md index 30dd860..0203048 100644 --- a/serverAppConfigs-docs/Infrastructure/Observability.md +++ b/serverAppConfigs-docs/Infrastructure/Observability.md @@ -67,10 +67,18 @@ Alertmanager has no vhost at all. node-exporter is the exception that must live in the host network namespace, and it binds exactly two addresses rather than the wildcard — see [[Node-Exporter]]. -The relay's bind is the one address a bridge container can reach on the host: -`192.168.0.143:18093`, with iptables INPUT admitting loopback, the host itself -and the podman bridge subnet `10.89.0.0/16` before the DROP -(`scripts/deploy-remote.sh:255-262`). +**Same-host reachability from a bridge container is measured, not inferred** +(2026-10-21, from inside the Prometheus container): `host.containers.internal` +(the podman gateway) reaches host services whether they bind host-net or are +published ports — node `:9100`, cadvisor `:8080`, gpu `:9401` all answer there +— while the host's own LAN IP is refused (`192.168.0.143:8080`, `:9401`), so +every same-host Prometheus target uses the gateway and only cross-host targets +(dankserver) use plain LAN. Whether LAN works at all is network-specific: the +relay at `192.168.0.143:18093` IS reached from alertmanager's network, with +iptables INPUT admitting loopback, the host itself and the podman bridge subnet +`10.89.0.0/16` before the DROP (`scripts/deploy-remote.sh:255-262`). Never +point a scrape target at a host LAN IP from its own host on the strength of +one working example; dial the gateway and probe. ## Deployment diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index 2342955..94a7c49 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -140,12 +140,15 @@ scrape_configs: # system.slice daemons and machine.slice scopes carry no such name and are # dropped, as is the per-CPU-core split (only cpu="total" survives), so the # scrape stays small. cadvisor cannot bind per-interface, so it is reached at - # the host's PUBLISHED LAN port: unlike node-exporter's host-net bind (which - # is why node-dankclaw scrapes via host.containers.internal), a published port - # IS reachable from this bridge container at the host LAN address. + # the podman gateway like every same-host target: a BRIDGE CONTAINER CANNOT + # DIAL ITS OWN HOST'S LAN IP AT ALL — the netavark hairpin gap refuses the + # connection whether the port is a host-net bind or a published port (the + # original note here claimed published ports were exempt; measured 2026-10-21 + # from inside this container: LAN refused, gateway answers). Cross-host + # targets (dankserver) have no hairpin problem and stay plain LAN. - job_name: "cadvisor-dankclaw" static_configs: - - targets: ["192.168.0.143:8080"] + - targets: ["host.containers.internal:8080"] labels: host: dankclaw metric_relabel_configs: diff --git a/tests/gpu_exporter.bats b/tests/gpu_exporter.bats index 6275c79..2f4cde9 100755 --- a/tests/gpu_exporter.bats +++ b/tests/gpu_exporter.bats @@ -195,3 +195,33 @@ for want in ('gpu_utilization_percent', 'gpu_memory_used_bytes', print('gpu dashboard panels ok') " } + +@test "--serve answers on EVERY declared bind, not just the first" { + # The deployed unit serves loopback + the host LAN address, and Prometheus + # reaches the host through the gateway — which dials the LAN-class bind. + # A server that only runs serve_forever() on the first socket leaves the + # second one listen()ing-but-never-accepting: the deploy health gate on + # loopback passes, every scrape on the other bind dies on the SYN backlog, + # and no gate but this one sees it. (This exact bug shipped 2026-10-21 and + # left both gpu-* targets down beside a green deploy.) + mk_amd 0 42 1073741824 34359738368 "" 71000 + port=$((19400 + RANDOM % 500)) + python3 "$EXPORTER" --serve --amdgpu-sysfs "$SYS" --nvidia-cmd false \ + --bind 127.0.0.1 --bind 127.0.0.2 --port "$port" \ + > "$BATS_TEST_TMPDIR/serve.log" 2>&1 & + local pid=$! + trap 'kill "$pid" 2>/dev/null' EXIT + local ok=0 + for _ in $(seq 1 20); do + curl -fsS -m2 "http://127.0.0.1:$port/metrics" >/dev/null 2>&1 && { ok=1; break; } + sleep 0.25 + done + [ "$ok" -eq 1 ] || { kill "$pid" 2>/dev/null; return 1; } + # Both binds must ANSWER within 2 s each — the second one is the bug. + for b in 127.0.0.1 127.0.0.2; do + curl -fsS -m2 "http://$b:$port/metrics" | grep -q '^gpu_utilization_percent{' \ + || { echo "bind $b did not answer /metrics"; kill "$pid" 2>/dev/null; return 1; } + done + kill "$pid" 2>/dev/null + trap - EXIT +} diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index 53547c7..ce22bc4 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -595,16 +595,22 @@ assert any('ALERTS' in e for e in exprs), exprs } @test "Prometheus scrapes cadvisor where it is published and relabels the cgroup path to a container name" { - # Cross-file contract: cadvisor publishes the LAN port; a cadvisor scrape - # job pointing anywhere else reads nothing (unlike node-exporter's host-net - # bind, a published port IS reachable from this bridge container at the LAN - # address). And the whole dashboard depends on the id -> container relabel: - # cadvisor names nothing itself, the quadlet unit name is only in the cgroup - # path. If either half changes alone, the Containers dashboard is blank. + # Cross-file contract, measured 2026-10-21 from inside the Prometheus + # container: a bridge container cannot dial its OWN host's LAN IP — the + # hairpin refuses published ports exactly like host-net binds, so + # cadvisor-dankclaw must scrape via the podman gateway (the original pin + # here demanded the LAN address and shipped a permanently-down target). + # dankserver is a different machine: plain LAN there. And the whole + # dashboard depends on the id -> container relabel: cadvisor names nothing + # itself, the quadlet unit name is only in the cgroup path. If either half + # changes alone, the Containers dashboard is blank. local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" grep -q 'job_name: "cadvisor-dankclaw"' "$cfg" || { echo "no cadvisor-dankclaw job"; return 1; } grep -q 'job_name: "cadvisor-dankserver"' "$cfg" || { echo "no cadvisor-dankserver job"; return 1; } - grep -q '"192.168.0.143:8080"' "$cfg" || { echo "cadvisor-dankclaw does not scrape the published LAN port"; return 1; } + grep -q '"host.containers.internal:8080"' "$cfg" \ + || { echo "cadvisor-dankclaw does not scrape via the podman gateway"; return 1; } + ! grep -q '"192.168.0.143:8080"' "$cfg" \ + || { echo "cadvisor-dankclaw dials its own host LAN IP — hairpin refuses it"; return 1; } grep -q '"192.168.0.235:8080"' "$cfg" || { echo "cadvisor-dankserver does not scrape the published LAN port"; return 1; } # The relabel that turns …/app.slice/.service… into a `container`. grep -q 'app\\.slice/(\[\^\.\]+)\\.service' "$cfg" || { echo "no cgroup-path -> container relabel"; return 1; }