diff --git a/scripts/gpu-exporter.py b/scripts/gpu-exporter.py index b315319..39dcafd 100755 --- a/scripts/gpu-exporter.py +++ b/scripts/gpu-exporter.py @@ -32,6 +32,7 @@ import glob import os import subprocess import sys +import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer FAMILIES = [ @@ -198,6 +199,11 @@ def main(argv): return 2 servers.append(ThreadingHTTPServer((b, args.port), Handler)) sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds))) + # EVERY socket must be served: a listen()ed-but-never-accept()ed bind looks + # healthy in `ss` yet times out every scrape once its SYN backlog fills — + # and the loopback deploy gate cannot see it. One thread per extra bind. + for s in servers[1:]: + threading.Thread(target=s.serve_forever, daemon=True).start() servers[0].serve_forever() return 0 diff --git a/serverAppConfigs-docs/Infrastructure/Observability.md b/serverAppConfigs-docs/Infrastructure/Observability.md index 30dd860..0203048 100644 --- a/serverAppConfigs-docs/Infrastructure/Observability.md +++ b/serverAppConfigs-docs/Infrastructure/Observability.md @@ -67,10 +67,18 @@ Alertmanager has no vhost at all. node-exporter is the exception that must live in the host network namespace, and it binds exactly two addresses rather than the wildcard — see [[Node-Exporter]]. -The relay's bind is the one address a bridge container can reach on the host: -`192.168.0.143:18093`, with iptables INPUT admitting loopback, the host itself -and the podman bridge subnet `10.89.0.0/16` before the DROP -(`scripts/deploy-remote.sh:255-262`). +**Same-host reachability from a bridge container is measured, not inferred** +(2026-10-21, from inside the Prometheus container): `host.containers.internal` +(the podman gateway) reaches host services whether they bind host-net or are +published ports — node `:9100`, cadvisor `:8080`, gpu `:9401` all answer there +— while the host's own LAN IP is refused (`192.168.0.143:8080`, `:9401`), so +every same-host Prometheus target uses the gateway and only cross-host targets +(dankserver) use plain LAN. Whether LAN works at all is network-specific: the +relay at `192.168.0.143:18093` IS reached from alertmanager's network, with +iptables INPUT admitting loopback, the host itself and the podman bridge subnet +`10.89.0.0/16` before the DROP (`scripts/deploy-remote.sh:255-262`). Never +point a scrape target at a host LAN IP from its own host on the strength of +one working example; dial the gateway and probe. ## Deployment diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index 2342955..94a7c49 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -140,12 +140,15 @@ scrape_configs: # system.slice daemons and machine.slice scopes carry no such name and are # dropped, as is the per-CPU-core split (only cpu="total" survives), so the # scrape stays small. cadvisor cannot bind per-interface, so it is reached at - # the host's PUBLISHED LAN port: unlike node-exporter's host-net bind (which - # is why node-dankclaw scrapes via host.containers.internal), a published port - # IS reachable from this bridge container at the host LAN address. + # the podman gateway like every same-host target: a BRIDGE CONTAINER CANNOT + # DIAL ITS OWN HOST'S LAN IP AT ALL — the netavark hairpin gap refuses the + # connection whether the port is a host-net bind or a published port (the + # original note here claimed published ports were exempt; measured 2026-10-21 + # from inside this container: LAN refused, gateway answers). Cross-host + # targets (dankserver) have no hairpin problem and stay plain LAN. - job_name: "cadvisor-dankclaw" static_configs: - - targets: ["192.168.0.143:8080"] + - targets: ["host.containers.internal:8080"] labels: host: dankclaw metric_relabel_configs: diff --git a/tests/gpu_exporter.bats b/tests/gpu_exporter.bats index 6275c79..2f4cde9 100755 --- a/tests/gpu_exporter.bats +++ b/tests/gpu_exporter.bats @@ -195,3 +195,33 @@ for want in ('gpu_utilization_percent', 'gpu_memory_used_bytes', print('gpu dashboard panels ok') " } + +@test "--serve answers on EVERY declared bind, not just the first" { + # The deployed unit serves loopback + the host LAN address, and Prometheus + # reaches the host through the gateway — which dials the LAN-class bind. + # A server that only runs serve_forever() on the first socket leaves the + # second one listen()ing-but-never-accepting: the deploy health gate on + # loopback passes, every scrape on the other bind dies on the SYN backlog, + # and no gate but this one sees it. (This exact bug shipped 2026-10-21 and + # left both gpu-* targets down beside a green deploy.) + mk_amd 0 42 1073741824 34359738368 "" 71000 + port=$((19400 + RANDOM % 500)) + python3 "$EXPORTER" --serve --amdgpu-sysfs "$SYS" --nvidia-cmd false \ + --bind 127.0.0.1 --bind 127.0.0.2 --port "$port" \ + > "$BATS_TEST_TMPDIR/serve.log" 2>&1 & + local pid=$! + trap 'kill "$pid" 2>/dev/null' EXIT + local ok=0 + for _ in $(seq 1 20); do + curl -fsS -m2 "http://127.0.0.1:$port/metrics" >/dev/null 2>&1 && { ok=1; break; } + sleep 0.25 + done + [ "$ok" -eq 1 ] || { kill "$pid" 2>/dev/null; return 1; } + # Both binds must ANSWER within 2 s each — the second one is the bug. + for b in 127.0.0.1 127.0.0.2; do + curl -fsS -m2 "http://$b:$port/metrics" | grep -q '^gpu_utilization_percent{' \ + || { echo "bind $b did not answer /metrics"; kill "$pid" 2>/dev/null; return 1; } + done + kill "$pid" 2>/dev/null + trap - EXIT +} diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index 53547c7..ce22bc4 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -595,16 +595,22 @@ assert any('ALERTS' in e for e in exprs), exprs } @test "Prometheus scrapes cadvisor where it is published and relabels the cgroup path to a container name" { - # Cross-file contract: cadvisor publishes the LAN port; a cadvisor scrape - # job pointing anywhere else reads nothing (unlike node-exporter's host-net - # bind, a published port IS reachable from this bridge container at the LAN - # address). And the whole dashboard depends on the id -> container relabel: - # cadvisor names nothing itself, the quadlet unit name is only in the cgroup - # path. If either half changes alone, the Containers dashboard is blank. + # Cross-file contract, measured 2026-10-21 from inside the Prometheus + # container: a bridge container cannot dial its OWN host's LAN IP — the + # hairpin refuses published ports exactly like host-net binds, so + # cadvisor-dankclaw must scrape via the podman gateway (the original pin + # here demanded the LAN address and shipped a permanently-down target). + # dankserver is a different machine: plain LAN there. And the whole + # dashboard depends on the id -> container relabel: cadvisor names nothing + # itself, the quadlet unit name is only in the cgroup path. If either half + # changes alone, the Containers dashboard is blank. local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" grep -q 'job_name: "cadvisor-dankclaw"' "$cfg" || { echo "no cadvisor-dankclaw job"; return 1; } grep -q 'job_name: "cadvisor-dankserver"' "$cfg" || { echo "no cadvisor-dankserver job"; return 1; } - grep -q '"192.168.0.143:8080"' "$cfg" || { echo "cadvisor-dankclaw does not scrape the published LAN port"; return 1; } + grep -q '"host.containers.internal:8080"' "$cfg" \ + || { echo "cadvisor-dankclaw does not scrape via the podman gateway"; return 1; } + ! grep -q '"192.168.0.143:8080"' "$cfg" \ + || { echo "cadvisor-dankclaw dials its own host LAN IP — hairpin refuses it"; return 1; } grep -q '"192.168.0.235:8080"' "$cfg" || { echo "cadvisor-dankserver does not scrape the published LAN port"; return 1; } # The relabel that turns …/app.slice/.service… into a `container`. grep -q 'app\\.slice/(\[\^\.\]+)\\.service' "$cfg" || { echo "no cgroup-path -> container relabel"; return 1; }