Merge pull request 'fix(observability): serve every exporter bind; scrape same-host cadvisor via gateway' (#64) from fix/gpu-serve-and-cadvisor-target into main

scripts/gpu-exporter.py

@@ -32,6 +32,7 @@ import glob
 import os
 import subprocess
 import sys
+import threading
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 FAMILIES = [
@@ -198,6 +199,11 @@ def main(argv):
             return 2
         servers.append(ThreadingHTTPServer((b, args.port), Handler))
     sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds)))
+    # EVERY socket must be served: a listen()ed-but-never-accept()ed bind looks
+    # healthy in `ss` yet times out every scrape once its SYN backlog fills —
+    # and the loopback deploy gate cannot see it. One thread per extra bind.
+    for s in servers[1:]:
+        threading.Thread(target=s.serve_forever, daemon=True).start()
     servers[0].serve_forever()
     return 0
 

serverAppConfigs-docs/Infrastructure/Observability.md

@@ -67,10 +67,18 @@ Alertmanager has no vhost at all. node-exporter is the exception that must live
 in the host network namespace, and it binds exactly two addresses rather than
 the wildcard — see [[Node-Exporter]].
 
-The relay's bind is the one address a bridge container can reach on the host:
-`192.168.0.143:18093`, with iptables INPUT admitting loopback, the host itself
-and the podman bridge subnet `10.89.0.0/16` before the DROP
-(`scripts/deploy-remote.sh:255-262`).
+**Same-host reachability from a bridge container is measured, not inferred**
+(2026-10-21, from inside the Prometheus container): `host.containers.internal`
+(the podman gateway) reaches host services whether they bind host-net or are
+published ports — node `:9100`, cadvisor `:8080`, gpu `:9401` all answer there
+— while the host's own LAN IP is refused (`192.168.0.143:8080`, `:9401`), so
+every same-host Prometheus target uses the gateway and only cross-host targets
+(dankserver) use plain LAN. Whether LAN works at all is network-specific: the
+relay at `192.168.0.143:18093` IS reached from alertmanager's network, with
+iptables INPUT admitting loopback, the host itself and the podman bridge subnet
+`10.89.0.0/16` before the DROP (`scripts/deploy-remote.sh:255-262`). Never
+point a scrape target at a host LAN IP from its own host on the strength of
+one working example; dial the gateway and probe.
 
 ## Deployment
 

shared/prometheus/prometheus.yml

@@ -140,12 +140,15 @@ scrape_configs:
   # system.slice daemons and machine.slice scopes carry no such name and are
   # dropped, as is the per-CPU-core split (only cpu="total" survives), so the
   # scrape stays small. cadvisor cannot bind per-interface, so it is reached at
-  # the host's PUBLISHED LAN port: unlike node-exporter's host-net bind (which
-  # is why node-dankclaw scrapes via host.containers.internal), a published port
-  # IS reachable from this bridge container at the host LAN address.
+  # the podman gateway like every same-host target: a BRIDGE CONTAINER CANNOT
+  # DIAL ITS OWN HOST'S LAN IP AT ALL — the netavark hairpin gap refuses the
+  # connection whether the port is a host-net bind or a published port (the
+  # original note here claimed published ports were exempt; measured 2026-10-21
+  # from inside this container: LAN refused, gateway answers). Cross-host
+  # targets (dankserver) have no hairpin problem and stay plain LAN.
   - job_name: "cadvisor-dankclaw"
     static_configs:
-      - targets: ["192.168.0.143:8080"]
+      - targets: ["host.containers.internal:8080"]
         labels:
           host: dankclaw
     metric_relabel_configs:

tests/gpu_exporter.bats

@@ -195,3 +195,33 @@ for want in ('gpu_utilization_percent', 'gpu_memory_used_bytes',
 print('gpu dashboard panels ok')
 "
 }
+
+@test "--serve answers on EVERY declared bind, not just the first" {
+    # The deployed unit serves loopback + the host LAN address, and Prometheus
+    # reaches the host through the gateway — which dials the LAN-class bind.
+    # A server that only runs serve_forever() on the first socket leaves the
+    # second one listen()ing-but-never-accepting: the deploy health gate on
+    # loopback passes, every scrape on the other bind dies on the SYN backlog,
+    # and no gate but this one sees it. (This exact bug shipped 2026-10-21 and
+    # left both gpu-* targets down beside a green deploy.)
+    mk_amd 0 42 1073741824 34359738368 "" 71000
+    port=$((19400 + RANDOM % 500))
+    python3 "$EXPORTER" --serve --amdgpu-sysfs "$SYS" --nvidia-cmd false \
+        --bind 127.0.0.1 --bind 127.0.0.2 --port "$port" \
+        > "$BATS_TEST_TMPDIR/serve.log" 2>&1 &
+    local pid=$!
+    trap 'kill "$pid" 2>/dev/null' EXIT
+    local ok=0
+    for _ in $(seq 1 20); do
+        curl -fsS -m2 "http://127.0.0.1:$port/metrics" >/dev/null 2>&1 && { ok=1; break; }
+        sleep 0.25
+    done
+    [ "$ok" -eq 1 ] || { kill "$pid" 2>/dev/null; return 1; }
+    # Both binds must ANSWER within 2 s each — the second one is the bug.
+    for b in 127.0.0.1 127.0.0.2; do
+        curl -fsS -m2 "http://$b:$port/metrics" | grep -q '^gpu_utilization_percent{' \
+            || { echo "bind $b did not answer /metrics"; kill "$pid" 2>/dev/null; return 1; }
+    done
+    kill "$pid" 2>/dev/null
+    trap - EXIT
+}

tests/observability_quadlet.bats

@@ -595,16 +595,22 @@ assert any('ALERTS' in e for e in exprs), exprs
 }
 
 @test "Prometheus scrapes cadvisor where it is published and relabels the cgroup path to a container name" {
-    # Cross-file contract: cadvisor publishes the LAN port; a cadvisor scrape
-    # job pointing anywhere else reads nothing (unlike node-exporter's host-net
-    # bind, a published port IS reachable from this bridge container at the LAN
-    # address). And the whole dashboard depends on the id -> container relabel:
-    # cadvisor names nothing itself, the quadlet unit name is only in the cgroup
-    # path. If either half changes alone, the Containers dashboard is blank.
+    # Cross-file contract, measured 2026-10-21 from inside the Prometheus
+    # container: a bridge container cannot dial its OWN host's LAN IP — the
+    # hairpin refuses published ports exactly like host-net binds, so
+    # cadvisor-dankclaw must scrape via the podman gateway (the original pin
+    # here demanded the LAN address and shipped a permanently-down target).
+    # dankserver is a different machine: plain LAN there. And the whole
+    # dashboard depends on the id -> container relabel: cadvisor names nothing
+    # itself, the quadlet unit name is only in the cgroup path. If either half
+    # changes alone, the Containers dashboard is blank.
     local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
     grep -q 'job_name: "cadvisor-dankclaw"' "$cfg" || { echo "no cadvisor-dankclaw job"; return 1; }
     grep -q 'job_name: "cadvisor-dankserver"' "$cfg" || { echo "no cadvisor-dankserver job"; return 1; }
-    grep -q '"192.168.0.143:8080"' "$cfg" || { echo "cadvisor-dankclaw does not scrape the published LAN port"; return 1; }
+    grep -q '"host.containers.internal:8080"' "$cfg" \
+        || { echo "cadvisor-dankclaw does not scrape via the podman gateway"; return 1; }
+    ! grep -q '"192.168.0.143:8080"' "$cfg" \
+        || { echo "cadvisor-dankclaw dials its own host LAN IP — hairpin refuses it"; return 1; }
     grep -q '"192.168.0.235:8080"' "$cfg" || { echo "cadvisor-dankserver does not scrape the published LAN port"; return 1; }
     # The relabel that turns …/app.slice/<name>.service… into a `container`.
     grep -q 'app\\.slice/(\[\^\.\]+)\\.service' "$cfg" || { echo "no cgroup-path -> container relabel"; return 1; }