fix(observability): serve every exporter bind; scrape same-host cadvisor via gateway

Two shipped-2026-10-20 bugs that left gpu-dankclaw, gpu-dankserver and
cadvisor-dankclaw DOWN beside a green deploy:

- gpu-exporter.py built one ThreadingHTTPServer per bind but ran
  serve_forever() on servers[0] only: the LAN-class socket sat listen()ing
  but never accept()ed, loopback deploy gate passed, every scrape that
  reached the other bind timed out on the SYN backlog. Now one thread per
  extra bind. Regression test spins --serve on two loopback binds and
  requires BOTH to answer /metrics.

- cadvisor-dankclaw scraped its own host's LAN IP. Measured from inside the
  Prometheus container: a bridge container cannot dial its own host's LAN
  address at all (hairpin refuses published ports too); the podman gateway
  answers. Target moved to host.containers.internal:8080; the seam that
  pinned the LAN address (asserting the wrong doctrine) is re-pinned to the
  gateway plus a negative pin against the own-host LAN dial. Observability
  page records the measured reachability law replacing the inferred one.
This commit is contained in:
Frank committed 2026-10-21 00:34:17 -07:00
1 parent b62a05837f
commit 650b6e0281
5 files changed
+68 -15

No files matched your search

+30
View File
@@ -195,3 +195,33 @@ for want in ('gpu_utilization_percent', 'gpu_memory_used_bytes',
print('gpu dashboard panels ok')
"
}
@test "--serve answers on EVERY declared bind, not just the first" {
# The deployed unit serves loopback + the host LAN address, and Prometheus
# reaches the host through the gateway — which dials the LAN-class bind.
# A server that only runs serve_forever() on the first socket leaves the
# second one listen()ing-but-never-accepting: the deploy health gate on
# loopback passes, every scrape on the other bind dies on the SYN backlog,
# and no gate but this one sees it. (This exact bug shipped 2026-10-21 and
# left both gpu-* targets down beside a green deploy.)
mk_amd 0 42 1073741824 34359738368 "" 71000
port=$((19400 + RANDOM % 500))
python3 "$EXPORTER" --serve --amdgpu-sysfs "$SYS" --nvidia-cmd false \
--bind 127.0.0.1 --bind 127.0.0.2 --port "$port" \
> "$BATS_TEST_TMPDIR/serve.log" 2>&1 &
local pid=$!
trap 'kill "$pid" 2>/dev/null' EXIT
local ok=0
for _ in $(seq 1 20); do
curl -fsS -m2 "http://127.0.0.1:$port/metrics" >/dev/null 2>&1 && { ok=1; break; }
sleep 0.25
done
[ "$ok" -eq 1 ] || { kill "$pid" 2>/dev/null; return 1; }
# Both binds must ANSWER within 2 s each — the second one is the bug.
for b in 127.0.0.1 127.0.0.2; do
curl -fsS -m2 "http://$b:$port/metrics" | grep -q '^gpu_utilization_percent{' \
|| { echo "bind $b did not answer /metrics"; kill "$pid" 2>/dev/null; return 1; }
done
kill "$pid" 2>/dev/null
trap - EXIT
}
+13 -7
View File
@@ -595,16 +595,22 @@ assert any('ALERTS' in e for e in exprs), exprs
}
@test "Prometheus scrapes cadvisor where it is published and relabels the cgroup path to a container name" {
# Cross-file contract: cadvisor publishes the LAN port; a cadvisor scrape
# job pointing anywhere else reads nothing (unlike node-exporter's host-net
# bind, a published port IS reachable from this bridge container at the LAN
# address). And the whole dashboard depends on the id -> container relabel:
# cadvisor names nothing itself, the quadlet unit name is only in the cgroup
# path. If either half changes alone, the Containers dashboard is blank.
# Cross-file contract, measured 2026-10-21 from inside the Prometheus
# container: a bridge container cannot dial its OWN host's LAN IP — the
# hairpin refuses published ports exactly like host-net binds, so
# cadvisor-dankclaw must scrape via the podman gateway (the original pin
# here demanded the LAN address and shipped a permanently-down target).
# dankserver is a different machine: plain LAN there. And the whole
# dashboard depends on the id -> container relabel: cadvisor names nothing
# itself, the quadlet unit name is only in the cgroup path. If either half
# changes alone, the Containers dashboard is blank.
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
grep -q 'job_name: "cadvisor-dankclaw"' "$cfg" || { echo "no cadvisor-dankclaw job"; return 1; }
grep -q 'job_name: "cadvisor-dankserver"' "$cfg" || { echo "no cadvisor-dankserver job"; return 1; }
grep -q '"192.168.0.143:8080"' "$cfg" || { echo "cadvisor-dankclaw does not scrape the published LAN port"; return 1; }
grep -q '"host.containers.internal:8080"' "$cfg" \
|| { echo "cadvisor-dankclaw does not scrape via the podman gateway"; return 1; }
! grep -q '"192.168.0.143:8080"' "$cfg" \
|| { echo "cadvisor-dankclaw dials its own host LAN IP — hairpin refuses it"; return 1; }
grep -q '"192.168.0.235:8080"' "$cfg" || { echo "cadvisor-dankserver does not scrape the published LAN port"; return 1; }
# The relabel that turns …/app.slice/<name>.service… into a `container`.
grep -q 'app\\.slice/(\[\^\.\]+)\\.service' "$cfg" || { echo "no cgroup-path -> container relabel"; return 1; }