Frank
650b6e0281
Two shipped-2026-10-20 bugs that left gpu-dankclaw, gpu-dankserver and cadvisor-dankclaw DOWN beside a green deploy: - gpu-exporter.py built one ThreadingHTTPServer per bind but ran serve_forever() on servers[0] only: the LAN-class socket sat listen()ing but never accept()ed, loopback deploy gate passed, every scrape that reached the other bind timed out on the SYN backlog. Now one thread per extra bind. Regression test spins --serve on two loopback binds and requires BOTH to answer /metrics. - cadvisor-dankclaw scraped its own host's LAN IP. Measured from inside the Prometheus container: a bridge container cannot dial its own host's LAN address at all (hairpin refuses published ports too); the podman gateway answers. Target moved to host.containers.internal:8080; the seam that pinned the LAN address (asserting the wrong doctrine) is re-pinned to the gateway plus a negative pin against the own-host LAN dial. Observability page records the measured reachability law replacing the inferred one.
228 lines
11 KiB
Bash
Executable File
228 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the GPU metrics exporter's black-box contract.
|
|
#
|
|
# The exporter reads amdgpu sysfs (one card per /sys/class/drm/cardN that has a
|
|
# device/gpu_busy_percent file) and, on the box that has a working NVIDIA stack,
|
|
# one `nvidia-smi --query-gpu` CSV, and emits the unified `gpu_*` Prometheus
|
|
# families the host-system dashboard graphs. Both sources are injectable so the
|
|
# whole thing is testable with fixtures on a GPU-less dev box.
|
|
#
|
|
# Only the inputs a consumer sees are pinned: the metric families, their label
|
|
# set, and that absent hardware yields ABSENT series (never a fabricated 0 — a
|
|
# wrong number on a dashboard gets believed). The R9700s expose no sysfs
|
|
# power1_average, so those cards legitimately carry no gpu_power_watts line.
|
|
|
|
load helpers
|
|
|
|
EXPORTER="$REPO_ROOT/scripts/gpu-exporter.py"
|
|
|
|
# Build a fixture amdgpu sysfs tree. $BATS_TEST_TMPDIR/sys
|
|
mk_amd() { # gpu busy vram_used vram_total power_uw_or_blank temp_mc
|
|
local card=$1 busy=$2 used=$3 total=$4 power=$5 temp=$6
|
|
local d="$SYS/card$card/device"
|
|
mkdir -p "$d/hwmon/hwmon9"
|
|
printf '%s\n' "$busy" > "$d/gpu_busy_percent"
|
|
printf '%s\n' "$used" > "$d/mem_info_vram_used"
|
|
printf '%s\n' "$total" > "$d/mem_info_vram_total"
|
|
printf '%s\n' "$power" > "$d/power1_average" # blank => absent on R9700
|
|
[ -n "$temp" ] && printf '%s\n' "$temp" > "$d/hwmon/hwmon9/temp1_input"
|
|
printf 'DRIVER=amdgpu\nPCI_ID=1002:7551\nPCI_SLOT_NAME=0000:03:00.0\n' > "$d/uevent"
|
|
}
|
|
|
|
setup() {
|
|
SYS="$BATS_TEST_TMPDIR/sys"
|
|
mkdir -p "$SYS"
|
|
}
|
|
|
|
@test "exporter exists and compiles" {
|
|
[ -f "$EXPORTER" ]
|
|
python3 -m py_compile "$EXPORTER"
|
|
}
|
|
|
|
@test "amd card with full sysfs yields util/vram/temp/power with the right labels and units" {
|
|
mk_amd 1 42 1073741824 34359738368 250000000 71000 # 250 W, 71 C
|
|
run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
|
|
[ "$status" -eq 0 ] || { echo "$output"; return 1; }
|
|
# util is a raw percent, VRAM verbatim bytes, temp milli->C, power microW->W.
|
|
echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="amd"[^}]*\} 42$' || { echo "$output"; return 1; }
|
|
echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*\} 1073741824$' || { echo "$output"; return 1; }
|
|
echo "$output" | grep -Eq 'gpu_memory_total_bytes\{[^}]*\} 34359738368$' || { echo "$output"; return 1; }
|
|
echo "$output" | grep -Eq 'gpu_temperature_celsius\{[^}]*\} 71$' || { echo "$output"; return 1; }
|
|
echo "$output" | grep -Eq 'gpu_power_watts\{[^}]*\} 250$' || { echo "$output"; return 1; }
|
|
# identity labels: the PCI slot is a stable unique id, vendor marks the family.
|
|
echo "$output" | grep -Eq 'gpu_utilization_percent\{.*gpu="0000:03:00\.0"' || { echo "$output"; return 1; }
|
|
}
|
|
|
|
@test "card with an empty power file and no hwmon yields util/vram but omits power and temp" {
|
|
mk_amd 1 99 34108338176 34208743424 "" "" # the R9700 shape
|
|
run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
|
|
[ "$status" -eq 0 ] || { echo "$output"; return 1; }
|
|
echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*\} 99$' || { echo "$output"; return 1; }
|
|
echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*\} 34108338176$'
|
|
# Absent means ABSENT, not a zero the dashboard would plot as a real reading.
|
|
! echo "$output" | grep -Eq '^gpu_power_watts\{' || { echo "fabricated power line: $output"; return 1; }
|
|
! echo "$output" | grep -Eq '^gpu_temperature_celsius\{' || { echo "fabricated temp line: $output"; return 1; }
|
|
}
|
|
|
|
@test "nvidia csv yields util/vram/temp/power tagged vendor=nvidia" {
|
|
nvidia="$BATS_TEST_TMPDIR/nvidia-smi"
|
|
cat > "$nvidia" <<'STUB'
|
|
echo '0, NVIDIA GeForce RTX 3090, 93, 23004, 24576, 76, 397.40'
|
|
STUB
|
|
chmod +x "$nvidia"
|
|
run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd "$nvidia"
|
|
[ "$status" -eq 0 ] || { echo "$output"; return 1; }
|
|
echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="nvidia"[^}]*\} 93$' || { echo "$output"; return 1; }
|
|
# memory.used MiB -> bytes (23004 * 1048576)
|
|
echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*vendor="nvidia"[^}]*\} 24121442304$' || { echo "$output"; return 1; }
|
|
echo "$output" | grep -Eq 'gpu_temperature_celsius\{[^}]*vendor="nvidia"[^}]*\} 76$'
|
|
echo "$output" | grep -Eq 'gpu_power_watts\{[^}]*vendor="nvidia"[^}]*\} 397\.4$'
|
|
echo "$output" | grep -Eq 'model="NVIDIA GeForce RTX 3090"'
|
|
}
|
|
|
|
@test "a failing nvidia command is tolerated: amd still emits and exit is 0" {
|
|
mk_amd 1 50 1000000 2000000 "" 40000
|
|
failing="$BATS_TEST_TMPDIR/nvidia-fail"
|
|
printf '#!/bin/sh\nexit 1\n' > "$failing"; chmod +x "$failing"
|
|
run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd "$failing"
|
|
[ "$status" -eq 0 ] || { echo "$output"; return 1; }
|
|
echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="amd"[^}]*\} 50$'
|
|
! echo "$output" | grep -q 'vendor="nvidia"'
|
|
}
|
|
|
|
@test "no GPUs at all yields only HELP/TYPE headers and exits 0" {
|
|
run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
|
|
[ "$status" -eq 0 ] || { echo "$output"; return 1; }
|
|
# families are declared even when empty, so the dashboard/queries stay valid.
|
|
echo "$output" | grep -q '# TYPE gpu_utilization_percent gauge'
|
|
echo "$output" | grep -q '# TYPE gpu_memory_used_bytes gauge'
|
|
! echo "$output" | grep -Eq '^gpu_utilization_percent\{'
|
|
}
|
|
|
|
@test "output is valid Prometheus text" {
|
|
mk_amd 1 42 1000 2000 "" 50000
|
|
run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
|
|
[ "$status" -eq 0 ]
|
|
# every non-comment line is "<name>{labels} value" or "<name> value"
|
|
printf '%s\n' "$output" | python3 -c '
|
|
import sys, re
|
|
pat = re.compile(r"^[a-z_]+(\{[^}]*\})? [-+0-9.eE+]+$")
|
|
for i, l in enumerate(sys.stdin):
|
|
l = l.rstrip("\n")
|
|
if not l or l.startswith("#"):
|
|
continue
|
|
assert pat.match(l), "malformed line %d: %r" % (i, l)
|
|
print("prometheus text ok")
|
|
'
|
|
}
|
|
|
|
# ---- deployment + wiring contracts -------------------------------------
|
|
# The exporter is a HOST-PROCESS user unit (not a quadlet): it must read the
|
|
# host's own DRM sysfs and shell to nvidia-smi, which a container could not do
|
|
# without GPU passthrough. These tests pin the cross-file wiring a reader
|
|
# cannot eyeball — a lone change on one side leaves the metric silently absent.
|
|
|
|
@test "the base unit runs the exporter from the checkout and never binds a wildcard" {
|
|
local u="$REPO_ROOT/scripts/gpu-exporter.service"
|
|
[ -f "$u" ]
|
|
grep -q '^ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/gpu-exporter.py --serve$' "$u" \
|
|
|| { echo "unit does not run the exporter --serve from the checkout"; return 1; }
|
|
grep -q '^Restart=always$' "$u"
|
|
grep -q '^WantedBy=default.target$' "$u"
|
|
# Base unit binds loopback only; the per-host LAN address is added by a
|
|
# deploy drop-in, so a checked-in unit is never a wildcard.
|
|
grep -q '^Environment=GPU_EXPORTER_BINDS=127.0.0.1$' "$u"
|
|
! grep -qE '0\.0\.0\.0' "$u"
|
|
}
|
|
|
|
@test "the deployer installs, drops in the LAN bind, and health-gates gpu-exporter" {
|
|
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
grep -q '^deploy_gpu_exporter() {' "$handler"
|
|
# Per-host LAN from $TARGET (never a hardcoded host, never a wildcard).
|
|
grep -q 'dankclaw) lan=192.168.0.143' "$handler"
|
|
grep -q 'dankserver) lan=192.168.0.235' "$handler"
|
|
# Drop-in binds loopback + this host's LAN as one comma list.
|
|
grep -q "GPU_EXPORTER_BINDS=127.0.0.1,%s" "$handler"
|
|
grep -q 'systemctl --user enable gpu-exporter.service' "$handler"
|
|
grep -q 'systemctl --user restart gpu-exporter.service' "$handler"
|
|
# Health gate waits on the loopback metrics endpoint.
|
|
grep -q 'http://127.0.0.1:9401/metrics' "$handler"
|
|
# Dispatched from the central case.
|
|
grep -q 'gpu-exporter) deploy_gpu_exporter ;;' "$handler"
|
|
# Allowlisted on BOTH hosts.
|
|
grep -q 'dankclaw:gpu-exporter' "$handler"
|
|
grep -q 'dankserver:gpu-exporter' "$handler"
|
|
}
|
|
|
|
@test "the planner selects gpu-exporter for its files on each target" {
|
|
local t f
|
|
for t in dankclaw dankserver; do
|
|
for f in scripts/gpu-exporter.py scripts/gpu-exporter.service; do
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target "$t" --paths "$f"
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "gpu-exporter" ] || { echo "$t/$f -> '$output'"; return 1; }
|
|
done
|
|
done
|
|
}
|
|
|
|
@test "Prometheus scrapes gpu-exporter where node/cadvisor are reachable on each host" {
|
|
# Cross-file contract, same posture as node-exporter: the local instance is
|
|
# reached via the podman gateway (this Prometheus is a bridge container and
|
|
# a host-process bind is refused at the host LAN IP); the Dankserver
|
|
# instance is a different machine, so plain LAN routing reaches it. If either
|
|
# address drifts from the exporter's bind, GPU panels go blank with no error.
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q 'job_name: "gpu-dankclaw"' "$cfg" || { echo "no gpu-dankclaw job"; return 1; }
|
|
grep -q 'job_name: "gpu-dankserver"' "$cfg" || { echo "no gpu-dankserver job"; return 1; }
|
|
grep -q '"host.containers.internal:9401"' "$cfg" \
|
|
|| { echo "gpu-dankclaw does not scrape the podman gateway"; return 1; }
|
|
grep -q '"192.168.0.235:9401"' "$cfg" \
|
|
|| { echo "gpu-dankserver does not scrape the LAN address"; return 1; }
|
|
# A vhost target would be red forever (Prometheus cannot solve Anubis).
|
|
! grep -qE '"[^"]*franksdank[^"]*:9401"' "$cfg"
|
|
}
|
|
|
|
@test "the host-system dashboard graphs the gpu_* families the exporter emits" {
|
|
local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/host-system.json"
|
|
[ -f "$dash" ]
|
|
python3 -c "
|
|
import json
|
|
d = json.load(open('$dash'))
|
|
exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
|
|
for want in ('gpu_utilization_percent', 'gpu_memory_used_bytes',
|
|
'gpu_temperature_celsius', 'gpu_power_watts'):
|
|
assert any(want in e for e in exprs), ('missing', want, exprs)
|
|
print('gpu dashboard panels ok')
|
|
"
|
|
}
|
|
|
|
@test "--serve answers on EVERY declared bind, not just the first" {
|
|
# The deployed unit serves loopback + the host LAN address, and Prometheus
|
|
# reaches the host through the gateway — which dials the LAN-class bind.
|
|
# A server that only runs serve_forever() on the first socket leaves the
|
|
# second one listen()ing-but-never-accepting: the deploy health gate on
|
|
# loopback passes, every scrape on the other bind dies on the SYN backlog,
|
|
# and no gate but this one sees it. (This exact bug shipped 2026-10-21 and
|
|
# left both gpu-* targets down beside a green deploy.)
|
|
mk_amd 0 42 1073741824 34359738368 "" 71000
|
|
port=$((19400 + RANDOM % 500))
|
|
python3 "$EXPORTER" --serve --amdgpu-sysfs "$SYS" --nvidia-cmd false \
|
|
--bind 127.0.0.1 --bind 127.0.0.2 --port "$port" \
|
|
> "$BATS_TEST_TMPDIR/serve.log" 2>&1 &
|
|
local pid=$!
|
|
trap 'kill "$pid" 2>/dev/null' EXIT
|
|
local ok=0
|
|
for _ in $(seq 1 20); do
|
|
curl -fsS -m2 "http://127.0.0.1:$port/metrics" >/dev/null 2>&1 && { ok=1; break; }
|
|
sleep 0.25
|
|
done
|
|
[ "$ok" -eq 1 ] || { kill "$pid" 2>/dev/null; return 1; }
|
|
# Both binds must ANSWER within 2 s each — the second one is the bug.
|
|
for b in 127.0.0.1 127.0.0.2; do
|
|
curl -fsS -m2 "http://$b:$port/metrics" | grep -q '^gpu_utilization_percent{' \
|
|
|| { echo "bind $b did not answer /metrics"; kill "$pid" 2>/dev/null; return 1; }
|
|
done
|
|
kill "$pid" 2>/dev/null
|
|
trap - EXIT
|
|
}
|