serverAppConfigs/tests/gpu_exporter.bats

#!/usr/bin/env bats
# Seam: the GPU metrics exporter's black-box contract.
#
# The exporter reads amdgpu sysfs (one card per /sys/class/drm/cardN that has a
# device/gpu_busy_percent file) and, on the box that has a working NVIDIA stack,
# one `nvidia-smi --query-gpu` CSV, and emits the unified `gpu_*` Prometheus
# families the host-system dashboard graphs. Both sources are injectable so the
# whole thing is testable with fixtures on a GPU-less dev box.
#
# Only the inputs a consumer sees are pinned: the metric families, their label
# set, and that absent hardware yields ABSENT series (never a fabricated 0 — a
# wrong number on a dashboard gets believed). The R9700s expose no sysfs
# power1_average, so those cards legitimately carry no gpu_power_watts line.

load helpers

EXPORTER="$REPO_ROOT/scripts/gpu-exporter.py"

# Build a fixture amdgpu sysfs tree. $BATS_TEST_TMPDIR/sys
mk_amd() { # gpu busy vram_used vram_total power_uw_or_blank temp_mc
    local card=$1 busy=$2 used=$3 total=$4 power=$5 temp=$6
    local d="$SYS/card$card/device"
    mkdir -p "$d/hwmon/hwmon9"
    printf '%s\n' "$busy" > "$d/gpu_busy_percent"
    printf '%s\n' "$used" > "$d/mem_info_vram_used"
    printf '%s\n' "$total" > "$d/mem_info_vram_total"
    printf '%s\n' "$power" > "$d/power1_average"          # blank => absent on R9700
    [ -n "$temp" ] && printf '%s\n' "$temp" > "$d/hwmon/hwmon9/temp1_input"
    printf 'DRIVER=amdgpu\nPCI_ID=1002:7551\nPCI_SLOT_NAME=0000:03:00.0\n' > "$d/uevent"
}

setup() {
    SYS="$BATS_TEST_TMPDIR/sys"
    mkdir -p "$SYS"
}

@test "exporter exists and compiles" {
    [ -f "$EXPORTER" ]
    python3 -m py_compile "$EXPORTER"
}

@test "amd card with full sysfs yields util/vram/temp/power with the right labels and units" {
    mk_amd 1 42 1073741824 34359738368 250000000 71000   # 250 W, 71 C
    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
    # util is a raw percent, VRAM verbatim bytes, temp milli->C, power microW->W.
    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="amd"[^}]*\} 42$' || { echo "$output"; return 1; }
    echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*\} 1073741824$' || { echo "$output"; return 1; }
    echo "$output" | grep -Eq 'gpu_memory_total_bytes\{[^}]*\} 34359738368$' || { echo "$output"; return 1; }
    echo "$output" | grep -Eq 'gpu_temperature_celsius\{[^}]*\} 71$' || { echo "$output"; return 1; }
    echo "$output" | grep -Eq 'gpu_power_watts\{[^}]*\} 250$' || { echo "$output"; return 1; }
    # identity labels: the PCI slot is a stable unique id, vendor marks the family.
    echo "$output" | grep -Eq 'gpu_utilization_percent\{.*gpu="0000:03:00\.0"' || { echo "$output"; return 1; }
}

@test "card with an empty power file and no hwmon yields util/vram but omits power and temp" {
    mk_amd 1 99 34108338176 34208743424 "" ""           # the R9700 shape
    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*\} 99$' || { echo "$output"; return 1; }
    echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*\} 34108338176$'
    # Absent means ABSENT, not a zero the dashboard would plot as a real reading.
    ! echo "$output" | grep -Eq '^gpu_power_watts\{' || { echo "fabricated power line: $output"; return 1; }
    ! echo "$output" | grep -Eq '^gpu_temperature_celsius\{' || { echo "fabricated temp line: $output"; return 1; }
}

@test "nvidia csv yields util/vram/temp/power tagged vendor=nvidia" {
    nvidia="$BATS_TEST_TMPDIR/nvidia-smi"
    cat > "$nvidia" <<'STUB'
echo '0, NVIDIA GeForce RTX 3090, 93, 23004, 24576, 76, 397.40'
STUB
    chmod +x "$nvidia"
    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd "$nvidia"
    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="nvidia"[^}]*\} 93$' || { echo "$output"; return 1; }
    # memory.used MiB -> bytes (23004 * 1048576)
    echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*vendor="nvidia"[^}]*\} 24121442304$' || { echo "$output"; return 1; }
    echo "$output" | grep -Eq 'gpu_temperature_celsius\{[^}]*vendor="nvidia"[^}]*\} 76$'
    echo "$output" | grep -Eq 'gpu_power_watts\{[^}]*vendor="nvidia"[^}]*\} 397\.4$'
    echo "$output" | grep -Eq 'model="NVIDIA GeForce RTX 3090"'
}

@test "a failing nvidia command is tolerated: amd still emits and exit is 0" {
    mk_amd 1 50 1000000 2000000 "" 40000
    failing="$BATS_TEST_TMPDIR/nvidia-fail"
    printf '#!/bin/sh\nexit 1\n' > "$failing"; chmod +x "$failing"
    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd "$failing"
    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="amd"[^}]*\} 50$'
    ! echo "$output" | grep -q 'vendor="nvidia"'
}

@test "no GPUs at all yields only HELP/TYPE headers and exits 0" {
    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
    # families are declared even when empty, so the dashboard/queries stay valid.
    echo "$output" | grep -q '# TYPE gpu_utilization_percent gauge'
    echo "$output" | grep -q '# TYPE gpu_memory_used_bytes gauge'
    ! echo "$output" | grep -Eq '^gpu_utilization_percent\{'
}

@test "output is valid Prometheus text" {
    mk_amd 1 42 1000 2000 "" 50000
    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
    [ "$status" -eq 0 ]
    # every non-comment line is "<name>{labels} value" or "<name> value"
    printf '%s\n' "$output" | python3 -c '
import sys, re
pat = re.compile(r"^[a-z_]+(\{[^}]*\})? [-+0-9.eE+]+$")
for i, l in enumerate(sys.stdin):
    l = l.rstrip("\n")
    if not l or l.startswith("#"):
        continue
    assert pat.match(l), "malformed line %d: %r" % (i, l)
print("prometheus text ok")
'
}

# ---- deployment + wiring contracts -------------------------------------
# The exporter is a HOST-PROCESS user unit (not a quadlet): it must read the
# host's own DRM sysfs and shell to nvidia-smi, which a container could not do
# without GPU passthrough. These tests pin the cross-file wiring a reader
# cannot eyeball — a lone change on one side leaves the metric silently absent.

@test "the base unit runs the exporter from the checkout and never binds a wildcard" {
    local u="$REPO_ROOT/scripts/gpu-exporter.service"
    [ -f "$u" ]
    grep -q '^ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/gpu-exporter.py --serve$' "$u" \
        || { echo "unit does not run the exporter --serve from the checkout"; return 1; }
    grep -q '^Restart=always$' "$u"
    grep -q '^WantedBy=default.target$' "$u"
    # Base unit binds loopback only; the per-host LAN address is added by a
    # deploy drop-in, so a checked-in unit is never a wildcard.
    grep -q '^Environment=GPU_EXPORTER_BINDS=127.0.0.1$' "$u"
    ! grep -qE '0\.0\.0\.0' "$u"
}

@test "the deployer installs, drops in the LAN bind, and health-gates gpu-exporter" {
    local handler="$REPO_ROOT/scripts/deploy-remote.sh"
    grep -q '^deploy_gpu_exporter() {' "$handler"
    # Per-host LAN from $TARGET (never a hardcoded host, never a wildcard).
    grep -q 'dankclaw) lan=192.168.0.143' "$handler"
    grep -q 'dankserver) lan=192.168.0.235' "$handler"
    # Drop-in binds loopback + this host's LAN as one comma list.
    grep -q "GPU_EXPORTER_BINDS=127.0.0.1,%s" "$handler"
    grep -q 'systemctl --user enable gpu-exporter.service' "$handler"
    grep -q 'systemctl --user restart gpu-exporter.service' "$handler"
    # Health gate waits on the loopback metrics endpoint.
    grep -q 'http://127.0.0.1:9401/metrics' "$handler"
    # Dispatched from the central case.
    grep -q 'gpu-exporter) deploy_gpu_exporter ;;' "$handler"
    # Allowlisted on BOTH hosts.
    grep -q 'dankclaw:gpu-exporter' "$handler"
    grep -q 'dankserver:gpu-exporter' "$handler"
}

@test "the planner selects gpu-exporter for its files on each target" {
    local t f
    for t in dankclaw dankserver; do
        for f in scripts/gpu-exporter.py scripts/gpu-exporter.service; do
            run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target "$t" --paths "$f"
            [ "$status" -eq 0 ]
            [ "$output" = "gpu-exporter" ] || { echo "$t/$f -> '$output'"; return 1; }
        done
    done
}

@test "Prometheus scrapes gpu-exporter where node/cadvisor are reachable on each host" {
    # Cross-file contract, same posture as node-exporter: the local instance is
    # reached via the podman gateway (this Prometheus is a bridge container and
    # a host-process bind is refused at the host LAN IP); the Dankserver
    # instance is a different machine, so plain LAN routing reaches it. If either
    # address drifts from the exporter's bind, GPU panels go blank with no error.
    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
    grep -q 'job_name: "gpu-dankclaw"' "$cfg" || { echo "no gpu-dankclaw job"; return 1; }
    grep -q 'job_name: "gpu-dankserver"' "$cfg" || { echo "no gpu-dankserver job"; return 1; }
    grep -q '"host.containers.internal:9401"' "$cfg" \
        || { echo "gpu-dankclaw does not scrape the podman gateway"; return 1; }
    grep -q '"192.168.0.235:9401"' "$cfg" \
        || { echo "gpu-dankserver does not scrape the LAN address"; return 1; }
    # A vhost target would be red forever (Prometheus cannot solve Anubis).
    ! grep -qE '"[^"]*franksdank[^"]*:9401"' "$cfg"
}

@test "the host-system dashboard graphs the gpu_* families the exporter emits" {
    local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/host-system.json"
    [ -f "$dash" ]
    python3 -c "
import json
d = json.load(open('$dash'))
exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
for want in ('gpu_utilization_percent', 'gpu_memory_used_bytes',
             'gpu_temperature_celsius', 'gpu_power_watts'):
    assert any(want in e for e in exprs), ('missing', want, exprs)
print('gpu dashboard panels ok')
"
}