diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index 95aca5c..a03231c 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -40,6 +40,7 @@ SERVICE_ORDER = ( "alloy", "node-exporter", "cadvisor", + "gpu-exporter", "alertmanager", "authentik-postgres", "authentik-server", @@ -94,6 +95,7 @@ DANKCLAW_SERVICES = { "alloy", "node-exporter", "cadvisor", + "gpu-exporter", "alertmanager", "authentik-postgres", "authentik-server", @@ -127,6 +129,7 @@ DANKSERVER_SERVICES = { "frank-archon", "node-exporter", "cadvisor", + "gpu-exporter", } # FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on @@ -452,6 +455,12 @@ def services_for_path(path: str, target: str) -> set[str]: # change to it redeploying alertmanager is the only path that # reaches it on the host. _add(services, "alertmanager") + if name in {"gpu-exporter.py", "gpu-exporter.service"}: + # The host-process GPU exporter is installed directly by the + # gpu-exporter deploy on BOTH hosts, so its script and unit select + # that service (target_services then keeps it only on the host being + # planned). Editing the collector restarts the running exporter. + _add(services, "gpu-exporter") if name in {"frank-zealot.env", "frank-zealot.service", "serve-frank-zealot.sh"}: # llama.cpp frank-zealot (2026-09-16) keeps its unit, env and serve # files here; each one selects frank-zealot only. The ik_llama.cpp diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 57a3ca7..e56a28f 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -86,7 +86,7 @@ service_is_allowed() { dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\ dankclaw:comfyui|dankclaw:hermes|\ dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\ - dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:alertmanager|\ + dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:alertmanager|\ dankclaw:authentik-postgres|dankclaw:authentik-server|\ dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\ dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\ @@ -97,7 +97,7 @@ service_is_allowed() { dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\ dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\ dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\ - dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|\ + dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|dankserver:gpu-exporter|\ frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\ frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\ frankbook-pro:readarr) @@ -275,6 +275,36 @@ install_signal_relay() { return 1 } +# Host-process GPU metrics exporter (amdgpu sysfs + nvidia-smi). Not a quadlet: +# it runs python3 from the checkout on the host so it can read the DRM/cgroup +# sysfs and shell out to nvidia-smi. Same two-address bind posture as +# node-exporter/cadvisor (loopback + this host's LAN), so it is never a wildcard. +deploy_gpu_exporter() { + local unit_dir="$HOME/.config/systemd/user" lan dropin i + case "$TARGET" in + dankclaw) lan=192.168.0.143 ;; + dankserver) lan=192.168.0.235 ;; + *) echo "gpu-exporter: unknown target '$TARGET'" >&2; return 2 ;; + esac + mkdir -p "$unit_dir" + cp scripts/gpu-exporter.service "$unit_dir/gpu-exporter.service" + dropin="$unit_dir/gpu-exporter.service.d" + mkdir -p "$dropin" + printf '[Service]\nEnvironment=GPU_EXPORTER_BINDS=127.0.0.1,%s\n' "$lan" \ + > "$dropin/bind.conf" + systemctl --user daemon-reload + systemctl --user enable gpu-exporter.service + systemctl --user restart gpu-exporter.service + # The interpreter needs a moment to bind after restart (the same race that + # once failed a healthy signal-relay deploy). + for i in $(seq 1 15); do + curl -fsS -m 5 http://127.0.0.1:9401/metrics >/dev/null && return 0 + sleep 1 + done + echo "gpu-exporter is not answering on 127.0.0.1:9401/metrics" >&2 + return 1 +} + copy_loki_config() { run_privileged mkdir -p /opt/loki/config run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml @@ -2004,6 +2034,7 @@ deploy_service() { tautulli) deploy_tautulli ;; alloy) deploy_alloy ;; prometheus|grafana|loki|node-exporter|cadvisor|alertmanager) deploy_observability "$service" ;; + gpu-exporter) deploy_gpu_exporter ;; comfyui) deploy_comfyui ;; *) deploy_compose_service "$service" ;; esac diff --git a/scripts/gpu-exporter.py b/scripts/gpu-exporter.py new file mode 100755 index 0000000..b315319 --- /dev/null +++ b/scripts/gpu-exporter.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""gpu-exporter: unified per-GPU metrics for a mixed AMD/NVIDIA host. + +Emits the `gpu_*` Prometheus families the host-system dashboard graphs: + + gpu_utilization_percent gpu_memory_used_bytes gpu_memory_total_bytes + gpu_temperature_celsius gpu_power_watts + +each labelled {gpu, vendor, model}. The scrape job adds `host`. + +Two sources, both host-native — no container, no GPU passthrough: + + * AMD (R9700s, the 8060S iGPU, the Raphael display GPU): the amdgpu sysfs tree. + Utilization comes ONLY from gpu_busy_percent — amd-smi reports gfx_busy_inst + as N/A on gfx1201 (verified 2026-10-20). VRAM is already bytes, temperature + is milli-degrees, power is micro-watts. Crucially power1_average is EMPTY on + the R9700s, so those cards legitimately have no gpu_power_watts line; a + missing file becomes an ABSENT metric, never a fabricated 0 that a dashboard + would plot as a real reading. + + * NVIDIA (the RTX 3090 on Dankclaw): one `nvidia-smi --query-gpu` CSV. On + Dankserver there is an nvidia-smi binary but no NVIDIA GPU, so the command + fails; that is tolerated and simply yields no NVIDIA series. + +Default mode prints one scrape to stdout and exits (the test harness, and any +`service ... --oneshot` one-off). `--serve` binds the HTTP endpoint for +Prometheus; it binds EXPLICITLY (loopback + the host LAN address), never a +wildcard, matching node-exporter's posture on these boxes. +""" +import argparse +import glob +import os +import subprocess +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +FAMILIES = [ + ("gpu_utilization_percent", "gauge", "GPU busy time as a percentage of wall time."), + ("gpu_memory_used_bytes", "gauge", "Device memory in use, in bytes."), + ("gpu_memory_total_bytes", "gauge", "Device memory capacity, in bytes."), + ("gpu_temperature_celsius", "gauge", "Package temperature in degrees Celsius."), + ("gpu_power_watts", "gauge", "Board power draw in watts."), +] + +DEFAULT_NVIDIA_CMD = ( + "nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used," + "memory.total,temperature.gpu,power.draw --format=csv,noheader,nounits" +) + + +def _read(path): + try: + with open(path) as fh: + return fh.read().strip() + except OSError: + return "" + + +def _pci_slot(device_dir): + uevent = _read(os.path.join(device_dir, "uevent")) + for line in uevent.splitlines(): + if line.startswith("PCI_SLOT_NAME="): + return line.split("=", 1)[1] + return "" + + +def _pci_id(device_dir): + uevent = _read(os.path.join(device_dir, "uevent")) + for line in uevent.splitlines(): + if line.startswith("PCI_ID="): + return "amd:" + line.split("=", 1)[1].lower() + return "amd" + + +def collect_amd(sys_dir): + for card in sorted(glob.glob(os.path.join(sys_dir, "card[0-9]*"))): + device = os.path.join(card, "device") + busy = _read(os.path.join(device, "gpu_busy_percent")) + if busy == "": + continue # not an amdgpu compute node (e.g. a render node w/o metrics) + labels = {"gpu": _pci_slot(device) or os.path.basename(card), + "vendor": "amd", "model": _pci_id(device)} + yield "gpu_utilization_percent", labels, busy + for name, leaf in (("gpu_memory_used_bytes", "mem_info_vram_used"), + ("gpu_memory_total_bytes", "mem_info_vram_total")): + v = _read(os.path.join(device, leaf)) + if v != "": + yield name, labels, v + temp = "" + for hwmon in sorted(glob.glob(os.path.join(device, "hwmon", "hwmon*", "temp1_input"))): + temp = _read(hwmon) + if temp != "": + break + if temp != "": + yield "gpu_temperature_celsius", labels, str(int(temp) / 1000.0) + power = _read(os.path.join(device, "power1_average")) + if power != "": # empty on the R9700s -> metric absent, not zero + yield "gpu_power_watts", labels, str(int(power) / 1e6) + + +def collect_nvidia(cmd): + if not cmd: + return + try: + out = subprocess.run(cmd, shell=True, capture_output=True, text=True, + timeout=10, check=True).stdout + except (subprocess.SubprocessError, OSError): + return # no NVIDIA GPU / no driver: tolerate, emit nothing + for line in out.splitlines(): + parts = [p.strip() for p in line.split(",")] + if len(parts) < 7: + continue + idx, name, util, mused, mtotal, temp, power = parts[:7] + labels = {"gpu": idx, "vendor": "nvidia", "model": name} + yield "gpu_utilization_percent", labels, util + yield "gpu_memory_used_bytes", labels, str(int(float(mused)) * 1048576) + yield "gpu_memory_total_bytes", labels, str(int(float(mtotal)) * 1048576) + if temp not in ("", "[N/A]"): + yield "gpu_temperature_celsius", labels, temp + if power not in ("", "[N/A]"): + yield "gpu_power_watts", labels, power + + +def collect(sys_dir, nvidia_cmd): + samples = {} + for source in (collect_amd(sys_dir), collect_nvidia(nvidia_cmd)): + for name, labels, value in source: + try: + value = str(float(value)) # normalise; drop trailing junk + except ValueError: + continue + samples.setdefault(name, []).append((labels, value)) + return samples + + +def _fmt(value): + f = float(value) + if f == int(f) and abs(f) < 1e15: + return str(int(f)) + return repr(f) + + +def render(samples): + out = [] + for name, kind, help_text in FAMILIES: + out.append("# HELP %s %s" % (name, help_text)) + out.append("# TYPE %s %s" % (name, kind)) + for labels, value in sorted(samples.get(name, []), key=lambda x: sorted(x[0].items())): + lset = ",".join('%s="%s"' % (k, v) for k, v in sorted(labels.items())) + out.append("%s{%s} %s" % (name, lset, _fmt(value))) + return "\n".join(out) + "\n" + + +class Handler(BaseHTTPRequestHandler): + sys_dir = "/sys/class/drm" + nvidia_cmd = DEFAULT_NVIDIA_CMD + + def do_GET(self): # noqa: N802 (http.server API) + if self.path.split("?")[0] not in ("/metrics", "/"): + self.send_error(404) + return + body = render(collect(self.sys_dir, self.nvidia_cmd)).encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--amdgpu-sysfs", default="/sys/class/drm") + ap.add_argument("--nvidia-cmd", default=DEFAULT_NVIDIA_CMD) + ap.add_argument("--serve", action="store_true", help="serve /metrics instead of one-shot") + ap.add_argument("--port", type=int, default=9401) + ap.add_argument("--bind", action="append", default=None, + help="explicit bind address (repeatable); env GPU_EXPORTER_BINDS " + "comma-list if unset. NEVER a wildcard.") + args = ap.parse_args(argv) + + if not args.serve: + sys.stdout.write(render(collect(args.amdgpu_sysfs, args.nvidia_cmd))) + return 0 + + binds = args.bind + if not binds: + env = os.environ.get("GPU_EXPORTER_BINDS", "") + binds = [b.strip() for b in env.split(",") if b.strip()] or ["127.0.0.1"] + Handler.sys_dir = args.amdgpu_sysfs + Handler.nvidia_cmd = args.nvidia_cmd + servers = [] + for b in binds: + if b in ("0.0.0.0", "::", "*"): + sys.stderr.write("refusing wildcard bind %s\n" % b) + return 2 + servers.append(ThreadingHTTPServer((b, args.port), Handler)) + sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds))) + servers[0].serve_forever() + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/gpu-exporter.service b/scripts/gpu-exporter.service new file mode 100644 index 0000000..27e71b1 --- /dev/null +++ b/scripts/gpu-exporter.service @@ -0,0 +1,35 @@ +# Systemd USER unit: gpu-exporter (both Dankclaw and Dankserver). +# +# Install path: ~/.config/systemd/user/gpu-exporter.service +# Installed and enabled by deploy_gpu_exporter in scripts/deploy-remote.sh. +# +# Per-GPU metrics (utilization / VRAM / temperature / power) for the mixed +# AMD + NVIDIA fleet, scraped by Prometheus. Like signal-relay it is a plain +# user unit, not a quadlet: it runs python3 straight from the checkout (a pull +# updates it with no copy step), a hand-written user unit can be enabled where +# the podman v1 generator's units cannot, and with Linger=yes for dank it starts +# at boot. It reads the host's own cgroup/drm sysfs and shells to nvidia-smi, so +# it must run on the host — a container would need GPU/DRM passthrough for +# nothing gained. +# +# BINDING. The base unit serves loopback only. deploy_gpu_exporter writes a +# drop-in adding THIS host's LAN address (the same two-address posture +# node-exporter and cadvisor use), so the endpoint is never a wildcard. It is +# unauthenticated metrics on the LAN, identical to node-exporter :9100 and +# cadvisor :8080 — Prometheus reaches same-host via host.containers.internal and +# the Dankserver instance over the LAN from Dankclaw. + +[Unit] +Description=GPU metrics exporter (amdgpu sysfs + nvidia-smi) +After=network-online.target +Wants=network-online.target + +[Service] +# Loopback by default; the deploy drop-in appends the host LAN address. +Environment=GPU_EXPORTER_BINDS=127.0.0.1 +ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/gpu-exporter.py --serve +Restart=always +RestartSec=5 + +[Install] +WantedBy=default.target diff --git a/scripts/test-changed.sh b/scripts/test-changed.sh index 3c831ae..bbad299 100755 --- a/scripts/test-changed.sh +++ b/scripts/test-changed.sh @@ -293,6 +293,14 @@ for path in "${CHANGED[@]}"; do add_test tests/caddyfile.bats add_test tests/deployment_scoping.bats ;; + # The GPU exporter is a host process; its seam pins the collector, the + # user unit, the deployer function, the planner route, the Prometheus + # scrape jobs and the host-system GPU panels — a cross-file contract, so + # the whole seam rides on any of its own files. + scripts/gpu-exporter.py|scripts/gpu-exporter.service) + add_test tests/gpu_exporter.bats + add_test tests/deployment_scoping.bats + ;; dankclaw/udev/*) # Host-level udev rules are applied by manual sudo install + udevadm # trigger, not dispatched as a service; nothing here is bats-testable. diff --git a/serverAppConfigs-docs/Infrastructure/Observability.md b/serverAppConfigs-docs/Infrastructure/Observability.md index d95e343..30dd860 100644 --- a/serverAppConfigs-docs/Infrastructure/Observability.md +++ b/serverAppConfigs-docs/Infrastructure/Observability.md @@ -2,7 +2,7 @@ type: reference title: "Observability on Dankclaw" description: "The observability stack: what runs, how it is deployed, how it is exposed, and what is still unconfigured." -tags: [observability, prometheus, grafana, loki, alloy, node-exporter, quadlet, dankclaw] +tags: [observability, prometheus, grafana, loki, alloy, node-exporter, gpu-exporter, quadlet, dankclaw] timestamp: 2026-10-20 --- @@ -10,10 +10,10 @@ timestamp: 2026-10-20 ## Summary -Seven containers and one host-side process: Prometheus (metrics), Grafana +Seven containers and two host-side processes: Prometheus (metrics), Grafana (dashboards), Loki (log store), Alloy (log shipper), node-exporter (host -metrics), cadvisor (per-container metrics) and Alertmanager (alert delivery), with `signal-relay` on the host -carrying every firing group to Signal. All of it is Quadlet-managed, +metrics), cadvisor (per-container metrics) and Alertmanager (alert delivery), +plus two host `python3` processes — `gpu-exporter` (per-GPU metrics) and `signal-relay` (Signal delivery). The containers are all Quadlet-managed, boot-persistent and loopback-only, deployed by `scripts/deploy-remote.sh` → `deploy_observability`. The metrics half works end to end; the log half is configured, healthy, and carries no data. @@ -35,6 +35,7 @@ that proves it today. | Alloy | `localhost/franksdank/alloy:latest` | `127.0.0.1:12345` | `alloy.container` | [[Alloy]] | | node-exporter | `docker.io/prom/node-exporter:latest` | host netns: `127.0.0.1:9100` + `192.168.0.143:9100` | `node-exporter.container` | [[Node-Exporter]] | | cadvisor | `gcr.io/cadvisor/cadvisor:v0.49.1` | bridge publish `127.0.0.1:8080` + `192.168.0.143:8080` | `cadvisor.container` | — | +| GPU exporter | host `python3`, not a container | `127.0.0.1:9401` + host LAN `:9401` | `scripts/gpu-exporter.service` (user unit) | — | | Signal relay | host `python3`, not a container | `192.168.0.143:18093` | `scripts/signal-relay.service` (user unit) | [[Alertmanager]] | Every unit lives in `dankclaw/observability/quadlet/`, carries @@ -45,10 +46,11 @@ linger is on for `dank`). `signal-relay.service` reports `enabled` + `active` the same day, and `homepage-status.timer` — the root unit that probes all of these on loopback — is `enabled` + `active` (`deploy-remote.sh:319-321` copies the probe registry `shared/homepage/services.json` to `/srv/homepage`). -Dankserver runs two members of this family, node-exporter and cadvisor, as -its own units (`dankserver/observability/quadlet/{node-exporter,cadvisor}.container`, +Dankserver runs the same family: node-exporter and cadvisor as its own quadlets +(`dankserver/observability/quadlet/{node-exporter,cadvisor}.container`, `generated` + `active`; node-exporter binds `127.0.0.1:9100` + `192.168.0.235:9100`, -cadvisor publishes `127.0.0.1:8080` + `192.168.0.235:8080`). +cadvisor publishes `127.0.0.1:8080` + `192.168.0.235:8080`), plus the host-process +`gpu-exporter` (a user unit installed the same way as `signal-relay`). ## Exposure @@ -117,6 +119,18 @@ verify it through the APIs. `container` label. Network and filesystem are NOT exported for these containers (no netns/classid attribution under the host-PID layout), so the dashboard graphs only CPU, memory and OOM and does not fake the rest. +* **Per-GPU metrics: the `host-system` GPU panels (added 2026-10-20; live after + `deploy.sh --service gpu-exporter`).** `scripts/gpu-exporter.py` is a + host-process user unit on BOTH hosts — not a quadlet, because it reads the host's + own DRM sysfs and shells to `nvidia-smi`, neither of which a container can do + without GPU passthrough. It reads each AMD card's `gpu_busy_percent` / + `mem_info_vram_*` / `hwmon temp1_input` and one `nvidia-smi --query-gpu` CSV, and + emits unified `gpu_*` families (utilization %, VRAM used/total bytes, temperature + °C, power W) labelled `gpu` / `vendor` / `model`, with `host` added by the scrape + job. Two laws: **absent means absent, never a zero** — the R9700s expose no sysfs + `power1_average`, so they carry no `gpu_power_watts` line at all (only NVIDIA and + the Dankclaw 8060S iGPU report power); and it binds loopback + the host LAN, like + node-exporter, never the wildcard. The dashboard graphs util/VRAM/VRAM-%/temp/power. * **Logs: fixed in code, awaiting deploy.** The pipeline shipped nothing because a rootless Alloy cannot read the `0640` journal files, and the container-log globs point at a root-storage layout that does not exist here. The fix is built @@ -135,6 +149,7 @@ verify it through the APIs. - `dankclaw/observability/quadlet/{prometheus,alertmanager,grafana,loki,alloy,node-exporter,cadvisor}.container` - `scripts/deploy-remote.sh:219-300` (copiers + relay), `:521-536` (health waits), `:477-486` (grafana env seed) - `scripts/signal-relay.py:26-49`, `scripts/signal-relay.service` +- `scripts/gpu-exporter.py`, `scripts/gpu-exporter.service` (per-GPU host exporter) - `shared/prometheus/prometheus.yml`, `shared/prometheus/alertmanager.yml`, `shared/prometheus/rules/llm-fleet.yml` - `shared/grafana/provisioning/`, `shared/loki/loki-config.yaml`, `shared/alloy/config.alloy` - `tests/observability_quadlet.bats` (boot persistence, no wildcard publish, probe addresses, delivery contract) diff --git a/shared/grafana/provisioning/dashboards/host-system.json b/shared/grafana/provisioning/dashboards/host-system.json index 2c7fb19..487a712 100644 --- a/shared/grafana/provisioning/dashboards/host-system.json +++ b/shared/grafana/provisioning/dashboards/host-system.json @@ -18,6 +18,16 @@ "includeAll": true, "multi": true, "current": { "text": "All", "value": "$__all" } + }, + { + "name": "gpuhost", + "label": "gpu host", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": { "query": "label_values(gpu_utilization_percent, host)", "refId": "A" }, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } } ] }, @@ -139,6 +149,56 @@ { "expr": "sum by (job,device) (rate(node_network_receive_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} RX err", "refId": "A" }, { "expr": "sum by (job,device) (rate(node_network_transmit_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} TX err", "refId": "B" } ] + }, + { + "id": 13, "type": "timeseries", "title": "GPU busy %", + "description": "Per-GPU utilization from gpu-exporter (amdgpu gpu_busy_percent / nvidia-smi). The picker above selects the host; each series is one card.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, + "targets": [ + { "expr": "gpu_utilization_percent{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" } + ] + }, + { + "id": 14, "type": "timeseries", "title": "GPU VRAM used", + "description": "Device memory in use. On the R9700s (frank-archon) and the 3090 (frank-zealot) this is the loaded model's footprint.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes", "min": 0 }, "overrides": [] }, + "targets": [ + { "expr": "gpu_memory_used_bytes{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" } + ] + }, + { + "id": 15, "type": "timeseries", "title": "GPU VRAM used %", + "description": "Used / total device memory. Near 100% is the pre-OOM wall.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, + "targets": [ + { "expr": "100 * gpu_memory_used_bytes{host=~\"$gpuhost\"} / gpu_memory_total_bytes{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" } + ] + }, + { + "id": 16, "type": "timeseries", "title": "GPU temperature", + "description": "Package temperature. AMD edge temp (hwmon temp1_input); NVIDIA gpu temp.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "celsius", "min": 0 }, "overrides": [] }, + "targets": [ + { "expr": "gpu_temperature_celsius{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" } + ] + }, + { + "id": 17, "type": "timeseries", "title": "GPU power draw", + "description": "Board power draw. Emitted only where readable: NVIDIA and the Dankclaw iGPU. The R9700s have no sysfs power sensor, so they are simply absent here — never a fabricated 0.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "watt", "min": 0 }, "overrides": [] }, + "targets": [ + { "expr": "gpu_power_watts{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" } + ] } ] } diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index ef8584d..2342955 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -191,3 +191,25 @@ scrape_configs: action: keep - regex: 'id|name|image|container_label.*|machine_id|boot_id' action: labeldrop + + # Per-GPU metrics (gpu_utilization_percent / gpu_memory_used_bytes / + # gpu_memory_total_bytes / gpu_temperature_celsius / gpu_power_watts) from the + # host-process gpu-exporter (scripts/gpu-exporter.py). One job per host, same + # reach rules as node/cadvisor: Dankclaw is scraped at the podman gateway + # because the netavark hairpin quirk makes Prometheus fail to dial its own + # host LAN IP; Dankserver is plain LAN. The gpu label (PCI BDF on AMD, index on + # NVIDIA) + vendor/model come from the exporter, so the GPU row on the + # host-system dashboard filters by host. Power is emitted only where readable + # (NVIDIA + the Dankclaw iGPU); the R9700s have no sysfs power, so they simply + # have no gpu_power_watts series — no fabricated zero. + - job_name: "gpu-dankclaw" + static_configs: + - targets: ["host.containers.internal:9401"] + labels: + host: dankclaw + + - job_name: "gpu-dankserver" + static_configs: + - targets: ["192.168.0.235:9401"] + labels: + host: dankserver diff --git a/tests/gpu_exporter.bats b/tests/gpu_exporter.bats new file mode 100755 index 0000000..6275c79 --- /dev/null +++ b/tests/gpu_exporter.bats @@ -0,0 +1,197 @@ +#!/usr/bin/env bats +# Seam: the GPU metrics exporter's black-box contract. +# +# The exporter reads amdgpu sysfs (one card per /sys/class/drm/cardN that has a +# device/gpu_busy_percent file) and, on the box that has a working NVIDIA stack, +# one `nvidia-smi --query-gpu` CSV, and emits the unified `gpu_*` Prometheus +# families the host-system dashboard graphs. Both sources are injectable so the +# whole thing is testable with fixtures on a GPU-less dev box. +# +# Only the inputs a consumer sees are pinned: the metric families, their label +# set, and that absent hardware yields ABSENT series (never a fabricated 0 — a +# wrong number on a dashboard gets believed). The R9700s expose no sysfs +# power1_average, so those cards legitimately carry no gpu_power_watts line. + +load helpers + +EXPORTER="$REPO_ROOT/scripts/gpu-exporter.py" + +# Build a fixture amdgpu sysfs tree. $BATS_TEST_TMPDIR/sys +mk_amd() { # gpu busy vram_used vram_total power_uw_or_blank temp_mc + local card=$1 busy=$2 used=$3 total=$4 power=$5 temp=$6 + local d="$SYS/card$card/device" + mkdir -p "$d/hwmon/hwmon9" + printf '%s\n' "$busy" > "$d/gpu_busy_percent" + printf '%s\n' "$used" > "$d/mem_info_vram_used" + printf '%s\n' "$total" > "$d/mem_info_vram_total" + printf '%s\n' "$power" > "$d/power1_average" # blank => absent on R9700 + [ -n "$temp" ] && printf '%s\n' "$temp" > "$d/hwmon/hwmon9/temp1_input" + printf 'DRIVER=amdgpu\nPCI_ID=1002:7551\nPCI_SLOT_NAME=0000:03:00.0\n' > "$d/uevent" +} + +setup() { + SYS="$BATS_TEST_TMPDIR/sys" + mkdir -p "$SYS" +} + +@test "exporter exists and compiles" { + [ -f "$EXPORTER" ] + python3 -m py_compile "$EXPORTER" +} + +@test "amd card with full sysfs yields util/vram/temp/power with the right labels and units" { + mk_amd 1 42 1073741824 34359738368 250000000 71000 # 250 W, 71 C + run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + # util is a raw percent, VRAM verbatim bytes, temp milli->C, power microW->W. + echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="amd"[^}]*\} 42$' || { echo "$output"; return 1; } + echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*\} 1073741824$' || { echo "$output"; return 1; } + echo "$output" | grep -Eq 'gpu_memory_total_bytes\{[^}]*\} 34359738368$' || { echo "$output"; return 1; } + echo "$output" | grep -Eq 'gpu_temperature_celsius\{[^}]*\} 71$' || { echo "$output"; return 1; } + echo "$output" | grep -Eq 'gpu_power_watts\{[^}]*\} 250$' || { echo "$output"; return 1; } + # identity labels: the PCI slot is a stable unique id, vendor marks the family. + echo "$output" | grep -Eq 'gpu_utilization_percent\{.*gpu="0000:03:00\.0"' || { echo "$output"; return 1; } +} + +@test "card with an empty power file and no hwmon yields util/vram but omits power and temp" { + mk_amd 1 99 34108338176 34208743424 "" "" # the R9700 shape + run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*\} 99$' || { echo "$output"; return 1; } + echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*\} 34108338176$' + # Absent means ABSENT, not a zero the dashboard would plot as a real reading. + ! echo "$output" | grep -Eq '^gpu_power_watts\{' || { echo "fabricated power line: $output"; return 1; } + ! echo "$output" | grep -Eq '^gpu_temperature_celsius\{' || { echo "fabricated temp line: $output"; return 1; } +} + +@test "nvidia csv yields util/vram/temp/power tagged vendor=nvidia" { + nvidia="$BATS_TEST_TMPDIR/nvidia-smi" + cat > "$nvidia" <<'STUB' +echo '0, NVIDIA GeForce RTX 3090, 93, 23004, 24576, 76, 397.40' +STUB + chmod +x "$nvidia" + run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd "$nvidia" + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="nvidia"[^}]*\} 93$' || { echo "$output"; return 1; } + # memory.used MiB -> bytes (23004 * 1048576) + echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*vendor="nvidia"[^}]*\} 24121442304$' || { echo "$output"; return 1; } + echo "$output" | grep -Eq 'gpu_temperature_celsius\{[^}]*vendor="nvidia"[^}]*\} 76$' + echo "$output" | grep -Eq 'gpu_power_watts\{[^}]*vendor="nvidia"[^}]*\} 397\.4$' + echo "$output" | grep -Eq 'model="NVIDIA GeForce RTX 3090"' +} + +@test "a failing nvidia command is tolerated: amd still emits and exit is 0" { + mk_amd 1 50 1000000 2000000 "" 40000 + failing="$BATS_TEST_TMPDIR/nvidia-fail" + printf '#!/bin/sh\nexit 1\n' > "$failing"; chmod +x "$failing" + run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd "$failing" + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="amd"[^}]*\} 50$' + ! echo "$output" | grep -q 'vendor="nvidia"' +} + +@test "no GPUs at all yields only HELP/TYPE headers and exits 0" { + run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + # families are declared even when empty, so the dashboard/queries stay valid. + echo "$output" | grep -q '# TYPE gpu_utilization_percent gauge' + echo "$output" | grep -q '# TYPE gpu_memory_used_bytes gauge' + ! echo "$output" | grep -Eq '^gpu_utilization_percent\{' +} + +@test "output is valid Prometheus text" { + mk_amd 1 42 1000 2000 "" 50000 + run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false + [ "$status" -eq 0 ] + # every non-comment line is "{labels} value" or " value" + printf '%s\n' "$output" | python3 -c ' +import sys, re +pat = re.compile(r"^[a-z_]+(\{[^}]*\})? [-+0-9.eE+]+$") +for i, l in enumerate(sys.stdin): + l = l.rstrip("\n") + if not l or l.startswith("#"): + continue + assert pat.match(l), "malformed line %d: %r" % (i, l) +print("prometheus text ok") +' +} + +# ---- deployment + wiring contracts ------------------------------------- +# The exporter is a HOST-PROCESS user unit (not a quadlet): it must read the +# host's own DRM sysfs and shell to nvidia-smi, which a container could not do +# without GPU passthrough. These tests pin the cross-file wiring a reader +# cannot eyeball — a lone change on one side leaves the metric silently absent. + +@test "the base unit runs the exporter from the checkout and never binds a wildcard" { + local u="$REPO_ROOT/scripts/gpu-exporter.service" + [ -f "$u" ] + grep -q '^ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/gpu-exporter.py --serve$' "$u" \ + || { echo "unit does not run the exporter --serve from the checkout"; return 1; } + grep -q '^Restart=always$' "$u" + grep -q '^WantedBy=default.target$' "$u" + # Base unit binds loopback only; the per-host LAN address is added by a + # deploy drop-in, so a checked-in unit is never a wildcard. + grep -q '^Environment=GPU_EXPORTER_BINDS=127.0.0.1$' "$u" + ! grep -qE '0\.0\.0\.0' "$u" +} + +@test "the deployer installs, drops in the LAN bind, and health-gates gpu-exporter" { + local handler="$REPO_ROOT/scripts/deploy-remote.sh" + grep -q '^deploy_gpu_exporter() {' "$handler" + # Per-host LAN from $TARGET (never a hardcoded host, never a wildcard). + grep -q 'dankclaw) lan=192.168.0.143' "$handler" + grep -q 'dankserver) lan=192.168.0.235' "$handler" + # Drop-in binds loopback + this host's LAN as one comma list. + grep -q "GPU_EXPORTER_BINDS=127.0.0.1,%s" "$handler" + grep -q 'systemctl --user enable gpu-exporter.service' "$handler" + grep -q 'systemctl --user restart gpu-exporter.service' "$handler" + # Health gate waits on the loopback metrics endpoint. + grep -q 'http://127.0.0.1:9401/metrics' "$handler" + # Dispatched from the central case. + grep -q 'gpu-exporter) deploy_gpu_exporter ;;' "$handler" + # Allowlisted on BOTH hosts. + grep -q 'dankclaw:gpu-exporter' "$handler" + grep -q 'dankserver:gpu-exporter' "$handler" +} + +@test "the planner selects gpu-exporter for its files on each target" { + local t f + for t in dankclaw dankserver; do + for f in scripts/gpu-exporter.py scripts/gpu-exporter.service; do + run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target "$t" --paths "$f" + [ "$status" -eq 0 ] + [ "$output" = "gpu-exporter" ] || { echo "$t/$f -> '$output'"; return 1; } + done + done +} + +@test "Prometheus scrapes gpu-exporter where node/cadvisor are reachable on each host" { + # Cross-file contract, same posture as node-exporter: the local instance is + # reached via the podman gateway (this Prometheus is a bridge container and + # a host-process bind is refused at the host LAN IP); the Dankserver + # instance is a different machine, so plain LAN routing reaches it. If either + # address drifts from the exporter's bind, GPU panels go blank with no error. + local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" + grep -q 'job_name: "gpu-dankclaw"' "$cfg" || { echo "no gpu-dankclaw job"; return 1; } + grep -q 'job_name: "gpu-dankserver"' "$cfg" || { echo "no gpu-dankserver job"; return 1; } + grep -q '"host.containers.internal:9401"' "$cfg" \ + || { echo "gpu-dankclaw does not scrape the podman gateway"; return 1; } + grep -q '"192.168.0.235:9401"' "$cfg" \ + || { echo "gpu-dankserver does not scrape the LAN address"; return 1; } + # A vhost target would be red forever (Prometheus cannot solve Anubis). + ! grep -qE '"[^"]*franksdank[^"]*:9401"' "$cfg" +} + +@test "the host-system dashboard graphs the gpu_* families the exporter emits" { + local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/host-system.json" + [ -f "$dash" ] + python3 -c " +import json +d = json.load(open('$dash')) +exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])] +for want in ('gpu_utilization_percent', 'gpu_memory_used_bytes', + 'gpu_temperature_celsius', 'gpu_power_watts'): + assert any(want in e for e in exprs), ('missing', want, exprs) +print('gpu dashboard panels ok') +" +}