feat(observability): per-GPU metrics for the mixed AMD/NVIDIA fleet
Add a host-process GPU exporter, scrape it on both servers, and graph it on the host-system dashboard. The fleet has four GPUs across three vendors of interest (2x R9700 on Dankserver, the 8060S iGPU + RTX 3090 on Dankclaw) and nothing exposed them until now. scripts/gpu-exporter.py reads each AMD card's amdgpu sysfs (gpu_busy_percent / mem_info_vram_* / hwmon temp1_input / power1_average) plus one nvidia-smi --query-gpu CSV, and emits unified gpu_* Prometheus families (utilization_percent, memory_used/total_bytes, temperature_celsius, power_watts) labelled gpu/vendor/model. Absent hardware yields ABSENT series, never a fabricated 0: the R9700s expose no sysfs power, so they simply carry no gpu_power_watts line. Both sources are injectable, so the seam runs on a GPU-less dev box; live-verified on Dankserver's real sysfs. It runs as a plain user unit (not a quadlet): it must read the host's own DRM sysfs and shell to nvidia-smi, which a container cannot do without GPU passthrough. Same posture as signal-relay. The base unit binds loopback; the deploy drop-in appends the host LAN address, never a wildcard (matching node-exporter/cadvisor). deploy_gpu_exporter installs/enables/health-gates it on both hosts and is wired into the planner (both targets), the test-changed router, and two Prometheus scrape jobs using the node/cadvisor reach rules (gateway on Dankclaw, LAN on Dankserver).
This commit is contained in:
Frank
@@ -40,6 +40,7 @@ SERVICE_ORDER = (
|
||||
"alloy",
|
||||
"node-exporter",
|
||||
"cadvisor",
|
||||
"gpu-exporter",
|
||||
"alertmanager",
|
||||
"authentik-postgres",
|
||||
"authentik-server",
|
||||
@@ -94,6 +95,7 @@ DANKCLAW_SERVICES = {
|
||||
"alloy",
|
||||
"node-exporter",
|
||||
"cadvisor",
|
||||
"gpu-exporter",
|
||||
"alertmanager",
|
||||
"authentik-postgres",
|
||||
"authentik-server",
|
||||
@@ -127,6 +129,7 @@ DANKSERVER_SERVICES = {
|
||||
"frank-archon",
|
||||
"node-exporter",
|
||||
"cadvisor",
|
||||
"gpu-exporter",
|
||||
}
|
||||
|
||||
# FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on
|
||||
@@ -452,6 +455,12 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
# change to it redeploying alertmanager is the only path that
|
||||
# reaches it on the host.
|
||||
_add(services, "alertmanager")
|
||||
if name in {"gpu-exporter.py", "gpu-exporter.service"}:
|
||||
# The host-process GPU exporter is installed directly by the
|
||||
# gpu-exporter deploy on BOTH hosts, so its script and unit select
|
||||
# that service (target_services then keeps it only on the host being
|
||||
# planned). Editing the collector restarts the running exporter.
|
||||
_add(services, "gpu-exporter")
|
||||
if name in {"frank-zealot.env", "frank-zealot.service", "serve-frank-zealot.sh"}:
|
||||
# llama.cpp frank-zealot (2026-09-16) keeps its unit, env and serve
|
||||
# files here; each one selects frank-zealot only. The ik_llama.cpp
|
||||
|
||||
@@ -86,7 +86,7 @@ service_is_allowed() {
|
||||
dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
|
||||
dankclaw:comfyui|dankclaw:hermes|\
|
||||
dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
|
||||
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:alertmanager|\
|
||||
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:alertmanager|\
|
||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
|
||||
@@ -97,7 +97,7 @@ service_is_allowed() {
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
||||
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
|
||||
dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|\
|
||||
dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|dankserver:gpu-exporter|\
|
||||
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
||||
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
||||
frankbook-pro:readarr)
|
||||
@@ -275,6 +275,36 @@ install_signal_relay() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# Host-process GPU metrics exporter (amdgpu sysfs + nvidia-smi). Not a quadlet:
|
||||
# it runs python3 from the checkout on the host so it can read the DRM/cgroup
|
||||
# sysfs and shell out to nvidia-smi. Same two-address bind posture as
|
||||
# node-exporter/cadvisor (loopback + this host's LAN), so it is never a wildcard.
|
||||
deploy_gpu_exporter() {
|
||||
local unit_dir="$HOME/.config/systemd/user" lan dropin i
|
||||
case "$TARGET" in
|
||||
dankclaw) lan=192.168.0.143 ;;
|
||||
dankserver) lan=192.168.0.235 ;;
|
||||
*) echo "gpu-exporter: unknown target '$TARGET'" >&2; return 2 ;;
|
||||
esac
|
||||
mkdir -p "$unit_dir"
|
||||
cp scripts/gpu-exporter.service "$unit_dir/gpu-exporter.service"
|
||||
dropin="$unit_dir/gpu-exporter.service.d"
|
||||
mkdir -p "$dropin"
|
||||
printf '[Service]\nEnvironment=GPU_EXPORTER_BINDS=127.0.0.1,%s\n' "$lan" \
|
||||
> "$dropin/bind.conf"
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable gpu-exporter.service
|
||||
systemctl --user restart gpu-exporter.service
|
||||
# The interpreter needs a moment to bind after restart (the same race that
|
||||
# once failed a healthy signal-relay deploy).
|
||||
for i in $(seq 1 15); do
|
||||
curl -fsS -m 5 http://127.0.0.1:9401/metrics >/dev/null && return 0
|
||||
sleep 1
|
||||
done
|
||||
echo "gpu-exporter is not answering on 127.0.0.1:9401/metrics" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
copy_loki_config() {
|
||||
run_privileged mkdir -p /opt/loki/config
|
||||
run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
|
||||
@@ -2004,6 +2034,7 @@ deploy_service() {
|
||||
tautulli) deploy_tautulli ;;
|
||||
alloy) deploy_alloy ;;
|
||||
prometheus|grafana|loki|node-exporter|cadvisor|alertmanager) deploy_observability "$service" ;;
|
||||
gpu-exporter) deploy_gpu_exporter ;;
|
||||
comfyui) deploy_comfyui ;;
|
||||
*) deploy_compose_service "$service" ;;
|
||||
esac
|
||||
|
||||
Executable
+206
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""gpu-exporter: unified per-GPU metrics for a mixed AMD/NVIDIA host.
|
||||
|
||||
Emits the `gpu_*` Prometheus families the host-system dashboard graphs:
|
||||
|
||||
gpu_utilization_percent gpu_memory_used_bytes gpu_memory_total_bytes
|
||||
gpu_temperature_celsius gpu_power_watts
|
||||
|
||||
each labelled {gpu, vendor, model}. The scrape job adds `host`.
|
||||
|
||||
Two sources, both host-native — no container, no GPU passthrough:
|
||||
|
||||
* AMD (R9700s, the 8060S iGPU, the Raphael display GPU): the amdgpu sysfs tree.
|
||||
Utilization comes ONLY from gpu_busy_percent — amd-smi reports gfx_busy_inst
|
||||
as N/A on gfx1201 (verified 2026-10-20). VRAM is already bytes, temperature
|
||||
is milli-degrees, power is micro-watts. Crucially power1_average is EMPTY on
|
||||
the R9700s, so those cards legitimately have no gpu_power_watts line; a
|
||||
missing file becomes an ABSENT metric, never a fabricated 0 that a dashboard
|
||||
would plot as a real reading.
|
||||
|
||||
* NVIDIA (the RTX 3090 on Dankclaw): one `nvidia-smi --query-gpu` CSV. On
|
||||
Dankserver there is an nvidia-smi binary but no NVIDIA GPU, so the command
|
||||
fails; that is tolerated and simply yields no NVIDIA series.
|
||||
|
||||
Default mode prints one scrape to stdout and exits (the test harness, and any
|
||||
`service ... --oneshot` one-off). `--serve` binds the HTTP endpoint for
|
||||
Prometheus; it binds EXPLICITLY (loopback + the host LAN address), never a
|
||||
wildcard, matching node-exporter's posture on these boxes.
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
FAMILIES = [
|
||||
("gpu_utilization_percent", "gauge", "GPU busy time as a percentage of wall time."),
|
||||
("gpu_memory_used_bytes", "gauge", "Device memory in use, in bytes."),
|
||||
("gpu_memory_total_bytes", "gauge", "Device memory capacity, in bytes."),
|
||||
("gpu_temperature_celsius", "gauge", "Package temperature in degrees Celsius."),
|
||||
("gpu_power_watts", "gauge", "Board power draw in watts."),
|
||||
]
|
||||
|
||||
DEFAULT_NVIDIA_CMD = (
|
||||
"nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,"
|
||||
"memory.total,temperature.gpu,power.draw --format=csv,noheader,nounits"
|
||||
)
|
||||
|
||||
|
||||
def _read(path):
|
||||
try:
|
||||
with open(path) as fh:
|
||||
return fh.read().strip()
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
|
||||
def _pci_slot(device_dir):
|
||||
uevent = _read(os.path.join(device_dir, "uevent"))
|
||||
for line in uevent.splitlines():
|
||||
if line.startswith("PCI_SLOT_NAME="):
|
||||
return line.split("=", 1)[1]
|
||||
return ""
|
||||
|
||||
|
||||
def _pci_id(device_dir):
|
||||
uevent = _read(os.path.join(device_dir, "uevent"))
|
||||
for line in uevent.splitlines():
|
||||
if line.startswith("PCI_ID="):
|
||||
return "amd:" + line.split("=", 1)[1].lower()
|
||||
return "amd"
|
||||
|
||||
|
||||
def collect_amd(sys_dir):
|
||||
for card in sorted(glob.glob(os.path.join(sys_dir, "card[0-9]*"))):
|
||||
device = os.path.join(card, "device")
|
||||
busy = _read(os.path.join(device, "gpu_busy_percent"))
|
||||
if busy == "":
|
||||
continue # not an amdgpu compute node (e.g. a render node w/o metrics)
|
||||
labels = {"gpu": _pci_slot(device) or os.path.basename(card),
|
||||
"vendor": "amd", "model": _pci_id(device)}
|
||||
yield "gpu_utilization_percent", labels, busy
|
||||
for name, leaf in (("gpu_memory_used_bytes", "mem_info_vram_used"),
|
||||
("gpu_memory_total_bytes", "mem_info_vram_total")):
|
||||
v = _read(os.path.join(device, leaf))
|
||||
if v != "":
|
||||
yield name, labels, v
|
||||
temp = ""
|
||||
for hwmon in sorted(glob.glob(os.path.join(device, "hwmon", "hwmon*", "temp1_input"))):
|
||||
temp = _read(hwmon)
|
||||
if temp != "":
|
||||
break
|
||||
if temp != "":
|
||||
yield "gpu_temperature_celsius", labels, str(int(temp) / 1000.0)
|
||||
power = _read(os.path.join(device, "power1_average"))
|
||||
if power != "": # empty on the R9700s -> metric absent, not zero
|
||||
yield "gpu_power_watts", labels, str(int(power) / 1e6)
|
||||
|
||||
|
||||
def collect_nvidia(cmd):
|
||||
if not cmd:
|
||||
return
|
||||
try:
|
||||
out = subprocess.run(cmd, shell=True, capture_output=True, text=True,
|
||||
timeout=10, check=True).stdout
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
return # no NVIDIA GPU / no driver: tolerate, emit nothing
|
||||
for line in out.splitlines():
|
||||
parts = [p.strip() for p in line.split(",")]
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
idx, name, util, mused, mtotal, temp, power = parts[:7]
|
||||
labels = {"gpu": idx, "vendor": "nvidia", "model": name}
|
||||
yield "gpu_utilization_percent", labels, util
|
||||
yield "gpu_memory_used_bytes", labels, str(int(float(mused)) * 1048576)
|
||||
yield "gpu_memory_total_bytes", labels, str(int(float(mtotal)) * 1048576)
|
||||
if temp not in ("", "[N/A]"):
|
||||
yield "gpu_temperature_celsius", labels, temp
|
||||
if power not in ("", "[N/A]"):
|
||||
yield "gpu_power_watts", labels, power
|
||||
|
||||
|
||||
def collect(sys_dir, nvidia_cmd):
|
||||
samples = {}
|
||||
for source in (collect_amd(sys_dir), collect_nvidia(nvidia_cmd)):
|
||||
for name, labels, value in source:
|
||||
try:
|
||||
value = str(float(value)) # normalise; drop trailing junk
|
||||
except ValueError:
|
||||
continue
|
||||
samples.setdefault(name, []).append((labels, value))
|
||||
return samples
|
||||
|
||||
|
||||
def _fmt(value):
|
||||
f = float(value)
|
||||
if f == int(f) and abs(f) < 1e15:
|
||||
return str(int(f))
|
||||
return repr(f)
|
||||
|
||||
|
||||
def render(samples):
|
||||
out = []
|
||||
for name, kind, help_text in FAMILIES:
|
||||
out.append("# HELP %s %s" % (name, help_text))
|
||||
out.append("# TYPE %s %s" % (name, kind))
|
||||
for labels, value in sorted(samples.get(name, []), key=lambda x: sorted(x[0].items())):
|
||||
lset = ",".join('%s="%s"' % (k, v) for k, v in sorted(labels.items()))
|
||||
out.append("%s{%s} %s" % (name, lset, _fmt(value)))
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
sys_dir = "/sys/class/drm"
|
||||
nvidia_cmd = DEFAULT_NVIDIA_CMD
|
||||
|
||||
def do_GET(self): # noqa: N802 (http.server API)
|
||||
if self.path.split("?")[0] not in ("/metrics", "/"):
|
||||
self.send_error(404)
|
||||
return
|
||||
body = render(collect(self.sys_dir, self.nvidia_cmd)).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain; version=0.0.4")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
def main(argv):
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--amdgpu-sysfs", default="/sys/class/drm")
|
||||
ap.add_argument("--nvidia-cmd", default=DEFAULT_NVIDIA_CMD)
|
||||
ap.add_argument("--serve", action="store_true", help="serve /metrics instead of one-shot")
|
||||
ap.add_argument("--port", type=int, default=9401)
|
||||
ap.add_argument("--bind", action="append", default=None,
|
||||
help="explicit bind address (repeatable); env GPU_EXPORTER_BINDS "
|
||||
"comma-list if unset. NEVER a wildcard.")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
if not args.serve:
|
||||
sys.stdout.write(render(collect(args.amdgpu_sysfs, args.nvidia_cmd)))
|
||||
return 0
|
||||
|
||||
binds = args.bind
|
||||
if not binds:
|
||||
env = os.environ.get("GPU_EXPORTER_BINDS", "")
|
||||
binds = [b.strip() for b in env.split(",") if b.strip()] or ["127.0.0.1"]
|
||||
Handler.sys_dir = args.amdgpu_sysfs
|
||||
Handler.nvidia_cmd = args.nvidia_cmd
|
||||
servers = []
|
||||
for b in binds:
|
||||
if b in ("0.0.0.0", "::", "*"):
|
||||
sys.stderr.write("refusing wildcard bind %s\n" % b)
|
||||
return 2
|
||||
servers.append(ThreadingHTTPServer((b, args.port), Handler))
|
||||
sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds)))
|
||||
servers[0].serve_forever()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,35 @@
|
||||
# Systemd USER unit: gpu-exporter (both Dankclaw and Dankserver).
|
||||
#
|
||||
# Install path: ~/.config/systemd/user/gpu-exporter.service
|
||||
# Installed and enabled by deploy_gpu_exporter in scripts/deploy-remote.sh.
|
||||
#
|
||||
# Per-GPU metrics (utilization / VRAM / temperature / power) for the mixed
|
||||
# AMD + NVIDIA fleet, scraped by Prometheus. Like signal-relay it is a plain
|
||||
# user unit, not a quadlet: it runs python3 straight from the checkout (a pull
|
||||
# updates it with no copy step), a hand-written user unit can be enabled where
|
||||
# the podman v1 generator's units cannot, and with Linger=yes for dank it starts
|
||||
# at boot. It reads the host's own cgroup/drm sysfs and shells to nvidia-smi, so
|
||||
# it must run on the host — a container would need GPU/DRM passthrough for
|
||||
# nothing gained.
|
||||
#
|
||||
# BINDING. The base unit serves loopback only. deploy_gpu_exporter writes a
|
||||
# drop-in adding THIS host's LAN address (the same two-address posture
|
||||
# node-exporter and cadvisor use), so the endpoint is never a wildcard. It is
|
||||
# unauthenticated metrics on the LAN, identical to node-exporter :9100 and
|
||||
# cadvisor :8080 — Prometheus reaches same-host via host.containers.internal and
|
||||
# the Dankserver instance over the LAN from Dankclaw.
|
||||
|
||||
[Unit]
|
||||
Description=GPU metrics exporter (amdgpu sysfs + nvidia-smi)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
# Loopback by default; the deploy drop-in appends the host LAN address.
|
||||
Environment=GPU_EXPORTER_BINDS=127.0.0.1
|
||||
ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/gpu-exporter.py --serve
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -293,6 +293,14 @@ for path in "${CHANGED[@]}"; do
|
||||
add_test tests/caddyfile.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
;;
|
||||
# The GPU exporter is a host process; its seam pins the collector, the
|
||||
# user unit, the deployer function, the planner route, the Prometheus
|
||||
# scrape jobs and the host-system GPU panels — a cross-file contract, so
|
||||
# the whole seam rides on any of its own files.
|
||||
scripts/gpu-exporter.py|scripts/gpu-exporter.service)
|
||||
add_test tests/gpu_exporter.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
;;
|
||||
dankclaw/udev/*)
|
||||
# Host-level udev rules are applied by manual sudo install + udevadm
|
||||
# trigger, not dispatched as a service; nothing here is bats-testable.
|
||||
|
||||
Reference in New Issue
Block a user