#!/usr/bin/env python3 """gpu-exporter: unified per-GPU metrics for a mixed AMD/NVIDIA host. Emits the `gpu_*` Prometheus families the host-system dashboard graphs: gpu_utilization_percent gpu_memory_used_bytes gpu_memory_total_bytes gpu_temperature_celsius gpu_power_watts each labelled {gpu, vendor, model}. The scrape job adds `host`. Two sources, both host-native — no container, no GPU passthrough: * AMD (R9700s, the 8060S iGPU, the Raphael display GPU): the amdgpu sysfs tree. Utilization comes ONLY from gpu_busy_percent — amd-smi reports gfx_busy_inst as N/A on gfx1201 (verified 2026-10-20). VRAM is already bytes, temperature is milli-degrees, power is micro-watts. Crucially power1_average is EMPTY on the R9700s, so those cards legitimately have no gpu_power_watts line; a missing file becomes an ABSENT metric, never a fabricated 0 that a dashboard would plot as a real reading. * NVIDIA (the RTX 3090 on Dankclaw): one `nvidia-smi --query-gpu` CSV. On Dankserver there is an nvidia-smi binary but no NVIDIA GPU, so the command fails; that is tolerated and simply yields no NVIDIA series. Default mode prints one scrape to stdout and exits (the test harness, and any `service ... --oneshot` one-off). `--serve` binds the HTTP endpoint for Prometheus; it binds EXPLICITLY (loopback + the host LAN address), never a wildcard, matching node-exporter's posture on these boxes. """ import argparse import glob import os import subprocess import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer FAMILIES = [ ("gpu_utilization_percent", "gauge", "GPU busy time as a percentage of wall time."), ("gpu_memory_used_bytes", "gauge", "Device memory in use, in bytes."), ("gpu_memory_total_bytes", "gauge", "Device memory capacity, in bytes."), ("gpu_temperature_celsius", "gauge", "Package temperature in degrees Celsius."), ("gpu_power_watts", "gauge", "Board power draw in watts."), ] DEFAULT_NVIDIA_CMD = ( "nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used," "memory.total,temperature.gpu,power.draw --format=csv,noheader,nounits" ) def _read(path): try: with open(path) as fh: return fh.read().strip() except OSError: return "" def _pci_slot(device_dir): uevent = _read(os.path.join(device_dir, "uevent")) for line in uevent.splitlines(): if line.startswith("PCI_SLOT_NAME="): return line.split("=", 1)[1] return "" def _pci_id(device_dir): uevent = _read(os.path.join(device_dir, "uevent")) for line in uevent.splitlines(): if line.startswith("PCI_ID="): return "amd:" + line.split("=", 1)[1].lower() return "amd" def collect_amd(sys_dir): for card in sorted(glob.glob(os.path.join(sys_dir, "card[0-9]*"))): device = os.path.join(card, "device") busy = _read(os.path.join(device, "gpu_busy_percent")) if busy == "": continue # not an amdgpu compute node (e.g. a render node w/o metrics) labels = {"gpu": _pci_slot(device) or os.path.basename(card), "vendor": "amd", "model": _pci_id(device)} yield "gpu_utilization_percent", labels, busy for name, leaf in (("gpu_memory_used_bytes", "mem_info_vram_used"), ("gpu_memory_total_bytes", "mem_info_vram_total")): v = _read(os.path.join(device, leaf)) if v != "": yield name, labels, v temp = "" for hwmon in sorted(glob.glob(os.path.join(device, "hwmon", "hwmon*", "temp1_input"))): temp = _read(hwmon) if temp != "": break if temp != "": yield "gpu_temperature_celsius", labels, str(int(temp) / 1000.0) power = _read(os.path.join(device, "power1_average")) if power != "": # empty on the R9700s -> metric absent, not zero yield "gpu_power_watts", labels, str(int(power) / 1e6) def collect_nvidia(cmd): if not cmd: return try: out = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=10, check=True).stdout except (subprocess.SubprocessError, OSError): return # no NVIDIA GPU / no driver: tolerate, emit nothing for line in out.splitlines(): parts = [p.strip() for p in line.split(",")] if len(parts) < 7: continue idx, name, util, mused, mtotal, temp, power = parts[:7] labels = {"gpu": idx, "vendor": "nvidia", "model": name} yield "gpu_utilization_percent", labels, util yield "gpu_memory_used_bytes", labels, str(int(float(mused)) * 1048576) yield "gpu_memory_total_bytes", labels, str(int(float(mtotal)) * 1048576) if temp not in ("", "[N/A]"): yield "gpu_temperature_celsius", labels, temp if power not in ("", "[N/A]"): yield "gpu_power_watts", labels, power def collect(sys_dir, nvidia_cmd): samples = {} for source in (collect_amd(sys_dir), collect_nvidia(nvidia_cmd)): for name, labels, value in source: try: value = str(float(value)) # normalise; drop trailing junk except ValueError: continue samples.setdefault(name, []).append((labels, value)) return samples def _fmt(value): f = float(value) if f == int(f) and abs(f) < 1e15: return str(int(f)) return repr(f) def render(samples): out = [] for name, kind, help_text in FAMILIES: out.append("# HELP %s %s" % (name, help_text)) out.append("# TYPE %s %s" % (name, kind)) for labels, value in sorted(samples.get(name, []), key=lambda x: sorted(x[0].items())): lset = ",".join('%s="%s"' % (k, v) for k, v in sorted(labels.items())) out.append("%s{%s} %s" % (name, lset, _fmt(value))) return "\n".join(out) + "\n" class Handler(BaseHTTPRequestHandler): sys_dir = "/sys/class/drm" nvidia_cmd = DEFAULT_NVIDIA_CMD def do_GET(self): # noqa: N802 (http.server API) if self.path.split("?")[0] not in ("/metrics", "/"): self.send_error(404) return body = render(collect(self.sys_dir, self.nvidia_cmd)).encode() self.send_response(200) self.send_header("Content-Type", "text/plain; version=0.0.4") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *args): pass def main(argv): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--amdgpu-sysfs", default="/sys/class/drm") ap.add_argument("--nvidia-cmd", default=DEFAULT_NVIDIA_CMD) ap.add_argument("--serve", action="store_true", help="serve /metrics instead of one-shot") ap.add_argument("--port", type=int, default=9401) ap.add_argument("--bind", action="append", default=None, help="explicit bind address (repeatable); env GPU_EXPORTER_BINDS " "comma-list if unset. NEVER a wildcard.") args = ap.parse_args(argv) if not args.serve: sys.stdout.write(render(collect(args.amdgpu_sysfs, args.nvidia_cmd))) return 0 binds = args.bind if not binds: env = os.environ.get("GPU_EXPORTER_BINDS", "") binds = [b.strip() for b in env.split(",") if b.strip()] or ["127.0.0.1"] Handler.sys_dir = args.amdgpu_sysfs Handler.nvidia_cmd = args.nvidia_cmd servers = [] for b in binds: if b in ("0.0.0.0", "::", "*"): sys.stderr.write("refusing wildcard bind %s\n" % b) return 2 servers.append(ThreadingHTTPServer((b, args.port), Handler)) sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds))) # EVERY socket must be served: a listen()ed-but-never-accept()ed bind looks # healthy in `ss` yet times out every scrape once its SYN backlog fills — # and the loopback deploy gate cannot see it. One thread per extra bind. for s in servers[1:]: threading.Thread(target=s.serve_forever, daemon=True).start() servers[0].serve_forever() return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))