serverAppConfigs/scripts/gpu-exporter.py

#!/usr/bin/env python3
"""gpu-exporter: unified per-GPU metrics for a mixed AMD/NVIDIA host.

Emits the `gpu_*` Prometheus families the host-system dashboard graphs:

    gpu_utilization_percent  gpu_memory_used_bytes  gpu_memory_total_bytes
    gpu_temperature_celsius  gpu_power_watts

each labelled {gpu, vendor, model}. The scrape job adds `host`.

Two sources, both host-native — no container, no GPU passthrough:

  * AMD (R9700s, the 8060S iGPU, the Raphael display GPU): the amdgpu sysfs tree.
    Utilization comes ONLY from gpu_busy_percent — amd-smi reports gfx_busy_inst
    as N/A on gfx1201 (verified 2026-10-20). VRAM is already bytes, temperature
    is milli-degrees, power is micro-watts. Crucially power1_average is EMPTY on
    the R9700s, so those cards legitimately have no gpu_power_watts line; a
    missing file becomes an ABSENT metric, never a fabricated 0 that a dashboard
    would plot as a real reading.

  * NVIDIA (the RTX 3090 on Dankclaw): one `nvidia-smi --query-gpu` CSV. On
    Dankserver there is an nvidia-smi binary but no NVIDIA GPU, so the command
    fails; that is tolerated and simply yields no NVIDIA series.

Default mode prints one scrape to stdout and exits (the test harness, and any
`service ... --oneshot` one-off). `--serve` binds the HTTP endpoint for
Prometheus; it binds EXPLICITLY (loopback + the host LAN address), never a
wildcard, matching node-exporter's posture on these boxes.
"""
import argparse
import glob
import os
import subprocess
import sys
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

FAMILIES = [
    ("gpu_utilization_percent", "gauge", "GPU busy time as a percentage of wall time."),
    ("gpu_memory_used_bytes", "gauge", "Device memory in use, in bytes."),
    ("gpu_memory_total_bytes", "gauge", "Device memory capacity, in bytes."),
    ("gpu_temperature_celsius", "gauge", "Package temperature in degrees Celsius."),
    ("gpu_power_watts", "gauge", "Board power draw in watts."),
]

DEFAULT_NVIDIA_CMD = (
    "nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,"
    "memory.total,temperature.gpu,power.draw --format=csv,noheader,nounits"
)


def _read(path):
    try:
        with open(path) as fh:
            return fh.read().strip()
    except OSError:
        return ""


def _pci_slot(device_dir):
    uevent = _read(os.path.join(device_dir, "uevent"))
    for line in uevent.splitlines():
        if line.startswith("PCI_SLOT_NAME="):
            return line.split("=", 1)[1]
    return ""


def _pci_id(device_dir):
    uevent = _read(os.path.join(device_dir, "uevent"))
    for line in uevent.splitlines():
        if line.startswith("PCI_ID="):
            return "amd:" + line.split("=", 1)[1].lower()
    return "amd"


def collect_amd(sys_dir):
    for card in sorted(glob.glob(os.path.join(sys_dir, "card[0-9]*"))):
        device = os.path.join(card, "device")
        busy = _read(os.path.join(device, "gpu_busy_percent"))
        if busy == "":
            continue  # not an amdgpu compute node (e.g. a render node w/o metrics)
        labels = {"gpu": _pci_slot(device) or os.path.basename(card),
                  "vendor": "amd", "model": _pci_id(device)}
        yield "gpu_utilization_percent", labels, busy
        for name, leaf in (("gpu_memory_used_bytes", "mem_info_vram_used"),
                           ("gpu_memory_total_bytes", "mem_info_vram_total")):
            v = _read(os.path.join(device, leaf))
            if v != "":
                yield name, labels, v
        temp = ""
        for hwmon in sorted(glob.glob(os.path.join(device, "hwmon", "hwmon*", "temp1_input"))):
            temp = _read(hwmon)
            if temp != "":
                break
        if temp != "":
            yield "gpu_temperature_celsius", labels, str(int(temp) / 1000.0)
        power = _read(os.path.join(device, "power1_average"))
        if power != "":  # empty on the R9700s -> metric absent, not zero
            yield "gpu_power_watts", labels, str(int(power) / 1e6)


def collect_nvidia(cmd):
    if not cmd:
        return
    try:
        out = subprocess.run(cmd, shell=True, capture_output=True, text=True,
                             timeout=10, check=True).stdout
    except (subprocess.SubprocessError, OSError):
        return  # no NVIDIA GPU / no driver: tolerate, emit nothing
    for line in out.splitlines():
        parts = [p.strip() for p in line.split(",")]
        if len(parts) < 7:
            continue
        idx, name, util, mused, mtotal, temp, power = parts[:7]
        labels = {"gpu": idx, "vendor": "nvidia", "model": name}
        yield "gpu_utilization_percent", labels, util
        yield "gpu_memory_used_bytes", labels, str(int(float(mused)) * 1048576)
        yield "gpu_memory_total_bytes", labels, str(int(float(mtotal)) * 1048576)
        if temp not in ("", "[N/A]"):
            yield "gpu_temperature_celsius", labels, temp
        if power not in ("", "[N/A]"):
            yield "gpu_power_watts", labels, power


def collect(sys_dir, nvidia_cmd):
    samples = {}
    for source in (collect_amd(sys_dir), collect_nvidia(nvidia_cmd)):
        for name, labels, value in source:
            try:
                value = str(float(value))  # normalise; drop trailing junk
            except ValueError:
                continue
            samples.setdefault(name, []).append((labels, value))
    return samples


def _fmt(value):
    f = float(value)
    if f == int(f) and abs(f) < 1e15:
        return str(int(f))
    return repr(f)


def render(samples):
    out = []
    for name, kind, help_text in FAMILIES:
        out.append("# HELP %s %s" % (name, help_text))
        out.append("# TYPE %s %s" % (name, kind))
        for labels, value in sorted(samples.get(name, []), key=lambda x: sorted(x[0].items())):
            lset = ",".join('%s="%s"' % (k, v) for k, v in sorted(labels.items()))
            out.append("%s{%s} %s" % (name, lset, _fmt(value)))
    return "\n".join(out) + "\n"


class Handler(BaseHTTPRequestHandler):
    sys_dir = "/sys/class/drm"
    nvidia_cmd = DEFAULT_NVIDIA_CMD

    def do_GET(self):  # noqa: N802 (http.server API)
        if self.path.split("?")[0] not in ("/metrics", "/"):
            self.send_error(404)
            return
        body = render(collect(self.sys_dir, self.nvidia_cmd)).encode()
        self.send_response(200)
        self.send_header("Content-Type", "text/plain; version=0.0.4")
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self.wfile.write(body)

    def log_message(self, *args):
        pass


def main(argv):
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--amdgpu-sysfs", default="/sys/class/drm")
    ap.add_argument("--nvidia-cmd", default=DEFAULT_NVIDIA_CMD)
    ap.add_argument("--serve", action="store_true", help="serve /metrics instead of one-shot")
    ap.add_argument("--port", type=int, default=9401)
    ap.add_argument("--bind", action="append", default=None,
                    help="explicit bind address (repeatable); env GPU_EXPORTER_BINDS "
                         "comma-list if unset. NEVER a wildcard.")
    args = ap.parse_args(argv)

    if not args.serve:
        sys.stdout.write(render(collect(args.amdgpu_sysfs, args.nvidia_cmd)))
        return 0

    binds = args.bind
    if not binds:
        env = os.environ.get("GPU_EXPORTER_BINDS", "")
        binds = [b.strip() for b in env.split(",") if b.strip()] or ["127.0.0.1"]
    Handler.sys_dir = args.amdgpu_sysfs
    Handler.nvidia_cmd = args.nvidia_cmd
    servers = []
    for b in binds:
        if b in ("0.0.0.0", "::", "*"):
            sys.stderr.write("refusing wildcard bind %s\n" % b)
            return 2
        servers.append(ThreadingHTTPServer((b, args.port), Handler))
    sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds)))
    # EVERY socket must be served: a listen()ed-but-never-accept()ed bind looks
    # healthy in `ss` yet times out every scrape once its SYN backlog fills —
    # and the loopback deploy gate cannot see it. One thread per extra bind.
    for s in servers[1:]:
        threading.Thread(target=s.serve_forever, daemon=True).start()
    servers[0].serve_forever()
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))