feat(observability): per-GPU metrics for the mixed AMD/NVIDIA fleet

Add a host-process GPU exporter, scrape it on both servers, and graph it
on the host-system dashboard. The fleet has four GPUs across three vendors
of interest (2x R9700 on Dankserver, the 8060S iGPU + RTX 3090 on Dankclaw)
and nothing exposed them until now.

scripts/gpu-exporter.py reads each AMD card's amdgpu sysfs
(gpu_busy_percent / mem_info_vram_* / hwmon temp1_input / power1_average)
plus one nvidia-smi --query-gpu CSV, and emits unified gpu_* Prometheus
families (utilization_percent, memory_used/total_bytes, temperature_celsius,
power_watts) labelled gpu/vendor/model. Absent hardware yields ABSENT series,
never a fabricated 0: the R9700s expose no sysfs power, so they simply carry
no gpu_power_watts line. Both sources are injectable, so the seam runs on a
GPU-less dev box; live-verified on Dankserver's real sysfs.

It runs as a plain user unit (not a quadlet): it must read the host's own DRM
sysfs and shell to nvidia-smi, which a container cannot do without GPU
passthrough. Same posture as signal-relay. The base unit binds loopback; the
deploy drop-in appends the host LAN address, never a wildcard (matching
node-exporter/cadvisor). deploy_gpu_exporter installs/enables/health-gates it
on both hosts and is wired into the planner (both targets), the test-changed
router, and two Prometheus scrape jobs using the node/cadvisor reach rules
(gateway on Dankclaw, LAN on Dankserver).

scripts/deploy-plan.py

@@ -40,6 +40,7 @@ SERVICE_ORDER = (
     "alloy",
     "node-exporter",
     "cadvisor",
+    "gpu-exporter",
     "alertmanager",
     "authentik-postgres",
     "authentik-server",
@@ -94,6 +95,7 @@ DANKCLAW_SERVICES = {
     "alloy",
     "node-exporter",
     "cadvisor",
+    "gpu-exporter",
     "alertmanager",
     "authentik-postgres",
     "authentik-server",
@@ -127,6 +129,7 @@ DANKSERVER_SERVICES = {
     "frank-archon",
     "node-exporter",
     "cadvisor",
+    "gpu-exporter",
 }
 
 # FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on
@@ -452,6 +455,12 @@ def services_for_path(path: str, target: str) -> set[str]:
             # change to it redeploying alertmanager is the only path that
             # reaches it on the host.
             _add(services, "alertmanager")
+        if name in {"gpu-exporter.py", "gpu-exporter.service"}:
+            # The host-process GPU exporter is installed directly by the
+            # gpu-exporter deploy on BOTH hosts, so its script and unit select
+            # that service (target_services then keeps it only on the host being
+            # planned). Editing the collector restarts the running exporter.
+            _add(services, "gpu-exporter")
         if name in {"frank-zealot.env", "frank-zealot.service", "serve-frank-zealot.sh"}:
             # llama.cpp frank-zealot (2026-09-16) keeps its unit, env and serve
             # files here; each one selects frank-zealot only. The ik_llama.cpp

scripts/deploy-remote.sh

@@ -86,7 +86,7 @@ service_is_allowed() {
         dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
         dankclaw:comfyui|dankclaw:hermes|\
         dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
-        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:alertmanager|\
+        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:alertmanager|\
         dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
         dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
@@ -97,7 +97,7 @@ service_is_allowed() {
         dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
         dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
         dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
-        dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|\
+        dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|dankserver:gpu-exporter|\
         frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
         frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
         frankbook-pro:readarr)
@@ -275,6 +275,36 @@ install_signal_relay() {
     return 1
 }
 
+# Host-process GPU metrics exporter (amdgpu sysfs + nvidia-smi). Not a quadlet:
+# it runs python3 from the checkout on the host so it can read the DRM/cgroup
+# sysfs and shell out to nvidia-smi. Same two-address bind posture as
+# node-exporter/cadvisor (loopback + this host's LAN), so it is never a wildcard.
+deploy_gpu_exporter() {
+    local unit_dir="$HOME/.config/systemd/user" lan dropin i
+    case "$TARGET" in
+        dankclaw) lan=192.168.0.143 ;;
+        dankserver) lan=192.168.0.235 ;;
+        *) echo "gpu-exporter: unknown target '$TARGET'" >&2; return 2 ;;
+    esac
+    mkdir -p "$unit_dir"
+    cp scripts/gpu-exporter.service "$unit_dir/gpu-exporter.service"
+    dropin="$unit_dir/gpu-exporter.service.d"
+    mkdir -p "$dropin"
+    printf '[Service]\nEnvironment=GPU_EXPORTER_BINDS=127.0.0.1,%s\n' "$lan" \
+        > "$dropin/bind.conf"
+    systemctl --user daemon-reload
+    systemctl --user enable gpu-exporter.service
+    systemctl --user restart gpu-exporter.service
+    # The interpreter needs a moment to bind after restart (the same race that
+    # once failed a healthy signal-relay deploy).
+    for i in $(seq 1 15); do
+        curl -fsS -m 5 http://127.0.0.1:9401/metrics >/dev/null && return 0
+        sleep 1
+    done
+    echo "gpu-exporter is not answering on 127.0.0.1:9401/metrics" >&2
+    return 1
+}
+
 copy_loki_config() {
     run_privileged mkdir -p /opt/loki/config
     run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
@@ -2004,6 +2034,7 @@ deploy_service() {
         tautulli) deploy_tautulli ;;
         alloy) deploy_alloy ;;
         prometheus|grafana|loki|node-exporter|cadvisor|alertmanager) deploy_observability "$service" ;;
+        gpu-exporter) deploy_gpu_exporter ;;
         comfyui) deploy_comfyui ;;
         *) deploy_compose_service "$service" ;;
     esac

scripts/frank-archon.env

@@ -48,7 +48,7 @@ FRANK_ARCHON_TP=2
 FRANK_ARCHON_TP_WIRE=wht6
 # Decode slots. A slot costs KV and pinned RAM, never decode: the cache target is
 # exactly MAX_SEQS x MAX_MODEL_LEN and the checkpoint store asks (1 + what the
-# retention policy keeps) per slot. Ordered 2026-10-21: FOUR slots over a pool of at
+# retention policy keeps) per slot. Ordered 2026-10-20: FOUR slots over a pool of at
 # least 700,000 fp8 tokens, rounding UP if the resolver cannot land it exactly. The
 # target is therefore 4 x 200,000 = 800,000 tokens and HOST_POOL_MIB decides whether
 # the cache gets it (expert_vram_floor, core/place/planner.cpp:100-104: the card must

scripts/gpu-exporter.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""gpu-exporter: unified per-GPU metrics for a mixed AMD/NVIDIA host.
+
+Emits the `gpu_*` Prometheus families the host-system dashboard graphs:
+
+    gpu_utilization_percent  gpu_memory_used_bytes  gpu_memory_total_bytes
+    gpu_temperature_celsius  gpu_power_watts
+
+each labelled {gpu, vendor, model}. The scrape job adds `host`.
+
+Two sources, both host-native — no container, no GPU passthrough:
+
+  * AMD (R9700s, the 8060S iGPU, the Raphael display GPU): the amdgpu sysfs tree.
+    Utilization comes ONLY from gpu_busy_percent — amd-smi reports gfx_busy_inst
+    as N/A on gfx1201 (verified 2026-10-20). VRAM is already bytes, temperature
+    is milli-degrees, power is micro-watts. Crucially power1_average is EMPTY on
+    the R9700s, so those cards legitimately have no gpu_power_watts line; a
+    missing file becomes an ABSENT metric, never a fabricated 0 that a dashboard
+    would plot as a real reading.
+
+  * NVIDIA (the RTX 3090 on Dankclaw): one `nvidia-smi --query-gpu` CSV. On
+    Dankserver there is an nvidia-smi binary but no NVIDIA GPU, so the command
+    fails; that is tolerated and simply yields no NVIDIA series.
+
+Default mode prints one scrape to stdout and exits (the test harness, and any
+`service ... --oneshot` one-off). `--serve` binds the HTTP endpoint for
+Prometheus; it binds EXPLICITLY (loopback + the host LAN address), never a
+wildcard, matching node-exporter's posture on these boxes.
+"""
+import argparse
+import glob
+import os
+import subprocess
+import sys
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+FAMILIES = [
+    ("gpu_utilization_percent", "gauge", "GPU busy time as a percentage of wall time."),
+    ("gpu_memory_used_bytes", "gauge", "Device memory in use, in bytes."),
+    ("gpu_memory_total_bytes", "gauge", "Device memory capacity, in bytes."),
+    ("gpu_temperature_celsius", "gauge", "Package temperature in degrees Celsius."),
+    ("gpu_power_watts", "gauge", "Board power draw in watts."),
+]
+
+DEFAULT_NVIDIA_CMD = (
+    "nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,"
+    "memory.total,temperature.gpu,power.draw --format=csv,noheader,nounits"
+)
+
+
+def _read(path):
+    try:
+        with open(path) as fh:
+            return fh.read().strip()
+    except OSError:
+        return ""
+
+
+def _pci_slot(device_dir):
+    uevent = _read(os.path.join(device_dir, "uevent"))
+    for line in uevent.splitlines():
+        if line.startswith("PCI_SLOT_NAME="):
+            return line.split("=", 1)[1]
+    return ""
+
+
+def _pci_id(device_dir):
+    uevent = _read(os.path.join(device_dir, "uevent"))
+    for line in uevent.splitlines():
+        if line.startswith("PCI_ID="):
+            return "amd:" + line.split("=", 1)[1].lower()
+    return "amd"
+
+
+def collect_amd(sys_dir):
+    for card in sorted(glob.glob(os.path.join(sys_dir, "card[0-9]*"))):
+        device = os.path.join(card, "device")
+        busy = _read(os.path.join(device, "gpu_busy_percent"))
+        if busy == "":
+            continue  # not an amdgpu compute node (e.g. a render node w/o metrics)
+        labels = {"gpu": _pci_slot(device) or os.path.basename(card),
+                  "vendor": "amd", "model": _pci_id(device)}
+        yield "gpu_utilization_percent", labels, busy
+        for name, leaf in (("gpu_memory_used_bytes", "mem_info_vram_used"),
+                           ("gpu_memory_total_bytes", "mem_info_vram_total")):
+            v = _read(os.path.join(device, leaf))
+            if v != "":
+                yield name, labels, v
+        temp = ""
+        for hwmon in sorted(glob.glob(os.path.join(device, "hwmon", "hwmon*", "temp1_input"))):
+            temp = _read(hwmon)
+            if temp != "":
+                break
+        if temp != "":
+            yield "gpu_temperature_celsius", labels, str(int(temp) / 1000.0)
+        power = _read(os.path.join(device, "power1_average"))
+        if power != "":  # empty on the R9700s -> metric absent, not zero
+            yield "gpu_power_watts", labels, str(int(power) / 1e6)
+
+
+def collect_nvidia(cmd):
+    if not cmd:
+        return
+    try:
+        out = subprocess.run(cmd, shell=True, capture_output=True, text=True,
+                             timeout=10, check=True).stdout
+    except (subprocess.SubprocessError, OSError):
+        return  # no NVIDIA GPU / no driver: tolerate, emit nothing
+    for line in out.splitlines():
+        parts = [p.strip() for p in line.split(",")]
+        if len(parts) < 7:
+            continue
+        idx, name, util, mused, mtotal, temp, power = parts[:7]
+        labels = {"gpu": idx, "vendor": "nvidia", "model": name}
+        yield "gpu_utilization_percent", labels, util
+        yield "gpu_memory_used_bytes", labels, str(int(float(mused)) * 1048576)
+        yield "gpu_memory_total_bytes", labels, str(int(float(mtotal)) * 1048576)
+        if temp not in ("", "[N/A]"):
+            yield "gpu_temperature_celsius", labels, temp
+        if power not in ("", "[N/A]"):
+            yield "gpu_power_watts", labels, power
+
+
+def collect(sys_dir, nvidia_cmd):
+    samples = {}
+    for source in (collect_amd(sys_dir), collect_nvidia(nvidia_cmd)):
+        for name, labels, value in source:
+            try:
+                value = str(float(value))  # normalise; drop trailing junk
+            except ValueError:
+                continue
+            samples.setdefault(name, []).append((labels, value))
+    return samples
+
+
+def _fmt(value):
+    f = float(value)
+    if f == int(f) and abs(f) < 1e15:
+        return str(int(f))
+    return repr(f)
+
+
+def render(samples):
+    out = []
+    for name, kind, help_text in FAMILIES:
+        out.append("# HELP %s %s" % (name, help_text))
+        out.append("# TYPE %s %s" % (name, kind))
+        for labels, value in sorted(samples.get(name, []), key=lambda x: sorted(x[0].items())):
+            lset = ",".join('%s="%s"' % (k, v) for k, v in sorted(labels.items()))
+            out.append("%s{%s} %s" % (name, lset, _fmt(value)))
+    return "\n".join(out) + "\n"
+
+
+class Handler(BaseHTTPRequestHandler):
+    sys_dir = "/sys/class/drm"
+    nvidia_cmd = DEFAULT_NVIDIA_CMD
+
+    def do_GET(self):  # noqa: N802 (http.server API)
+        if self.path.split("?")[0] not in ("/metrics", "/"):
+            self.send_error(404)
+            return
+        body = render(collect(self.sys_dir, self.nvidia_cmd)).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, *args):
+        pass
+
+
+def main(argv):
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--amdgpu-sysfs", default="/sys/class/drm")
+    ap.add_argument("--nvidia-cmd", default=DEFAULT_NVIDIA_CMD)
+    ap.add_argument("--serve", action="store_true", help="serve /metrics instead of one-shot")
+    ap.add_argument("--port", type=int, default=9401)
+    ap.add_argument("--bind", action="append", default=None,
+                    help="explicit bind address (repeatable); env GPU_EXPORTER_BINDS "
+                         "comma-list if unset. NEVER a wildcard.")
+    args = ap.parse_args(argv)
+
+    if not args.serve:
+        sys.stdout.write(render(collect(args.amdgpu_sysfs, args.nvidia_cmd)))
+        return 0
+
+    binds = args.bind
+    if not binds:
+        env = os.environ.get("GPU_EXPORTER_BINDS", "")
+        binds = [b.strip() for b in env.split(",") if b.strip()] or ["127.0.0.1"]
+    Handler.sys_dir = args.amdgpu_sysfs
+    Handler.nvidia_cmd = args.nvidia_cmd
+    servers = []
+    for b in binds:
+        if b in ("0.0.0.0", "::", "*"):
+            sys.stderr.write("refusing wildcard bind %s\n" % b)
+            return 2
+        servers.append(ThreadingHTTPServer((b, args.port), Handler))
+    sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds)))
+    servers[0].serve_forever()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

scripts/gpu-exporter.service

@@ -0,0 +1,35 @@
+# Systemd USER unit: gpu-exporter (both Dankclaw and Dankserver).
+#
+# Install path: ~/.config/systemd/user/gpu-exporter.service
+# Installed and enabled by deploy_gpu_exporter in scripts/deploy-remote.sh.
+#
+# Per-GPU metrics (utilization / VRAM / temperature / power) for the mixed
+# AMD + NVIDIA fleet, scraped by Prometheus. Like signal-relay it is a plain
+# user unit, not a quadlet: it runs python3 straight from the checkout (a pull
+# updates it with no copy step), a hand-written user unit can be enabled where
+# the podman v1 generator's units cannot, and with Linger=yes for dank it starts
+# at boot. It reads the host's own cgroup/drm sysfs and shells to nvidia-smi, so
+# it must run on the host — a container would need GPU/DRM passthrough for
+# nothing gained.
+#
+# BINDING. The base unit serves loopback only. deploy_gpu_exporter writes a
+# drop-in adding THIS host's LAN address (the same two-address posture
+# node-exporter and cadvisor use), so the endpoint is never a wildcard. It is
+# unauthenticated metrics on the LAN, identical to node-exporter :9100 and
+# cadvisor :8080 — Prometheus reaches same-host via host.containers.internal and
+# the Dankserver instance over the LAN from Dankclaw.
+
+[Unit]
+Description=GPU metrics exporter (amdgpu sysfs + nvidia-smi)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+# Loopback by default; the deploy drop-in appends the host LAN address.
+Environment=GPU_EXPORTER_BINDS=127.0.0.1
+ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/gpu-exporter.py --serve
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=default.target

scripts/probe-frank-archon-kv.py

@@ -147,10 +147,21 @@ def main():
     deep = corpus(160_000)
     wide = corpus(1_100_000)
 
-    # 3.28 chars/token is the measured ratio of this corpus through the
+    # The corpus's chars/token ratio is MEASURED here, not stored. A stored 3.28
-    # froggeric template (646,000 chars -> 196,919 tokens at the 2026-10-11
+    # (646,000 chars -> 196,919 tokens, 2026-10-11) had drifted to 3.20 by
-    # deploy); 98.5 % of the window leaves the response room under the ceiling.
+    # 2026-10-20 (646,299 chars -> 201,876 tokens) as the repo's docs grew, and a
-    chars = int(0.985 * maxlen * 3.28)
+    # stale constant sends leg B past --max-model-len, where the engine answers a
+    # perfectly legitimate 400 ("the request (201900 tokens) exceeds the available
+    # context size (200000)") and the leg reads as a capacity failure. One small
+    # request prices the ratio; 97 % of the window leaves the response room.
+    sample = prompt(wide, 4000, "CAL", "cal-marker")
+    cal, _, _ = post({"model": MODEL,
+                      "messages": [{"role": "user", "content": sample}],
+                      "max_tokens": 1, "temperature": 0.0,
+                      "chat_template_kwargs": {"enable_thinking": False}})
+    ratio = len(sample) / cal["usage"]["prompt_tokens"]
+    print(f"[cal] corpus ratio {ratio:.2f} chars/token")
+    chars = int(0.97 * maxlen * ratio)
     # Each marker's FIRST hyphen-separated word is unique, because show()
     # scores the hit on that word — a shared prefix would score every slot HIT
     # on any answer that says the word.
@@ -176,7 +187,7 @@ def main():
                 print(f"[B-{s}] FAILED after {time.time() - t0:.1f}s: {e!r}")
     print(f"[B] total {time.time() - t0:.1f}s")
     # D — concurrency decode: seqs streams, 512 tokens each, ~25 % depth.
-    dchars = int(0.25 * maxlen * 3.28)
+    dchars = int(0.25 * maxlen * ratio)
     dbodies = [chat(deep, dchars, f"D{i + 1}-9137", f"flint-marker-{i}", 512)
                for i in range(seqs)]
     print(f"[D] {seqs} concurrent streams x 512 tokens, "

scripts/serve-frank-archon.sh

@@ -15,7 +15,7 @@
 # log states every budget before anything is allocated (weights vs KV
 # split, the expert host pool's coverage of the worst case, the PLE pin,
 # the P2P answer per pair) — read it after every resize.
-# Deployed shape (2026-10-21, radiance 1.8.1): FOUR decode slots x 200,000 tokens over a
+# Deployed shape (2026-10-20, radiance 1.8.1): FOUR decode slots x 200,000 tokens over a
 # >= 700,000-token fp8 KV pool, 48 GiB/rank pinned expert pool, MTP depth 2, prefix cache
 # in pinned host RAM, PLE pinned in RAM. The per-card split between the expert slab and the KV
 # pool is NO LONGER an operator number — 1.3.0 removed the VRAM pair and the budget

scripts/test-changed.sh

@@ -293,6 +293,14 @@ for path in "${CHANGED[@]}"; do
             add_test tests/caddyfile.bats
             add_test tests/deployment_scoping.bats
             ;;
+        # The GPU exporter is a host process; its seam pins the collector, the
+        # user unit, the deployer function, the planner route, the Prometheus
+        # scrape jobs and the host-system GPU panels — a cross-file contract, so
+        # the whole seam rides on any of its own files.
+        scripts/gpu-exporter.py|scripts/gpu-exporter.service)
+            add_test tests/gpu_exporter.bats
+            add_test tests/deployment_scoping.bats
+            ;;
         dankclaw/udev/*)
             # Host-level udev rules are applied by manual sudo install + udevadm
             # trigger, not dispatched as a service; nothing here is bats-testable.

serverAppConfigs-docs/Infrastructure/Observability.md

@@ -2,7 +2,7 @@
 type: reference
 title: "Observability on Dankclaw"
 description: "The observability stack: what runs, how it is deployed, how it is exposed, and what is still unconfigured."
-tags: [observability, prometheus, grafana, loki, alloy, node-exporter, quadlet, dankclaw]
+tags: [observability, prometheus, grafana, loki, alloy, node-exporter, gpu-exporter, quadlet, dankclaw]
 timestamp: 2026-10-20
 ---
 
@@ -10,10 +10,10 @@ timestamp: 2026-10-20
 
 ## Summary
 
-Seven containers and one host-side process: Prometheus (metrics), Grafana
+Seven containers and two host-side processes: Prometheus (metrics), Grafana
 (dashboards), Loki (log store), Alloy (log shipper), node-exporter (host
-metrics), cadvisor (per-container metrics) and Alertmanager (alert delivery), with `signal-relay` on the host
+metrics), cadvisor (per-container metrics) and Alertmanager (alert delivery),
-carrying every firing group to Signal. All of it is Quadlet-managed,
+plus two host `python3` processes — `gpu-exporter` (per-GPU metrics) and `signal-relay` (Signal delivery). The containers are all Quadlet-managed,
 boot-persistent and loopback-only, deployed by
 `scripts/deploy-remote.sh` → `deploy_observability`. The metrics half works
 end to end; the log half is configured, healthy, and carries no data.
@@ -35,6 +35,7 @@ that proves it today.
 | Alloy | `localhost/franksdank/alloy:latest` | `127.0.0.1:12345` | `alloy.container` | [[Alloy]] |
 | node-exporter | `docker.io/prom/node-exporter:latest` | host netns: `127.0.0.1:9100` + `192.168.0.143:9100` | `node-exporter.container` | [[Node-Exporter]] |
 | cadvisor | `gcr.io/cadvisor/cadvisor:v0.49.1` | bridge publish `127.0.0.1:8080` + `192.168.0.143:8080` | `cadvisor.container` | — |
+| GPU exporter | host `python3`, not a container | `127.0.0.1:9401` + host LAN `:9401` | `scripts/gpu-exporter.service` (user unit) | — |
 | Signal relay | host `python3`, not a container | `192.168.0.143:18093` | `scripts/signal-relay.service` (user unit) | [[Alertmanager]] |
 
 Every unit lives in `dankclaw/observability/quadlet/`, carries
@@ -45,10 +46,11 @@ linger is on for `dank`). `signal-relay.service` reports `enabled` + `active`
 the same day, and `homepage-status.timer` — the root unit that probes all of
 these on loopback — is `enabled` + `active` (`deploy-remote.sh:319-321` copies
 the probe registry `shared/homepage/services.json` to `/srv/homepage`).
-Dankserver runs two members of this family, node-exporter and cadvisor, as
+Dankserver runs the same family: node-exporter and cadvisor as its own quadlets
-its own units (`dankserver/observability/quadlet/{node-exporter,cadvisor}.container`,
+(`dankserver/observability/quadlet/{node-exporter,cadvisor}.container`,
 `generated` + `active`; node-exporter binds `127.0.0.1:9100` + `192.168.0.235:9100`,
-cadvisor publishes `127.0.0.1:8080` + `192.168.0.235:8080`).
+cadvisor publishes `127.0.0.1:8080` + `192.168.0.235:8080`), plus the host-process
+`gpu-exporter` (a user unit installed the same way as `signal-relay`).
 
 ## Exposure
 
@@ -117,6 +119,18 @@ verify it through the APIs.
   `container` label. Network and filesystem are NOT exported for these containers
   (no netns/classid attribution under the host-PID layout), so the dashboard
   graphs only CPU, memory and OOM and does not fake the rest.
+* **Per-GPU metrics: the `host-system` GPU panels (added 2026-10-20; live after
+  `deploy.sh <host> --service gpu-exporter`).** `scripts/gpu-exporter.py` is a
+  host-process user unit on BOTH hosts — not a quadlet, because it reads the host's
+  own DRM sysfs and shells to `nvidia-smi`, neither of which a container can do
+  without GPU passthrough. It reads each AMD card's `gpu_busy_percent` /
+  `mem_info_vram_*` / `hwmon temp1_input` and one `nvidia-smi --query-gpu` CSV, and
+  emits unified `gpu_*` families (utilization %, VRAM used/total bytes, temperature
+  °C, power W) labelled `gpu` / `vendor` / `model`, with `host` added by the scrape
+  job. Two laws: **absent means absent, never a zero** — the R9700s expose no sysfs
+  `power1_average`, so they carry no `gpu_power_watts` line at all (only NVIDIA and
+  the Dankclaw 8060S iGPU report power); and it binds loopback + the host LAN, like
+  node-exporter, never the wildcard. The dashboard graphs util/VRAM/VRAM-%/temp/power.
 * **Logs: fixed in code, awaiting deploy.** The pipeline shipped nothing because
   a rootless Alloy cannot read the `0640` journal files, and the container-log
   globs point at a root-storage layout that does not exist here. The fix is built
@@ -135,6 +149,7 @@ verify it through the APIs.
 - `dankclaw/observability/quadlet/{prometheus,alertmanager,grafana,loki,alloy,node-exporter,cadvisor}.container`
 - `scripts/deploy-remote.sh:219-300` (copiers + relay), `:521-536` (health waits), `:477-486` (grafana env seed)
 - `scripts/signal-relay.py:26-49`, `scripts/signal-relay.service`
+- `scripts/gpu-exporter.py`, `scripts/gpu-exporter.service` (per-GPU host exporter)
 - `shared/prometheus/prometheus.yml`, `shared/prometheus/alertmanager.yml`, `shared/prometheus/rules/llm-fleet.yml`
 - `shared/grafana/provisioning/`, `shared/loki/loki-config.yaml`, `shared/alloy/config.alloy`
 - `tests/observability_quadlet.bats` (boot persistence, no wildcard publish, probe addresses, delivery contract)

serverAppConfigs-docs/LLM/frank-archon.md

@@ -88,25 +88,30 @@ on stop. Boot budget 1800 s (the publisher's compose allows a 15-minute `start_p
 | Knob (env name, `FRANK_ARCHON_` dropped) | Live value |
 |---|---|
 | TP / wire | `TP=2`, `TP_WIRE=wht6` — libr4d one-shot all-reduce, not RCCL; `min-kb` stays the engine default |
-| Slots / window | `MAX_SEQS=2`, `MAX_MODEL_LEN=200000` (native 262,144; longer is refused, never truncated) — this pair is the **cache bound**, not a KV allocation |
+| Slots / window | `MAX_SEQS=4`, `MAX_MODEL_LEN=200000` (native 262,144; longer is refused, never truncated) — this pair is the **cache bound**: the resolver sizes the KV carve against `slots × window`, and no slot count funds itself (Laws) |
 | KV dtype | `KV_DTYPE=fp8` — the only dtype that funds the slot count (Laws) |
 | VRAM split | **derived at boot** from `GPU_HEADROOM_MIB=96` + the cache bound + `HOST_POOL_MIB`; naming `VRAM_WEIGHTS_MIB`/`VRAM_KV_MIB` is a boot refusal (removed at 1.3.0) |
-| Expert tier | `PLACEMENT=expert_tiered`, `HOST_POOL_MIB=44928` **per rank** |
+| Expert tier | `PLACEMENT=expert_tiered`, `HOST_POOL_MIB=49152` **per rank** — the pinned host pool is also the **KV lever**: the expert floor is `plane + 2 units − host room`, so every MiB pinned there lowers what the experts claim on the card one-for-one and the cache takes it (Laws) |
 | PLE | `NGRAM_PLACEMENT=ram` (47.7 GiB pinned; refuses boot on a short memlock) |
 | Spec / batch | `SPEC_TOKENS=2` — MTP depth 2, the `.rad` carries the model's own `mtp.*` head (`mtp.fc_hidden`/`fc_embedding`/`draft_head`); `MAX_BATCHED_TOKENS=2048` (MNBT — 2560 collapses decode, Laws) |
 | Prefix cache | `PREFIX_CACHE_MIB=8192`, pinned **host tier only** — the 64 GiB disk tier is gone, so an evicted prefix is re-prefilled; `STATE_DIR=/home/dank/radiance-state` now exists only to hold the rad-tune cache |
 | Template | `TEMPLATE=scripts/qwen38-froggeric-v22.5.jinja` (28,234 B) via `--override-chat-template` — froggeric's fixed Qwen3.8 template (the repo root's `chat_template.jinja`, vendored); the checkpoint's own template is baked into the `.rad` by rad-convert and this overrides it, and radiance derives the reasoning and tool-call reply format from it |
 | Step sharing | `BUSY_PREFILL_CHUNK=auto` — prefill-speed first: it reserves 3 tokens per waiting decoder and ⅛ of the budget per waiting request, and does **not** shrink the chunk for decoders alone · `PREFILL_INTERLEAVE=1` — the decoder dial, **armed 2026-10-20**: one decode-only step after each chunk step taken beside decoders; meter `engine_steps_held_total`, price is the prefilling prompt's TTFT |
-| TTM gate | `TTM_MIN_MIB=150000`, `TTM_TARGET_MIB=204800` |
+| TTM gate | `TTM_MIN_MIB=162000`, `TTM_TARGET_MIB=204800` |
 | Boot | `RAD_MIN_MIB=110000` deployer floor; the listener binds only after the full load (health ~100 s, budget 1800 s) |
 
-Boot ledger for the live shape (2026-10-19 11:10, pin 1.8.1, `MAX_SEQS=2`, `HOST_POOL_MIB=44928`):
+Boot ledger for the live shape (2026-10-20, pin 1.8.1, `MAX_SEQS=4`, `HOST_POOL_MIB=49152`):
-the resolver prints `elastic 26.73 GiB` → **experts 23.73 GiB (89 %) / kv cache 2.76 GiB (10 %)**,
+the resolver prints `elastic 26.73 GiB` → **experts 20.72 GiB (78 %) / kv cache 5.64 GiB (21 %)**,
-`vram_weights 27.31 GiB`, `host_pool 44.47 GiB`. `kv_attn` (13 layers, 4-token blocks) pages
+`prefill staging 373.65 MiB` out of the cache, `vram_weights 24.30 GiB`, `host_pool 48.59 GiB`.
-**26.00 KiB** → 91,301 blocks = **365,204 tokens = 1.8 full-length sessions = 91 % of the
+The referee line: `kv: the cache can grow to 5.64 GiB a card — 747,492 tokens of context, 3.7
-2 × 200,000 worst case** (the resolver never funds 100 %, see Laws); `mover` lends 2,146 slots
+sessions at 200,000 — with every block it lends the experts taken back; 440.04 MiB of that is
-(2.54 GiB) out of the cache to the expert slab; sampler **9 rows**, template 28,234 B, vision live
+per-sequence state, and up to 5.20 GiB is lent to the experts while no request holds it`.
-(≤ 4,194,304 px, 16,384 patches/pass).
+`--checkpoint-slots 40` is auto (`max_num_seqs × 10` slots of 111.98 MiB = 4,479 MiB of pinned
+host), `/health` reports `queue_cap 32` (`max(8, seqs × 8)`), sampler **12 rows**
+(= `seqs × (1 + speculative depth)`), 4 hardware queues a card, template 28,234 B, vision live
+(≤ 4,194,304 px, 16,384 patches/pass), P2P confirmed (`the 2 cards reach each other's memory
+directly`). Experts living in the pinned host pool: **8.99 GiB a rank** (was 5.99) — the price of
+the bigger cache, paid over PCIe when those experts are hit.
 
 The boot **ledger**, not this arithmetic, is the referee: radiance states every budget before
 anything is allocated (weights vs KV, the host pool's coverage of the worst case, the PLE pin,
@@ -145,6 +150,7 @@ marked "re-fetch" is gone from `/home/dank/models/`, which today holds only `Sti
 
 | Window | Payload | Runtime | Shape | Replaced because | Rollback |
 |---|---|---|---|---|---|
+| 2026-10-19 → 10-20 | same `qwen3.8-next-flash-fp8-iq4r-moe.rad` (payload unchanged) | radiance **1.8.1**, `sha256:d590e300…` (same digest) | 2 × 200,000 fp8 KV — resolver cache 2.76 GiB = 365,204 tokens (91 % backing); host pool 44,928 MiB a rank, `TTM_MIN_MIB=150000`, `queue_cap 16`, sampler 9 rows; `--prefill-interleave 1` armed on 10-20 | Frank ordered **four** concurrent conversations over a ≥ 700,000-token pool; the resolver funds that only by lowering the expert floor with more pinned host room (Laws) | revert `1bb6399` and redeploy `scripts/deploy.sh dankserver --service frank-archon` — trio back to `MAX_SEQS=2` / `HOST_POOL_MIB=44928` / `TTM_MIN_MIB=150000` |
 | 2026-10-15 → 10-19 | same `qwen3.8-next-flash-fp8-iq4r-moe.rad` (payload unchanged) | radiance **1.0.13**, `sha256:4c3f4845…` | 3 × 200,000 fp8 KV bought with the **explicit** pair `--vram-weights-mib 25600 --vram-kv-mib 5120`; host pool 44,928 → 45,568 MiB a rank; prefix cache 8 GiB host + 64 GiB disk | decode turned to sludge beside any long prompt — step contention, 18.7 ms decode-only steps against 317 ms steps carrying a prefill chunk (Laws) — and the only two knobs that fix it, `--busy-prefill-chunk` and `--prefill-interleave`, exist only after this pin | `FRANK_ARCHON_IMAGE` back to `sha256:4c3f4845…` **plus a trio rewrite**: restore the pair and the host+disk prefix knobs, delete `BUSY_PREFILL_CHUNK`/`PREFILL_INTERLEAVE`. A digest swap alone is a boot refusal (Runtime) |
 | 2026-10-08 → 10-15 | in-house `qwen4exp-swift15-w4dn8-i8-hc8m.rad`, 116,416 MiB (Swift1.5 abliterated BF16, GPTQ w4 experts + 24 bf16 protected, KLD 0.0131 / top-1 97.92 %) | radiance 1.0.13, same digest | 3 × 200,000 fp8 KV, slab 25,600 / pool 5,120 — the shape the live file still carries | the engine author's own container is the supported path and his quality ceiling; the finetune's refusal-bending left the endpoint | `.rad` purged 2026-10-17 (trash-staged then emptied, Frank's approval). Re-download `frank8465/Swift-1.5-Qwen3.8-Flash-Next-GPTQ-W4A8-Radiance-AMD-R9700` → `qwen4exp-w4nl64-i8-hc8m.rad`, 122,071,973,888 B, LFS sha256 `09dd9306…`, via the iq4r fetch script's hfget/verify pattern retargeted at that repo; point `MODEL_FILE` at it, redeploy. Re-**convert** is dead: the BF16 source was `rm`'d 2026-10-15 (`kld-ref`, grams, corpora and `/home/dank/logs/swift15_*.log` survive) |
 | 2026-10-05 → 10-08 | `qwen4exp-fp8-native.rad`, 177,109 MiB (173.0 GiB) — native modelopt block-FP8, no requantisation | radiance 1.0.7 line, same digest | — | native FP8 experts (56.4 GiB/rank) ate a 40 GiB pool and left KV at 3,543 MiB = 386,904 tokens, **1.93 sessions** — context economics, not quality; and the host pool could not be trimmed to pay for it (36,864 refused the boot) | rebuild-only: `scripts/download-frank-archon.sh` (official `Qwen/Qwen3.8-Flash-Next-FP8`, 131 shards, 172.8 GB, ungated) + `scripts/build-frank-archon-rad.sh` (no-recipe convert under the descriptor law); raw **and** `.rad` were `rm`'d 2026-10-15 (Frank's call, to free the volume for the iq4r download) |
@@ -211,27 +217,44 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
   never `--kv-cache-size-mib`; with the disk tier engaged the engine also forced ~9.6 GiB of weight
   buffers + arena onto the weights side (~29.6 GiB/card effective). **Neither flag exists at 1.8.1:**
   the same ceiling is enforced by the resolver, and passing the pair is a boot refusal.
-- **At 1.8.1 the split belongs to the resolver, and it funds ~90 % of the worst case, never 100 %.**
+- **At 1.8.1 the split belongs to the resolver, and the source fixes its order.**
-  The engine takes `--gpu-headroom-mib`, the cache bound (`--max-model-len` × `--max-num-seqs`) and
+  `core/mem/vram_budget.cpp` claims the card bottom-up (`capacity → already held → drift → arena →
-  `--host-pool-mib`, prints `claimable → elastic → experts / kv cache` before allocating anything, and
+  staging → queues → prefetch → headroom = claimable`), takes the **static weights off the top as a
-  sizes the expert plane from the model's own weight footprint plus slack — not from a number typed by
+  floor, not a share**, and divides the `elastic` remainder in three lines (146-156):
-  hand. Three boots of evidence:
 
-  | shape | demand | cache given | backing |
+  ```cpp
+  exp = std::min(b.expert_floor, b.elastic);
+  kv  = std::min(kv_cap, std::max(b.elastic - exp, kv_one));
+  exp = std::min(std::max(b.expert_ceiling, b.expert_floor), b.elastic - kv);
+  ```
+
+  `kv_cap` is `KVManager::ceiling()` = `state + Σ_paged max_seqs × ceil(max_ctx / block) × bytes`
+  (`core/mem/kv.cpp:266`), so `--max-num-seqs` and `--max-model-len` set the cache's **demand**, never
+  its funding. The funding is `elastic − expert_floor`, and the floor is
+  `min(plane + spare, plane + spare + 2·unit_max − host_room)` (`core/place/planner.cpp:100-104`, with
+  `host_room = host_pool − file_stage_reserve` at `core/engine_bringup.cpp:1418-1422`). **That is the
+  cache's lever: every MiB added to `--host-pool-mib` drops the expert floor one-for-one and the cache
+  takes it** — minus `stage`, the prefill stager's ring (two layers of the experts left off the card),
+  which is charged back out of the cache whenever the cache lends (`vram_budget.cpp:193-203`). Boots of
+  evidence, all on 1.8.1:
+
+  | shape | demand | cache given | engine's `backs %` |
   |---|---|---|---|
-  | 3 × 200,000, host pool 44,928 | 600,000 tok | 4.20 GiB | 93 % (2.8 sessions) |
+  | 3 × 200,000, host pool 44,928 | 600,000 tok | 4.20 GiB | 93 % |
   | 3 × 200,000, host pool 45,568 | 600,000 tok | **4.20 GiB — byte-identical** | 93 % |
-  | 2 × 200,000, host pool 44,928 (live) | 400,000 tok | 2.76 GiB = 365,204 tok | 91 % (1.8 sessions) |
+  | 2 × 200,000, host pool 44,928 | 400,000 tok | 2.76 GiB | 91 % |
+  | 4 × 200,000, host pool 49,152 (live) | 800,000 tok | **5.64 GiB = 747,492 tok** | 93 % |
 
-  So `--max-num-seqs` is a **demand signal** for the resolver and an admission cap for the scheduler —
+  The 45,568 row is the ceiling case, not a mystery: at 3 slots the cache already held its whole
-  never a KV allocation — and no slot count fully funds the worst case. Growing host RAM by less than a
+  `ceiling`, so extra host room could not grow it and shows up as `unclaimed`. **The referee is the
-  slab step (640 MiB) moved nothing: the expert plane's claim decides what is left for KV, so cache is
+  ledger line `kv: the pool backs N % … the cache can grow to … N tokens`, never arithmetic.** To buy
-  grown through slots × window or the dtype, and experts are grown by not leaving elastic pool idle.
+  window: raise `--host-pool-mib` (cost: pinned host RAM, and experts evicted to host cross PCIe —
-  `--host-pool-mib` is still load-bearing the other way — it is where the expert rows the cards cannot
+  `experts-from-host` was 5.99 GiB a rank at 44,928 and is 8.99 at 49,152) or cut the per-token cost
-  hold get pinned, and under-sizing it has always been a boot failure (36,864 refused a boot in the fp8
+  (`fp8` KV). Raising slots raises demand **and** costs 10 × 111.98 MiB of pinned checkpoint-store RAM
-  era). The two-slot boot's own behavioural check: three concurrent conversations all answer `200` —
+  per slot. Under-sizing the pool the other way has always been a boot failure (36,864 refused a boot in
-  the third **queues** (`queue_cap: 16`), it is not refused; the 400 class is a single request over
+  the fp8 era: the host cannot hold what the cards will not). Behavioural check on the live shape:
-  `--max-model-len`, or three concurrent full-height conversations against a 1.8-session cache.
+  `/health` reports `queue_cap 32`; a request beyond the slot count queues rather than being refused,
+  and the 400 class is a single request over `--max-model-len`.
 - **Idle KV VRAM is the expert cache's spillover — the price is paid twice.** `mover: 2275
   slots can be lent out of the KV cache (2.69 GiB)`, `arena: lends 740 MiB while steps are <=
   256 tokens`; the lend grows with the pool (2.69 GiB from 3.46, 6.08 from 6.912, 4.22 in the pair era)
@@ -366,17 +389,18 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
   `--checkpoint-interval` does **not** buy the deep case: the coarsest ladder scale is the largest
   `interval × 2^k` under the depth, so the deepest gap stays ≈ half the prompt (129,024 tokens at
   interval 2048 against 130,816 at 256) while every cut multiplies the snapshot writes.
-- **Slots cost KV, rows, and window — never speed.** `--max-num-seqs` carries no radiance constraint: it
+- **Slots cost KV, host RAM, rows, and pin — never speed.** `--max-num-seqs` carries no radiance
-  adds ≈110 MiB of recurrent state per slot (the live `kv_full_backing` arithmetic prices them exactly),
+  constraint: it adds ≈110 MiB of recurrent state per slot, 10 × 111.98 MiB of pinned checkpoint-store
-  declares the fused TP all-reduce over `seqs × (1 + spec)` rows, and `wht6` is bit-exact up to 25 rows
+  RAM per slot (`--checkpoint-slots auto`), and declares the fused TP all-reduce over
-  (single-sequence decode and any step of ≤ 25 rows is exact; only busier steps are lossy) — 2 × 3 = 6
+  `seqs × (1 + spec)` rows — `wht6` is bit-exact up to 25 rows, and the live shape is 4 × 3 = **12**, so
-  today. Measured 2026-10-06: seqs 2 and seqs 3 are statistically tied (30.9-33.1 t/s, inside the ±3 %
+  single-sequence decode stays exact and so does any step up to that many rows. Raising it also raises
-  floor). Measured 2026-10-19 at 1.8.1: the third slot's real price is **KV, not rows** — dropping it
+  the cache's *demand* (`ceiling = slots × window`), which the resolver funds from `elastic −
-  moved the resolver's cache from 4.20 GiB to 2.76 GiB and the backing from 2.8 to 1.8 sessions, while
+  expert_floor` — so a slot count without matching host room is paid for by expert residency. Measured
-  the ≈4 GiB of host RAM it freed came back unbudgeted, landing in elastic pool that turned into a bigger
+  2026-10-06: seqs 2 and 3 statistically tied (30.9-33.1 t/s, inside the ±3 % floor). Measured
-  mover lend instead of cache (resolver law above). The old "seqs 2 cannot boot" law belonged to the
+  2026-10-19 → 10-20 at 1.8.1: 2 → 4 slots cost **+4,224 MiB of pinned host pool** and **−3.01 GiB of
-  retired tcclaviger fork's `torch.compile` shape specialization, and radiance booted seqs 2 on its first
+  expert residency a card** and bought 2.76 → 5.64 GiB of cache (365,204 → 747,492 tokens). The old
-  fp8-native start.
+  "seqs 2 cannot boot" law belonged to the retired tcclaviger fork's `torch.compile` shape
+  specialization, and radiance booted seqs 2 on its first fp8-native start.
 - **A dequantize-to-compute path bills host RAM twice.** On the EMULATION backend an offload request
   of **118 GiB** landed as **~238 GiB** of real host RAM — the dequantized copy rides on top of the
   pinned weights, and the doubling is invisible in the weight count. Read the engine log for
@@ -570,13 +594,13 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
   — at 64 tokens you get empty `content` and the colour only inside `reasoning_content`.
 - **Capacity behaviour:** a prompt over `--max-model-len` is refused, never truncated; a request
   longer than the KV pool's token count is accepted then refused as too large to fit; a request
-  beyond `--max-num-seqs` is held by the HTTP port (`queue_cap`, default 8 × slots = 16 here) and
+  beyond `--max-num-seqs` is held by the HTTP port (`queue_cap` = `max(8, 8 × slots)` — **32** here)
-  answered 503 only past that, never paged or squeezed. Because the cache funds **1.8** full-height
+  and answered 503 only past that, never paged or squeezed. The cache funds **3.7** full-height
-  sessions, admitting a second near-full-height conversation can **preempt a running sequence from
+  sessions, so four conversations can be resident but a fourth near-full-height one can still
-  the back** to free its blocks (`core/sched/scheduler.cpp:1291-1322`); the victim is recomputed, so
+  **preempt a running sequence from the back** to free its blocks
-  its session halts and then re-prefills. Meter: `vllm:num_preemptions_total` (264 in the current
+  (`core/sched/scheduler.cpp:1291-1322`); the victim is recomputed, so its session halts and then
-  boot, all in prefill-coincident bursts — 108 inside 105 s); `/stats preemptions` is the cumulative
+  re-prefills. Meter: `vllm:num_preemptions_total` (cumulative for the boot, `/stats preemptions`);
-  count and `radiance:num_requests_preempted` only the instant depth.
+  `radiance:num_requests_preempted` is only the instant depth.
 - **Observability:** `/metrics` carries vLLM-compatible names plus `radiance:*`, scraped by the
   Dankclaw Prometheus job at `192.168.0.235:7770` (15 s interval) over plain LAN routing (the
   netavark published-port quirk is same-host only). **Latency lives under `vllm:`, cadence under

shared/grafana/provisioning/dashboards/host-system.json

@@ -18,6 +18,16 @@
         "includeAll": true,
         "multi": true,
         "current": { "text": "All", "value": "$__all" }
+      },
+      {
+        "name": "gpuhost",
+        "label": "gpu host",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(gpu_utilization_percent, host)", "refId": "A" },
+        "includeAll": true,
+        "multi": true,
+        "current": { "text": "All", "value": "$__all" }
       }
     ]
   },
@@ -139,6 +149,56 @@
         { "expr": "sum by (job,device) (rate(node_network_receive_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} RX err", "refId": "A" },
         { "expr": "sum by (job,device) (rate(node_network_transmit_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} TX err", "refId": "B" }
       ]
+    },
+    {
+      "id": 13, "type": "timeseries", "title": "GPU busy %",
+      "description": "Per-GPU utilization from gpu-exporter (amdgpu gpu_busy_percent / nvidia-smi). The picker above selects the host; each series is one card.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
+      "targets": [
+        { "expr": "gpu_utilization_percent{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
+      ]
+    },
+    {
+      "id": 14, "type": "timeseries", "title": "GPU VRAM used",
+      "description": "Device memory in use. On the R9700s (frank-archon) and the 3090 (frank-zealot) this is the loaded model's footprint.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "min": 0 }, "overrides": [] },
+      "targets": [
+        { "expr": "gpu_memory_used_bytes{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
+      ]
+    },
+    {
+      "id": 15, "type": "timeseries", "title": "GPU VRAM used %",
+      "description": "Used / total device memory. Near 100% is the pre-OOM wall.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
+      "targets": [
+        { "expr": "100 * gpu_memory_used_bytes{host=~\"$gpuhost\"} / gpu_memory_total_bytes{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
+      ]
+    },
+    {
+      "id": 16, "type": "timeseries", "title": "GPU temperature",
+      "description": "Package temperature. AMD edge temp (hwmon temp1_input); NVIDIA gpu temp.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "celsius", "min": 0 }, "overrides": [] },
+      "targets": [
+        { "expr": "gpu_temperature_celsius{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
+      ]
+    },
+    {
+      "id": 17, "type": "timeseries", "title": "GPU power draw",
+      "description": "Board power draw. Emitted only where readable: NVIDIA and the Dankclaw iGPU. The R9700s have no sysfs power sensor, so they are simply absent here — never a fabricated 0.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "watt", "min": 0 }, "overrides": [] },
+      "targets": [
+        { "expr": "gpu_power_watts{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
+      ]
     }
   ]
 }

shared/prometheus/prometheus.yml

@@ -191,3 +191,25 @@ scrape_configs:
         action: keep
       - regex: 'id|name|image|container_label.*|machine_id|boot_id'
         action: labeldrop
+
+  # Per-GPU metrics (gpu_utilization_percent / gpu_memory_used_bytes /
+  # gpu_memory_total_bytes / gpu_temperature_celsius / gpu_power_watts) from the
+  # host-process gpu-exporter (scripts/gpu-exporter.py). One job per host, same
+  # reach rules as node/cadvisor: Dankclaw is scraped at the podman gateway
+  # because the netavark hairpin quirk makes Prometheus fail to dial its own
+  # host LAN IP; Dankserver is plain LAN. The gpu label (PCI BDF on AMD, index on
+  # NVIDIA) + vendor/model come from the exporter, so the GPU row on the
+  # host-system dashboard filters by host. Power is emitted only where readable
+  # (NVIDIA + the Dankclaw iGPU); the R9700s have no sysfs power, so they simply
+  # have no gpu_power_watts series — no fabricated zero.
+  - job_name: "gpu-dankclaw"
+    static_configs:
+      - targets: ["host.containers.internal:9401"]
+        labels:
+          host: dankclaw
+
+  - job_name: "gpu-dankserver"
+    static_configs:
+      - targets: ["192.168.0.235:9401"]
+        labels:
+          host: dankserver

tests/gpu_exporter.bats

@@ -0,0 +1,197 @@
+#!/usr/bin/env bats
+# Seam: the GPU metrics exporter's black-box contract.
+#
+# The exporter reads amdgpu sysfs (one card per /sys/class/drm/cardN that has a
+# device/gpu_busy_percent file) and, on the box that has a working NVIDIA stack,
+# one `nvidia-smi --query-gpu` CSV, and emits the unified `gpu_*` Prometheus
+# families the host-system dashboard graphs. Both sources are injectable so the
+# whole thing is testable with fixtures on a GPU-less dev box.
+#
+# Only the inputs a consumer sees are pinned: the metric families, their label
+# set, and that absent hardware yields ABSENT series (never a fabricated 0 — a
+# wrong number on a dashboard gets believed). The R9700s expose no sysfs
+# power1_average, so those cards legitimately carry no gpu_power_watts line.
+
+load helpers
+
+EXPORTER="$REPO_ROOT/scripts/gpu-exporter.py"
+
+# Build a fixture amdgpu sysfs tree. $BATS_TEST_TMPDIR/sys
+mk_amd() { # gpu busy vram_used vram_total power_uw_or_blank temp_mc
+    local card=$1 busy=$2 used=$3 total=$4 power=$5 temp=$6
+    local d="$SYS/card$card/device"
+    mkdir -p "$d/hwmon/hwmon9"
+    printf '%s\n' "$busy" > "$d/gpu_busy_percent"
+    printf '%s\n' "$used" > "$d/mem_info_vram_used"
+    printf '%s\n' "$total" > "$d/mem_info_vram_total"
+    printf '%s\n' "$power" > "$d/power1_average"          # blank => absent on R9700
+    [ -n "$temp" ] && printf '%s\n' "$temp" > "$d/hwmon/hwmon9/temp1_input"
+    printf 'DRIVER=amdgpu\nPCI_ID=1002:7551\nPCI_SLOT_NAME=0000:03:00.0\n' > "$d/uevent"
+}
+
+setup() {
+    SYS="$BATS_TEST_TMPDIR/sys"
+    mkdir -p "$SYS"
+}
+
+@test "exporter exists and compiles" {
+    [ -f "$EXPORTER" ]
+    python3 -m py_compile "$EXPORTER"
+}
+
+@test "amd card with full sysfs yields util/vram/temp/power with the right labels and units" {
+    mk_amd 1 42 1073741824 34359738368 250000000 71000   # 250 W, 71 C
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
+    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+    # util is a raw percent, VRAM verbatim bytes, temp milli->C, power microW->W.
+    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="amd"[^}]*\} 42$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*\} 1073741824$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_memory_total_bytes\{[^}]*\} 34359738368$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_temperature_celsius\{[^}]*\} 71$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_power_watts\{[^}]*\} 250$' || { echo "$output"; return 1; }
+    # identity labels: the PCI slot is a stable unique id, vendor marks the family.
+    echo "$output" | grep -Eq 'gpu_utilization_percent\{.*gpu="0000:03:00\.0"' || { echo "$output"; return 1; }
+}
+
+@test "card with an empty power file and no hwmon yields util/vram but omits power and temp" {
+    mk_amd 1 99 34108338176 34208743424 "" ""           # the R9700 shape
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
+    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*\} 99$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*\} 34108338176$'
+    # Absent means ABSENT, not a zero the dashboard would plot as a real reading.
+    ! echo "$output" | grep -Eq '^gpu_power_watts\{' || { echo "fabricated power line: $output"; return 1; }
+    ! echo "$output" | grep -Eq '^gpu_temperature_celsius\{' || { echo "fabricated temp line: $output"; return 1; }
+}
+
+@test "nvidia csv yields util/vram/temp/power tagged vendor=nvidia" {
+    nvidia="$BATS_TEST_TMPDIR/nvidia-smi"
+    cat > "$nvidia" <<'STUB'
+echo '0, NVIDIA GeForce RTX 3090, 93, 23004, 24576, 76, 397.40'
+STUB
+    chmod +x "$nvidia"
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd "$nvidia"
+    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="nvidia"[^}]*\} 93$' || { echo "$output"; return 1; }
+    # memory.used MiB -> bytes (23004 * 1048576)
+    echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*vendor="nvidia"[^}]*\} 24121442304$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_temperature_celsius\{[^}]*vendor="nvidia"[^}]*\} 76$'
+    echo "$output" | grep -Eq 'gpu_power_watts\{[^}]*vendor="nvidia"[^}]*\} 397\.4$'
+    echo "$output" | grep -Eq 'model="NVIDIA GeForce RTX 3090"'
+}
+
+@test "a failing nvidia command is tolerated: amd still emits and exit is 0" {
+    mk_amd 1 50 1000000 2000000 "" 40000
+    failing="$BATS_TEST_TMPDIR/nvidia-fail"
+    printf '#!/bin/sh\nexit 1\n' > "$failing"; chmod +x "$failing"
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd "$failing"
+    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="amd"[^}]*\} 50$'
+    ! echo "$output" | grep -q 'vendor="nvidia"'
+}
+
+@test "no GPUs at all yields only HELP/TYPE headers and exits 0" {
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
+    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+    # families are declared even when empty, so the dashboard/queries stay valid.
+    echo "$output" | grep -q '# TYPE gpu_utilization_percent gauge'
+    echo "$output" | grep -q '# TYPE gpu_memory_used_bytes gauge'
+    ! echo "$output" | grep -Eq '^gpu_utilization_percent\{'
+}
+
+@test "output is valid Prometheus text" {
+    mk_amd 1 42 1000 2000 "" 50000
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
+    [ "$status" -eq 0 ]
+    # every non-comment line is "<name>{labels} value" or "<name> value"
+    printf '%s\n' "$output" | python3 -c '
+import sys, re
+pat = re.compile(r"^[a-z_]+(\{[^}]*\})? [-+0-9.eE+]+$")
+for i, l in enumerate(sys.stdin):
+    l = l.rstrip("\n")
+    if not l or l.startswith("#"):
+        continue
+    assert pat.match(l), "malformed line %d: %r" % (i, l)
+print("prometheus text ok")
+'
+}
+
+# ---- deployment + wiring contracts -------------------------------------
+# The exporter is a HOST-PROCESS user unit (not a quadlet): it must read the
+# host's own DRM sysfs and shell to nvidia-smi, which a container could not do
+# without GPU passthrough. These tests pin the cross-file wiring a reader
+# cannot eyeball — a lone change on one side leaves the metric silently absent.
+
+@test "the base unit runs the exporter from the checkout and never binds a wildcard" {
+    local u="$REPO_ROOT/scripts/gpu-exporter.service"
+    [ -f "$u" ]
+    grep -q '^ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/gpu-exporter.py --serve$' "$u" \
+        || { echo "unit does not run the exporter --serve from the checkout"; return 1; }
+    grep -q '^Restart=always$' "$u"
+    grep -q '^WantedBy=default.target$' "$u"
+    # Base unit binds loopback only; the per-host LAN address is added by a
+    # deploy drop-in, so a checked-in unit is never a wildcard.
+    grep -q '^Environment=GPU_EXPORTER_BINDS=127.0.0.1$' "$u"
+    ! grep -qE '0\.0\.0\.0' "$u"
+}
+
+@test "the deployer installs, drops in the LAN bind, and health-gates gpu-exporter" {
+    local handler="$REPO_ROOT/scripts/deploy-remote.sh"
+    grep -q '^deploy_gpu_exporter() {' "$handler"
+    # Per-host LAN from $TARGET (never a hardcoded host, never a wildcard).
+    grep -q 'dankclaw) lan=192.168.0.143' "$handler"
+    grep -q 'dankserver) lan=192.168.0.235' "$handler"
+    # Drop-in binds loopback + this host's LAN as one comma list.
+    grep -q "GPU_EXPORTER_BINDS=127.0.0.1,%s" "$handler"
+    grep -q 'systemctl --user enable gpu-exporter.service' "$handler"
+    grep -q 'systemctl --user restart gpu-exporter.service' "$handler"
+    # Health gate waits on the loopback metrics endpoint.
+    grep -q 'http://127.0.0.1:9401/metrics' "$handler"
+    # Dispatched from the central case.
+    grep -q 'gpu-exporter) deploy_gpu_exporter ;;' "$handler"
+    # Allowlisted on BOTH hosts.
+    grep -q 'dankclaw:gpu-exporter' "$handler"
+    grep -q 'dankserver:gpu-exporter' "$handler"
+}
+
+@test "the planner selects gpu-exporter for its files on each target" {
+    local t f
+    for t in dankclaw dankserver; do
+        for f in scripts/gpu-exporter.py scripts/gpu-exporter.service; do
+            run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target "$t" --paths "$f"
+            [ "$status" -eq 0 ]
+            [ "$output" = "gpu-exporter" ] || { echo "$t/$f -> '$output'"; return 1; }
+        done
+    done
+}
+
+@test "Prometheus scrapes gpu-exporter where node/cadvisor are reachable on each host" {
+    # Cross-file contract, same posture as node-exporter: the local instance is
+    # reached via the podman gateway (this Prometheus is a bridge container and
+    # a host-process bind is refused at the host LAN IP); the Dankserver
+    # instance is a different machine, so plain LAN routing reaches it. If either
+    # address drifts from the exporter's bind, GPU panels go blank with no error.
+    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
+    grep -q 'job_name: "gpu-dankclaw"' "$cfg" || { echo "no gpu-dankclaw job"; return 1; }
+    grep -q 'job_name: "gpu-dankserver"' "$cfg" || { echo "no gpu-dankserver job"; return 1; }
+    grep -q '"host.containers.internal:9401"' "$cfg" \
+        || { echo "gpu-dankclaw does not scrape the podman gateway"; return 1; }
+    grep -q '"192.168.0.235:9401"' "$cfg" \
+        || { echo "gpu-dankserver does not scrape the LAN address"; return 1; }
+    # A vhost target would be red forever (Prometheus cannot solve Anubis).
+    ! grep -qE '"[^"]*franksdank[^"]*:9401"' "$cfg"
+}
+
+@test "the host-system dashboard graphs the gpu_* families the exporter emits" {
+    local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/host-system.json"
+    [ -f "$dash" ]
+    python3 -c "
+import json
+d = json.load(open('$dash'))
+exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
+for want in ('gpu_utilization_percent', 'gpu_memory_used_bytes',
+             'gpu_temperature_celsius', 'gpu_power_watts'):
+    assert any(want in e for e in exprs), ('missing', want, exprs)
+print('gpu dashboard panels ok')
+"
+}