feat(observability): per-GPU metrics for the mixed AMD/NVIDIA fleet

Add a host-process GPU exporter, scrape it on both servers, and graph it
on the host-system dashboard. The fleet has four GPUs across three vendors
of interest (2x R9700 on Dankserver, the 8060S iGPU + RTX 3090 on Dankclaw)
and nothing exposed them until now.

scripts/gpu-exporter.py reads each AMD card's amdgpu sysfs
(gpu_busy_percent / mem_info_vram_* / hwmon temp1_input / power1_average)
plus one nvidia-smi --query-gpu CSV, and emits unified gpu_* Prometheus
families (utilization_percent, memory_used/total_bytes, temperature_celsius,
power_watts) labelled gpu/vendor/model. Absent hardware yields ABSENT series,
never a fabricated 0: the R9700s expose no sysfs power, so they simply carry
no gpu_power_watts line. Both sources are injectable, so the seam runs on a
GPU-less dev box; live-verified on Dankserver's real sysfs.

It runs as a plain user unit (not a quadlet): it must read the host's own DRM
sysfs and shell to nvidia-smi, which a container cannot do without GPU
passthrough. Same posture as signal-relay. The base unit binds loopback; the
deploy drop-in appends the host LAN address, never a wildcard (matching
node-exporter/cadvisor). deploy_gpu_exporter installs/enables/health-gates it
on both hosts and is wired into the planner (both targets), the test-changed
router, and two Prometheus scrape jobs using the node/cadvisor reach rules
(gateway on Dankclaw, LAN on Dankserver).

dankclaw/observability/quadlet/cadvisor.container

@@ -0,0 +1,56 @@
+# Quadlet unit: cadvisor (rootless, boot-persistent) — DANKCLAW.
+#
+# Install path on dankclaw: ~/.config/containers/systemd/cadvisor.container
+# Unit name: cadvisor.service (container name: cadvisor)
+#
+# WHY THIS EXISTS. The app stack is ~34 rootless Podman containers and until
+# now nothing reported per-container CPU/memory — Grafana could only see the
+# host (node_exporter) and the LLM engines. cadvisor reads the host cgroup
+# tree and exports container_cpu_* / container_memory_* per container.
+#
+# THE LOAD-BEARING FLAG IS --cgroupns=host. Podman starts every container in a
+# PRIVATE cgroup namespace, so an exporter container sees only its OWN cgroup
+# subtree: cadvisor exported a single root series until this flag was added,
+# then it walked the whole tree (996 per-container series measured 2026-10-20).
+# The quadlet unit name is carried in the cgroup path — app.slice/<name>.service/
+# libpod-payload-<hex> — so Prometheus relabels that path into a `container`
+# label (shared/prometheus/prometheus.yml, cadvisor-dankclaw job) and no
+# libpod/podman socket is needed to name containers.
+#
+# BINDING. cadvisor has no per-interface listen flag (unlike node_exporter's
+# --web.listen-address), so it binds 0.0.0.0 INSIDE its own network namespace.
+# The bridge PublishPort is what keeps the unauthenticated endpoint off the
+# host's other interfaces: it publishes exactly loopback (host-local probes and
+# the deployer's health gate) and this host's LAN address (Prometheus, a bridge
+# container, reaches a host-published port at its LAN address — the same netavark
+# rule node-exporter relies on, only here the port really is published).
+#
+# It is rootless on purpose: dank owns the app containers' cgroups, so a
+# rootless cadvisor reads every one of them; root would buy nothing here.
+
+[Unit]
+Description=cAdvisor container metrics (Dankclaw)
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+ContainerName=cadvisor
+Image=gcr.io/cadvisor/cadvisor:v0.49.1
+PodmanArgs=--pid=host --cgroupns=host
+PublishPort=127.0.0.1:8080:8080
+PublishPort=192.168.0.143:8080:8080
+Volume=/sys:/sys:ro
+Volume=/:/rootfs:ro
+Exec=--housekeeping_interval=30s
+HealthCmd=wget --spider -q http://localhost:8080/healthz
+HealthInterval=30s
+HealthTimeout=10s
+HealthRetries=3
+HealthStartPeriod=40s
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target

dankserver/observability/quadlet/cadvisor.container

@@ -0,0 +1,39 @@
+# Quadlet unit: cadvisor (rootless, boot-persistent) — DANKSERVER.
+#
+# Install path on dankserver: ~/.config/containers/systemd/cadvisor.container
+# Unit name: cadvisor.service (container name: cadvisor)
+#
+# The Dankserver twin of dankclaw/observability/quadlet/cadvisor.container.
+# Same rootless shape, same --cgroupns=host requirement (see that file for the
+# full story), only the LAN address differs: this host publishes on
+# 192.168.0.235:8080 so Prometheus on Dankclaw can scrape it over the real
+# network (cross-host, so the host.containers.internal trick does not apply).
+# It reports the media/library stack (romm, plex, jellyfin, copyparty, calibre,
+# tautulli) by their quadlet unit names via the cgroup path.
+
+[Unit]
+Description=cAdvisor container metrics (Dankserver)
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+ContainerName=cadvisor
+Image=gcr.io/cadvisor/cadvisor:v0.49.1
+PodmanArgs=--pid=host --cgroupns=host
+PublishPort=127.0.0.1:8080:8080
+PublishPort=192.168.0.235:8080:8080
+Volume=/sys:/sys:ro
+Volume=/:/rootfs:ro
+Exec=--housekeeping_interval=30s
+HealthCmd=wget --spider -q http://localhost:8080/healthz
+HealthInterval=30s
+HealthTimeout=10s
+HealthRetries=3
+HealthStartPeriod=40s
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target

scripts/deploy-plan.py

@@ -39,6 +39,8 @@ SERVICE_ORDER = (
     "loki",
     "alloy",
     "node-exporter",
+    "cadvisor",
+    "gpu-exporter",
     "alertmanager",
     "authentik-postgres",
     "authentik-server",
@@ -92,6 +94,8 @@ DANKCLAW_SERVICES = {
     "loki",
     "alloy",
     "node-exporter",
+    "cadvisor",
+    "gpu-exporter",
     "alertmanager",
     "authentik-postgres",
     "authentik-server",
@@ -124,6 +128,8 @@ DANKSERVER_SERVICES = {
     "readarr",
     "frank-archon",
     "node-exporter",
+    "cadvisor",
+    "gpu-exporter",
 }
 
 # FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on
@@ -374,7 +380,7 @@ def services_for_path(path: str, target: str) -> set[str]:
             # consumer (prometheus before grafana, loki before alloy).
             if len(parts) >= 4 and parts[2] == "quadlet":
                 name = parts[3].removesuffix(".container")
-                if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "alertmanager"}:
+                if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "cadvisor", "alertmanager"}:
                     _add(services, name)
             return services
         if len(parts) >= 2 and parts[1] == "gitea":
@@ -417,8 +423,8 @@ def services_for_path(path: str, target: str) -> set[str]:
             # node-dankserver job had nothing to scrape.
             if len(parts) >= 4 and parts[2] == "quadlet":
                 name = parts[3].removesuffix(".container")
-                if name == "node-exporter":
-                    _add(services, "node-exporter")
+                if name in {"node-exporter", "cadvisor"}:
+                    _add(services, name)
             return services
         # frank-archon's quadlet-era directory left the tree on 2026-09-26
         # (the Lucebox DS4.1 unit was retired for the llama.cpp root unit —
@@ -449,6 +455,12 @@ def services_for_path(path: str, target: str) -> set[str]:
             # change to it redeploying alertmanager is the only path that
             # reaches it on the host.
             _add(services, "alertmanager")
+        if name in {"gpu-exporter.py", "gpu-exporter.service"}:
+            # The host-process GPU exporter is installed directly by the
+            # gpu-exporter deploy on BOTH hosts, so its script and unit select
+            # that service (target_services then keeps it only on the host being
+            # planned). Editing the collector restarts the running exporter.
+            _add(services, "gpu-exporter")
         if name in {"frank-zealot.env", "frank-zealot.service", "serve-frank-zealot.sh"}:
             # llama.cpp frank-zealot (2026-09-16) keeps its unit, env and serve
             # files here; each one selects frank-zealot only. The ik_llama.cpp

scripts/deploy-remote.sh

@@ -86,7 +86,7 @@ service_is_allowed() {
         dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
         dankclaw:comfyui|dankclaw:hermes|\
         dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
-        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\
+        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:alertmanager|\
         dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
         dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
@@ -97,7 +97,7 @@ service_is_allowed() {
         dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
         dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
         dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
-        dankserver:frank-archon|dankserver:node-exporter|\
+        dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|dankserver:gpu-exporter|\
         frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
         frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
         frankbook-pro:readarr)
@@ -275,6 +275,36 @@ install_signal_relay() {
     return 1
 }
 
+# Host-process GPU metrics exporter (amdgpu sysfs + nvidia-smi). Not a quadlet:
+# it runs python3 from the checkout on the host so it can read the DRM/cgroup
+# sysfs and shell out to nvidia-smi. Same two-address bind posture as
+# node-exporter/cadvisor (loopback + this host's LAN), so it is never a wildcard.
+deploy_gpu_exporter() {
+    local unit_dir="$HOME/.config/systemd/user" lan dropin i
+    case "$TARGET" in
+        dankclaw) lan=192.168.0.143 ;;
+        dankserver) lan=192.168.0.235 ;;
+        *) echo "gpu-exporter: unknown target '$TARGET'" >&2; return 2 ;;
+    esac
+    mkdir -p "$unit_dir"
+    cp scripts/gpu-exporter.service "$unit_dir/gpu-exporter.service"
+    dropin="$unit_dir/gpu-exporter.service.d"
+    mkdir -p "$dropin"
+    printf '[Service]\nEnvironment=GPU_EXPORTER_BINDS=127.0.0.1,%s\n' "$lan" \
+        > "$dropin/bind.conf"
+    systemctl --user daemon-reload
+    systemctl --user enable gpu-exporter.service
+    systemctl --user restart gpu-exporter.service
+    # The interpreter needs a moment to bind after restart (the same race that
+    # once failed a healthy signal-relay deploy).
+    for i in $(seq 1 15); do
+        curl -fsS -m 5 http://127.0.0.1:9401/metrics >/dev/null && return 0
+        sleep 1
+    done
+    echo "gpu-exporter is not answering on 127.0.0.1:9401/metrics" >&2
+    return 1
+}
+
 copy_loki_config() {
     run_privileged mkdir -p /opt/loki/config
     run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
@@ -520,6 +550,8 @@ deploy_observability() {
             ;;
         node-exporter)
             ;;
+        cadvisor)
+            ;;
         *)
             echo "unknown observability service: $service" >&2
             return 2
@@ -533,6 +565,7 @@ deploy_observability() {
         grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
         loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
         node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
+        cadvisor) wait_for_health cadvisor http://127.0.0.1:8080/healthz 90 ;;
     esac
 }
 
@@ -2000,7 +2033,8 @@ deploy_service() {
         plex) deploy_plex ;;
         tautulli) deploy_tautulli ;;
         alloy) deploy_alloy ;;
-        prometheus|grafana|loki|node-exporter|alertmanager) deploy_observability "$service" ;;
+        prometheus|grafana|loki|node-exporter|cadvisor|alertmanager) deploy_observability "$service" ;;
+        gpu-exporter) deploy_gpu_exporter ;;
         comfyui) deploy_comfyui ;;
         *) deploy_compose_service "$service" ;;
     esac

scripts/gpu-exporter.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""gpu-exporter: unified per-GPU metrics for a mixed AMD/NVIDIA host.
+
+Emits the `gpu_*` Prometheus families the host-system dashboard graphs:
+
+    gpu_utilization_percent  gpu_memory_used_bytes  gpu_memory_total_bytes
+    gpu_temperature_celsius  gpu_power_watts
+
+each labelled {gpu, vendor, model}. The scrape job adds `host`.
+
+Two sources, both host-native — no container, no GPU passthrough:
+
+  * AMD (R9700s, the 8060S iGPU, the Raphael display GPU): the amdgpu sysfs tree.
+    Utilization comes ONLY from gpu_busy_percent — amd-smi reports gfx_busy_inst
+    as N/A on gfx1201 (verified 2026-10-20). VRAM is already bytes, temperature
+    is milli-degrees, power is micro-watts. Crucially power1_average is EMPTY on
+    the R9700s, so those cards legitimately have no gpu_power_watts line; a
+    missing file becomes an ABSENT metric, never a fabricated 0 that a dashboard
+    would plot as a real reading.
+
+  * NVIDIA (the RTX 3090 on Dankclaw): one `nvidia-smi --query-gpu` CSV. On
+    Dankserver there is an nvidia-smi binary but no NVIDIA GPU, so the command
+    fails; that is tolerated and simply yields no NVIDIA series.
+
+Default mode prints one scrape to stdout and exits (the test harness, and any
+`service ... --oneshot` one-off). `--serve` binds the HTTP endpoint for
+Prometheus; it binds EXPLICITLY (loopback + the host LAN address), never a
+wildcard, matching node-exporter's posture on these boxes.
+"""
+import argparse
+import glob
+import os
+import subprocess
+import sys
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+FAMILIES = [
+    ("gpu_utilization_percent", "gauge", "GPU busy time as a percentage of wall time."),
+    ("gpu_memory_used_bytes", "gauge", "Device memory in use, in bytes."),
+    ("gpu_memory_total_bytes", "gauge", "Device memory capacity, in bytes."),
+    ("gpu_temperature_celsius", "gauge", "Package temperature in degrees Celsius."),
+    ("gpu_power_watts", "gauge", "Board power draw in watts."),
+]
+
+DEFAULT_NVIDIA_CMD = (
+    "nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,"
+    "memory.total,temperature.gpu,power.draw --format=csv,noheader,nounits"
+)
+
+
+def _read(path):
+    try:
+        with open(path) as fh:
+            return fh.read().strip()
+    except OSError:
+        return ""
+
+
+def _pci_slot(device_dir):
+    uevent = _read(os.path.join(device_dir, "uevent"))
+    for line in uevent.splitlines():
+        if line.startswith("PCI_SLOT_NAME="):
+            return line.split("=", 1)[1]
+    return ""
+
+
+def _pci_id(device_dir):
+    uevent = _read(os.path.join(device_dir, "uevent"))
+    for line in uevent.splitlines():
+        if line.startswith("PCI_ID="):
+            return "amd:" + line.split("=", 1)[1].lower()
+    return "amd"
+
+
+def collect_amd(sys_dir):
+    for card in sorted(glob.glob(os.path.join(sys_dir, "card[0-9]*"))):
+        device = os.path.join(card, "device")
+        busy = _read(os.path.join(device, "gpu_busy_percent"))
+        if busy == "":
+            continue  # not an amdgpu compute node (e.g. a render node w/o metrics)
+        labels = {"gpu": _pci_slot(device) or os.path.basename(card),
+                  "vendor": "amd", "model": _pci_id(device)}
+        yield "gpu_utilization_percent", labels, busy
+        for name, leaf in (("gpu_memory_used_bytes", "mem_info_vram_used"),
+                           ("gpu_memory_total_bytes", "mem_info_vram_total")):
+            v = _read(os.path.join(device, leaf))
+            if v != "":
+                yield name, labels, v
+        temp = ""
+        for hwmon in sorted(glob.glob(os.path.join(device, "hwmon", "hwmon*", "temp1_input"))):
+            temp = _read(hwmon)
+            if temp != "":
+                break
+        if temp != "":
+            yield "gpu_temperature_celsius", labels, str(int(temp) / 1000.0)
+        power = _read(os.path.join(device, "power1_average"))
+        if power != "":  # empty on the R9700s -> metric absent, not zero
+            yield "gpu_power_watts", labels, str(int(power) / 1e6)
+
+
+def collect_nvidia(cmd):
+    if not cmd:
+        return
+    try:
+        out = subprocess.run(cmd, shell=True, capture_output=True, text=True,
+                             timeout=10, check=True).stdout
+    except (subprocess.SubprocessError, OSError):
+        return  # no NVIDIA GPU / no driver: tolerate, emit nothing
+    for line in out.splitlines():
+        parts = [p.strip() for p in line.split(",")]
+        if len(parts) < 7:
+            continue
+        idx, name, util, mused, mtotal, temp, power = parts[:7]
+        labels = {"gpu": idx, "vendor": "nvidia", "model": name}
+        yield "gpu_utilization_percent", labels, util
+        yield "gpu_memory_used_bytes", labels, str(int(float(mused)) * 1048576)
+        yield "gpu_memory_total_bytes", labels, str(int(float(mtotal)) * 1048576)
+        if temp not in ("", "[N/A]"):
+            yield "gpu_temperature_celsius", labels, temp
+        if power not in ("", "[N/A]"):
+            yield "gpu_power_watts", labels, power
+
+
+def collect(sys_dir, nvidia_cmd):
+    samples = {}
+    for source in (collect_amd(sys_dir), collect_nvidia(nvidia_cmd)):
+        for name, labels, value in source:
+            try:
+                value = str(float(value))  # normalise; drop trailing junk
+            except ValueError:
+                continue
+            samples.setdefault(name, []).append((labels, value))
+    return samples
+
+
+def _fmt(value):
+    f = float(value)
+    if f == int(f) and abs(f) < 1e15:
+        return str(int(f))
+    return repr(f)
+
+
+def render(samples):
+    out = []
+    for name, kind, help_text in FAMILIES:
+        out.append("# HELP %s %s" % (name, help_text))
+        out.append("# TYPE %s %s" % (name, kind))
+        for labels, value in sorted(samples.get(name, []), key=lambda x: sorted(x[0].items())):
+            lset = ",".join('%s="%s"' % (k, v) for k, v in sorted(labels.items()))
+            out.append("%s{%s} %s" % (name, lset, _fmt(value)))
+    return "\n".join(out) + "\n"
+
+
+class Handler(BaseHTTPRequestHandler):
+    sys_dir = "/sys/class/drm"
+    nvidia_cmd = DEFAULT_NVIDIA_CMD
+
+    def do_GET(self):  # noqa: N802 (http.server API)
+        if self.path.split("?")[0] not in ("/metrics", "/"):
+            self.send_error(404)
+            return
+        body = render(collect(self.sys_dir, self.nvidia_cmd)).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, *args):
+        pass
+
+
+def main(argv):
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--amdgpu-sysfs", default="/sys/class/drm")
+    ap.add_argument("--nvidia-cmd", default=DEFAULT_NVIDIA_CMD)
+    ap.add_argument("--serve", action="store_true", help="serve /metrics instead of one-shot")
+    ap.add_argument("--port", type=int, default=9401)
+    ap.add_argument("--bind", action="append", default=None,
+                    help="explicit bind address (repeatable); env GPU_EXPORTER_BINDS "
+                         "comma-list if unset. NEVER a wildcard.")
+    args = ap.parse_args(argv)
+
+    if not args.serve:
+        sys.stdout.write(render(collect(args.amdgpu_sysfs, args.nvidia_cmd)))
+        return 0
+
+    binds = args.bind
+    if not binds:
+        env = os.environ.get("GPU_EXPORTER_BINDS", "")
+        binds = [b.strip() for b in env.split(",") if b.strip()] or ["127.0.0.1"]
+    Handler.sys_dir = args.amdgpu_sysfs
+    Handler.nvidia_cmd = args.nvidia_cmd
+    servers = []
+    for b in binds:
+        if b in ("0.0.0.0", "::", "*"):
+            sys.stderr.write("refusing wildcard bind %s\n" % b)
+            return 2
+        servers.append(ThreadingHTTPServer((b, args.port), Handler))
+    sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds)))
+    servers[0].serve_forever()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

scripts/gpu-exporter.service

@@ -0,0 +1,35 @@
+# Systemd USER unit: gpu-exporter (both Dankclaw and Dankserver).
+#
+# Install path: ~/.config/systemd/user/gpu-exporter.service
+# Installed and enabled by deploy_gpu_exporter in scripts/deploy-remote.sh.
+#
+# Per-GPU metrics (utilization / VRAM / temperature / power) for the mixed
+# AMD + NVIDIA fleet, scraped by Prometheus. Like signal-relay it is a plain
+# user unit, not a quadlet: it runs python3 straight from the checkout (a pull
+# updates it with no copy step), a hand-written user unit can be enabled where
+# the podman v1 generator's units cannot, and with Linger=yes for dank it starts
+# at boot. It reads the host's own cgroup/drm sysfs and shells to nvidia-smi, so
+# it must run on the host — a container would need GPU/DRM passthrough for
+# nothing gained.
+#
+# BINDING. The base unit serves loopback only. deploy_gpu_exporter writes a
+# drop-in adding THIS host's LAN address (the same two-address posture
+# node-exporter and cadvisor use), so the endpoint is never a wildcard. It is
+# unauthenticated metrics on the LAN, identical to node-exporter :9100 and
+# cadvisor :8080 — Prometheus reaches same-host via host.containers.internal and
+# the Dankserver instance over the LAN from Dankclaw.
+
+[Unit]
+Description=GPU metrics exporter (amdgpu sysfs + nvidia-smi)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+# Loopback by default; the deploy drop-in appends the host LAN address.
+Environment=GPU_EXPORTER_BINDS=127.0.0.1
+ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/gpu-exporter.py --serve
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=default.target

scripts/test-changed.sh

@@ -293,6 +293,14 @@ for path in "${CHANGED[@]}"; do
             add_test tests/caddyfile.bats
             add_test tests/deployment_scoping.bats
             ;;
+        # The GPU exporter is a host process; its seam pins the collector, the
+        # user unit, the deployer function, the planner route, the Prometheus
+        # scrape jobs and the host-system GPU panels — a cross-file contract, so
+        # the whole seam rides on any of its own files.
+        scripts/gpu-exporter.py|scripts/gpu-exporter.service)
+            add_test tests/gpu_exporter.bats
+            add_test tests/deployment_scoping.bats
+            ;;
         dankclaw/udev/*)
             # Host-level udev rules are applied by manual sudo install + udevadm
             # trigger, not dispatched as a service; nothing here is bats-testable.

serverAppConfigs-docs/Infrastructure/Observability.md

@@ -2,7 +2,7 @@
 type: reference
 title: "Observability on Dankclaw"
 description: "The observability stack: what runs, how it is deployed, how it is exposed, and what is still unconfigured."
-tags: [observability, prometheus, grafana, loki, alloy, node-exporter, quadlet, dankclaw]
+tags: [observability, prometheus, grafana, loki, alloy, node-exporter, gpu-exporter, quadlet, dankclaw]
 timestamp: 2026-10-20
 ---
 
@@ -10,10 +10,10 @@ timestamp: 2026-10-20
 
 ## Summary
 
-Six containers and one host-side process: Prometheus (metrics), Grafana
+Seven containers and two host-side processes: Prometheus (metrics), Grafana
 (dashboards), Loki (log store), Alloy (log shipper), node-exporter (host
-metrics) and Alertmanager (alert delivery), with `signal-relay` on the host
-carrying every firing group to Signal. All of it is Quadlet-managed,
+metrics), cadvisor (per-container metrics) and Alertmanager (alert delivery),
+plus two host `python3` processes — `gpu-exporter` (per-GPU metrics) and `signal-relay` (Signal delivery). The containers are all Quadlet-managed,
 boot-persistent and loopback-only, deployed by
 `scripts/deploy-remote.sh` → `deploy_observability`. The metrics half works
 end to end; the log half is configured, healthy, and carries no data.
@@ -34,6 +34,8 @@ that proves it today.
 | Loki | `localhost/franksdank/loki:latest` | `127.0.0.1:3100` | `loki.container` | [[Loki]] |
 | Alloy | `localhost/franksdank/alloy:latest` | `127.0.0.1:12345` | `alloy.container` | [[Alloy]] |
 | node-exporter | `docker.io/prom/node-exporter:latest` | host netns: `127.0.0.1:9100` + `192.168.0.143:9100` | `node-exporter.container` | [[Node-Exporter]] |
+| cadvisor | `gcr.io/cadvisor/cadvisor:v0.49.1` | bridge publish `127.0.0.1:8080` + `192.168.0.143:8080` | `cadvisor.container` | — |
+| GPU exporter | host `python3`, not a container | `127.0.0.1:9401` + host LAN `:9401` | `scripts/gpu-exporter.service` (user unit) | — |
 | Signal relay | host `python3`, not a container | `192.168.0.143:18093` | `scripts/signal-relay.service` (user unit) | [[Alertmanager]] |
 
 Every unit lives in `dankclaw/observability/quadlet/`, carries
@@ -44,9 +46,11 @@ linger is on for `dank`). `signal-relay.service` reports `enabled` + `active`
 the same day, and `homepage-status.timer` — the root unit that probes all of
 these on loopback — is `enabled` + `active` (`deploy-remote.sh:319-321` copies
 the probe registry `shared/homepage/services.json` to `/srv/homepage`).
-Dankserver runs one member of this family, node-exporter, as its own unit
-(`dankserver/observability/quadlet/node-exporter.container`, `generated` +
-`active`, binding `127.0.0.1:9100` + `192.168.0.235:9100`).
+Dankserver runs the same family: node-exporter and cadvisor as its own quadlets
+(`dankserver/observability/quadlet/{node-exporter,cadvisor}.container`,
+`generated` + `active`; node-exporter binds `127.0.0.1:9100` + `192.168.0.235:9100`,
+cadvisor publishes `127.0.0.1:8080` + `192.168.0.235:8080`), plus the host-process
+`gpu-exporter` (a user unit installed the same way as `signal-relay`).
 
 ## Exposure
 
@@ -98,10 +102,35 @@ verify it through the APIs.
   relay, and `journalctl --user -u signal-relay` records successful
   `sent to +12062476336:` lines for every rule in the set.
 * **Dashboards: working.** Provisioned datasources — Prometheus (uid
-  `prometheus`) and Loki (uid `loki`) — and the `llm-fleet` dashboard, all
+  `prometheus`) and Loki (uid `loki`) — and the `llm-fleet`, `host-system`,
+  `containers`, `observability` and `logs-overview` dashboards, all
   installed from `shared/grafana/provisioning/`. The Loki datasource is how
   you read logs: Loki has no web UI of its own, so Grafana Explore is the
   human surface (the guarded `loki.franksdank.cloud` vhost serves only the API).
+* **Per-container metrics: the `containers` dashboard (added 2026-10-20; live
+  after `deploy.sh <host> --service cadvisor`).** cadvisor is the only way to see
+  per-container CPU/memory here, and on this rootless-Podman host it works on one
+  condition: `--cgroupns=host`. Podman starts every container in a PRIVATE cgroup
+  namespace, so without that flag cadvisor sees only its own cgroup root and
+  exports a single series (measured 2026-10-20: 1 series off, 996 on). cadvisor
+  names nothing itself — the libpod socket resolved only 2 of 34 rootless
+  containers — but the quadlet unit name is carried in the cgroup path
+  (`app.slice/<name>.service/…`), so `prometheus.yml` relabels that path into a
+  `container` label. Network and filesystem are NOT exported for these containers
+  (no netns/classid attribution under the host-PID layout), so the dashboard
+  graphs only CPU, memory and OOM and does not fake the rest.
+* **Per-GPU metrics: the `host-system` GPU panels (added 2026-10-20; live after
+  `deploy.sh <host> --service gpu-exporter`).** `scripts/gpu-exporter.py` is a
+  host-process user unit on BOTH hosts — not a quadlet, because it reads the host's
+  own DRM sysfs and shells to `nvidia-smi`, neither of which a container can do
+  without GPU passthrough. It reads each AMD card's `gpu_busy_percent` /
+  `mem_info_vram_*` / `hwmon temp1_input` and one `nvidia-smi --query-gpu` CSV, and
+  emits unified `gpu_*` families (utilization %, VRAM used/total bytes, temperature
+  °C, power W) labelled `gpu` / `vendor` / `model`, with `host` added by the scrape
+  job. Two laws: **absent means absent, never a zero** — the R9700s expose no sysfs
+  `power1_average`, so they carry no `gpu_power_watts` line at all (only NVIDIA and
+  the Dankclaw 8060S iGPU report power); and it binds loopback + the host LAN, like
+  node-exporter, never the wildcard. The dashboard graphs util/VRAM/VRAM-%/temp/power.
 * **Logs: fixed in code, awaiting deploy.** The pipeline shipped nothing because
   a rootless Alloy cannot read the `0640` journal files, and the container-log
   globs point at a root-storage layout that does not exist here. The fix is built
@@ -117,9 +146,10 @@ verify it through the APIs.
 
 ## Citations
 
-- `dankclaw/observability/quadlet/{prometheus,alertmanager,grafana,loki,alloy,node-exporter}.container`
+- `dankclaw/observability/quadlet/{prometheus,alertmanager,grafana,loki,alloy,node-exporter,cadvisor}.container`
 - `scripts/deploy-remote.sh:219-300` (copiers + relay), `:521-536` (health waits), `:477-486` (grafana env seed)
 - `scripts/signal-relay.py:26-49`, `scripts/signal-relay.service`
+- `scripts/gpu-exporter.py`, `scripts/gpu-exporter.service` (per-GPU host exporter)
 - `shared/prometheus/prometheus.yml`, `shared/prometheus/alertmanager.yml`, `shared/prometheus/rules/llm-fleet.yml`
 - `shared/grafana/provisioning/`, `shared/loki/loki-config.yaml`, `shared/alloy/config.alloy`
 - `tests/observability_quadlet.bats` (boot persistence, no wildcard publish, probe addresses, delivery contract)

shared/grafana/provisioning/dashboards/containers.json

@@ -0,0 +1,137 @@
+{
+  "uid": "containers",
+  "title": "Containers — Dankclaw & Dankserver",
+  "description": "Per-container CPU and memory for the quadlet app stacks, from cadvisor (a rootless quadlet on each host). cadvisor keys each series by the cgroup path, and Prometheus relabels the quadlet unit name (app.slice/<name>.service) into a `container` label. Panels use only the metric families cadvisor actually exports for these rootless containers: CPU usage and memory working-set/limit/OOM. Network and filesystem are NOT exported for them (no netns/classid attribution under the rootless+host-PID layout), so no panels pretend otherwise. See serverAppConfigs-docs/Infrastructure/Observability.md.",
+  "tags": ["fleet", "cadvisor", "containers"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "templating": {
+    "list": [
+      {
+        "name": "host",
+        "label": "Host",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\"}, host)", "refId": "host_var" },
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": { "text": "All", "value": "$__all" }
+      },
+      {
+        "name": "container",
+        "label": "Container",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\"}, container)", "refId": "container_var" },
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": { "text": "All", "value": "$__all" }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "row",
+      "title": "CPU",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "panels": []
+    },
+    {
+      "id": 2,
+      "type": "timeseries",
+      "title": "CPU usage (cores)",
+      "description": "CPU-seconds/s per container = cores busy. A steady high line is a busy service; a spike is a burst (a render, an index, a backup). Only containers are shown, not the host or its daemons.",
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 1 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
+      "targets": [
+        {
+          "expr": "sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
+          "legendFormat": "{{host}} / {{container}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "type": "bargauge",
+      "title": "Top containers by CPU (cores)",
+      "description": "Instant ranking over the selected window. Tells you which container to look at when the host is loaded.",
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": null, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 4 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true },
+      "targets": [
+        {
+          "expr": "topk(12, sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m])))",
+          "legendFormat": "{{host}} / {{container}}",
+          "format": "table",
+          "instant": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "type": "row",
+      "title": "Memory",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 },
+      "panels": []
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Memory working set",
+      "description": "container_memory_working_set_bytes per container — the number the OOM killer acts on. Cache-inclusive and reclaimable; a flat high line is resident data, a sawtooth is churn.",
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 11 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
+      "targets": [
+        {
+          "expr": "container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}",
+          "legendFormat": "{{host}} / {{container}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "Memory vs limit (%)",
+      "description": "Working set over the cgroup memory limit (container_spec_memory_limit_bytes). Only containers that carry a limit are shown; approaching 1.0 means an OOM is near. Containers with no limit are omitted.",
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 11 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
+      "targets": [
+        {
+          "expr": "(container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} / on (job, host, container) group_left container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}) and (container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} > 0)",
+          "legendFormat": "{{host}} / {{container}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 7,
+      "type": "timeseries",
+      "title": "OOM kills",
+      "description": "container_oom_events_total rate. Any non-zero line means the kernel killed a process inside that container — the honest signal that its memory limit is too tight for its real working set.",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 20, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] },
+      "targets": [
+        {
+          "expr": "sum by (host, container) (rate(container_oom_events_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
+          "legendFormat": "{{host}} / {{container}}",
+          "refId": "A"
+        }
+      ]
+    }
+  ]
+}

shared/grafana/provisioning/dashboards/host-system.json

@@ -18,6 +18,16 @@
         "includeAll": true,
         "multi": true,
         "current": { "text": "All", "value": "$__all" }
+      },
+      {
+        "name": "gpuhost",
+        "label": "gpu host",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(gpu_utilization_percent, host)", "refId": "A" },
+        "includeAll": true,
+        "multi": true,
+        "current": { "text": "All", "value": "$__all" }
       }
     ]
   },
@@ -139,6 +149,56 @@
         { "expr": "sum by (job,device) (rate(node_network_receive_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} RX err", "refId": "A" },
         { "expr": "sum by (job,device) (rate(node_network_transmit_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} TX err", "refId": "B" }
       ]
+    },
+    {
+      "id": 13, "type": "timeseries", "title": "GPU busy %",
+      "description": "Per-GPU utilization from gpu-exporter (amdgpu gpu_busy_percent / nvidia-smi). The picker above selects the host; each series is one card.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
+      "targets": [
+        { "expr": "gpu_utilization_percent{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
+      ]
+    },
+    {
+      "id": 14, "type": "timeseries", "title": "GPU VRAM used",
+      "description": "Device memory in use. On the R9700s (frank-archon) and the 3090 (frank-zealot) this is the loaded model's footprint.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "min": 0 }, "overrides": [] },
+      "targets": [
+        { "expr": "gpu_memory_used_bytes{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
+      ]
+    },
+    {
+      "id": 15, "type": "timeseries", "title": "GPU VRAM used %",
+      "description": "Used / total device memory. Near 100% is the pre-OOM wall.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
+      "targets": [
+        { "expr": "100 * gpu_memory_used_bytes{host=~\"$gpuhost\"} / gpu_memory_total_bytes{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
+      ]
+    },
+    {
+      "id": 16, "type": "timeseries", "title": "GPU temperature",
+      "description": "Package temperature. AMD edge temp (hwmon temp1_input); NVIDIA gpu temp.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "celsius", "min": 0 }, "overrides": [] },
+      "targets": [
+        { "expr": "gpu_temperature_celsius{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
+      ]
+    },
+    {
+      "id": 17, "type": "timeseries", "title": "GPU power draw",
+      "description": "Board power draw. Emitted only where readable: NVIDIA and the Dankclaw iGPU. The R9700s have no sysfs power sensor, so they are simply absent here — never a fabricated 0.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "watt", "min": 0 }, "overrides": [] },
+      "targets": [
+        { "expr": "gpu_power_watts{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
+      ]
     }
   ]
 }

shared/prometheus/prometheus.yml

@@ -131,3 +131,85 @@ scrape_configs:
       - targets: ["192.168.0.235:7770"]
         labels:
           service: frank-archon
+
+  # Per-container metrics from cadvisor (a rootless quadlet on each host).
+  # cadvisor keys every series by the cgroup path; for a quadlet container that
+  # path is …/app.slice/<name>.service[/libpod-payload-<hex>], so the unit NAME
+  # lives in it. The relabel below pulls that name out into a `container` label
+  # and keeps only the per-unit aggregate — cadvisor's raw host root,
+  # system.slice daemons and machine.slice scopes carry no such name and are
+  # dropped, as is the per-CPU-core split (only cpu="total" survives), so the
+  # scrape stays small. cadvisor cannot bind per-interface, so it is reached at
+  # the host's PUBLISHED LAN port: unlike node-exporter's host-net bind (which
+  # is why node-dankclaw scrapes via host.containers.internal), a published port
+  # IS reachable from this bridge container at the host LAN address.
+  - job_name: "cadvisor-dankclaw"
+    static_configs:
+      - targets: ["192.168.0.143:8080"]
+        labels:
+          host: dankclaw
+    metric_relabel_configs:
+      - source_labels: [id]
+        regex: '.*/app\.slice/([^.]+)\.service(?:/.*)?$'
+        target_label: container
+        replacement: '$1'
+      # Keep only the unit-level cgroup; drop its payload/control children so a
+      # container is not counted twice.
+      - source_labels: [id]
+        regex: '.*/\.service/.*'
+        action: drop
+      # Drop everything that is not a named unit (no `container` was extracted).
+      - source_labels: [container]
+        regex: '.+'
+        action: keep
+      # cpu="total" for cpu series; metrics with no cpu label (memory, network,
+      # filesystem) are kept.
+      - source_labels: [cpu]
+        regex: '^$|total'
+        action: keep
+      - regex: 'id|name|image|container_label.*|machine_id|boot_id'
+        action: labeldrop
+
+  - job_name: "cadvisor-dankserver"
+    static_configs:
+      - targets: ["192.168.0.235:8080"]
+        labels:
+          host: dankserver
+    metric_relabel_configs:
+      - source_labels: [id]
+        regex: '.*/app\.slice/([^.]+)\.service(?:/.*)?$'
+        target_label: container
+        replacement: '$1'
+      - source_labels: [id]
+        regex: '.*/\.service/.*'
+        action: drop
+      - source_labels: [container]
+        regex: '.+'
+        action: keep
+      - source_labels: [cpu]
+        regex: '^$|total'
+        action: keep
+      - regex: 'id|name|image|container_label.*|machine_id|boot_id'
+        action: labeldrop
+
+  # Per-GPU metrics (gpu_utilization_percent / gpu_memory_used_bytes /
+  # gpu_memory_total_bytes / gpu_temperature_celsius / gpu_power_watts) from the
+  # host-process gpu-exporter (scripts/gpu-exporter.py). One job per host, same
+  # reach rules as node/cadvisor: Dankclaw is scraped at the podman gateway
+  # because the netavark hairpin quirk makes Prometheus fail to dial its own
+  # host LAN IP; Dankserver is plain LAN. The gpu label (PCI BDF on AMD, index on
+  # NVIDIA) + vendor/model come from the exporter, so the GPU row on the
+  # host-system dashboard filters by host. Power is emitted only where readable
+  # (NVIDIA + the Dankclaw iGPU); the R9700s have no sysfs power, so they simply
+  # have no gpu_power_watts series — no fabricated zero.
+  - job_name: "gpu-dankclaw"
+    static_configs:
+      - targets: ["host.containers.internal:9401"]
+        labels:
+          host: dankclaw
+
+  - job_name: "gpu-dankserver"
+    static_configs:
+      - targets: ["192.168.0.235:9401"]
+        labels:
+          host: dankserver

tests/gpu_exporter.bats

@@ -0,0 +1,197 @@
+#!/usr/bin/env bats
+# Seam: the GPU metrics exporter's black-box contract.
+#
+# The exporter reads amdgpu sysfs (one card per /sys/class/drm/cardN that has a
+# device/gpu_busy_percent file) and, on the box that has a working NVIDIA stack,
+# one `nvidia-smi --query-gpu` CSV, and emits the unified `gpu_*` Prometheus
+# families the host-system dashboard graphs. Both sources are injectable so the
+# whole thing is testable with fixtures on a GPU-less dev box.
+#
+# Only the inputs a consumer sees are pinned: the metric families, their label
+# set, and that absent hardware yields ABSENT series (never a fabricated 0 — a
+# wrong number on a dashboard gets believed). The R9700s expose no sysfs
+# power1_average, so those cards legitimately carry no gpu_power_watts line.
+
+load helpers
+
+EXPORTER="$REPO_ROOT/scripts/gpu-exporter.py"
+
+# Build a fixture amdgpu sysfs tree. $BATS_TEST_TMPDIR/sys
+mk_amd() { # gpu busy vram_used vram_total power_uw_or_blank temp_mc
+    local card=$1 busy=$2 used=$3 total=$4 power=$5 temp=$6
+    local d="$SYS/card$card/device"
+    mkdir -p "$d/hwmon/hwmon9"
+    printf '%s\n' "$busy" > "$d/gpu_busy_percent"
+    printf '%s\n' "$used" > "$d/mem_info_vram_used"
+    printf '%s\n' "$total" > "$d/mem_info_vram_total"
+    printf '%s\n' "$power" > "$d/power1_average"          # blank => absent on R9700
+    [ -n "$temp" ] && printf '%s\n' "$temp" > "$d/hwmon/hwmon9/temp1_input"
+    printf 'DRIVER=amdgpu\nPCI_ID=1002:7551\nPCI_SLOT_NAME=0000:03:00.0\n' > "$d/uevent"
+}
+
+setup() {
+    SYS="$BATS_TEST_TMPDIR/sys"
+    mkdir -p "$SYS"
+}
+
+@test "exporter exists and compiles" {
+    [ -f "$EXPORTER" ]
+    python3 -m py_compile "$EXPORTER"
+}
+
+@test "amd card with full sysfs yields util/vram/temp/power with the right labels and units" {
+    mk_amd 1 42 1073741824 34359738368 250000000 71000   # 250 W, 71 C
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
+    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+    # util is a raw percent, VRAM verbatim bytes, temp milli->C, power microW->W.
+    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="amd"[^}]*\} 42$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*\} 1073741824$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_memory_total_bytes\{[^}]*\} 34359738368$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_temperature_celsius\{[^}]*\} 71$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_power_watts\{[^}]*\} 250$' || { echo "$output"; return 1; }
+    # identity labels: the PCI slot is a stable unique id, vendor marks the family.
+    echo "$output" | grep -Eq 'gpu_utilization_percent\{.*gpu="0000:03:00\.0"' || { echo "$output"; return 1; }
+}
+
+@test "card with an empty power file and no hwmon yields util/vram but omits power and temp" {
+    mk_amd 1 99 34108338176 34208743424 "" ""           # the R9700 shape
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
+    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*\} 99$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*\} 34108338176$'
+    # Absent means ABSENT, not a zero the dashboard would plot as a real reading.
+    ! echo "$output" | grep -Eq '^gpu_power_watts\{' || { echo "fabricated power line: $output"; return 1; }
+    ! echo "$output" | grep -Eq '^gpu_temperature_celsius\{' || { echo "fabricated temp line: $output"; return 1; }
+}
+
+@test "nvidia csv yields util/vram/temp/power tagged vendor=nvidia" {
+    nvidia="$BATS_TEST_TMPDIR/nvidia-smi"
+    cat > "$nvidia" <<'STUB'
+echo '0, NVIDIA GeForce RTX 3090, 93, 23004, 24576, 76, 397.40'
+STUB
+    chmod +x "$nvidia"
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd "$nvidia"
+    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="nvidia"[^}]*\} 93$' || { echo "$output"; return 1; }
+    # memory.used MiB -> bytes (23004 * 1048576)
+    echo "$output" | grep -Eq 'gpu_memory_used_bytes\{[^}]*vendor="nvidia"[^}]*\} 24121442304$' || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_temperature_celsius\{[^}]*vendor="nvidia"[^}]*\} 76$'
+    echo "$output" | grep -Eq 'gpu_power_watts\{[^}]*vendor="nvidia"[^}]*\} 397\.4$'
+    echo "$output" | grep -Eq 'model="NVIDIA GeForce RTX 3090"'
+}
+
+@test "a failing nvidia command is tolerated: amd still emits and exit is 0" {
+    mk_amd 1 50 1000000 2000000 "" 40000
+    failing="$BATS_TEST_TMPDIR/nvidia-fail"
+    printf '#!/bin/sh\nexit 1\n' > "$failing"; chmod +x "$failing"
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd "$failing"
+    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+    echo "$output" | grep -Eq 'gpu_utilization_percent\{[^}]*vendor="amd"[^}]*\} 50$'
+    ! echo "$output" | grep -q 'vendor="nvidia"'
+}
+
+@test "no GPUs at all yields only HELP/TYPE headers and exits 0" {
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
+    [ "$status" -eq 0 ] || { echo "$output"; return 1; }
+    # families are declared even when empty, so the dashboard/queries stay valid.
+    echo "$output" | grep -q '# TYPE gpu_utilization_percent gauge'
+    echo "$output" | grep -q '# TYPE gpu_memory_used_bytes gauge'
+    ! echo "$output" | grep -Eq '^gpu_utilization_percent\{'
+}
+
+@test "output is valid Prometheus text" {
+    mk_amd 1 42 1000 2000 "" 50000
+    run python3 "$EXPORTER" --amdgpu-sysfs "$SYS" --nvidia-cmd false
+    [ "$status" -eq 0 ]
+    # every non-comment line is "<name>{labels} value" or "<name> value"
+    printf '%s\n' "$output" | python3 -c '
+import sys, re
+pat = re.compile(r"^[a-z_]+(\{[^}]*\})? [-+0-9.eE+]+$")
+for i, l in enumerate(sys.stdin):
+    l = l.rstrip("\n")
+    if not l or l.startswith("#"):
+        continue
+    assert pat.match(l), "malformed line %d: %r" % (i, l)
+print("prometheus text ok")
+'
+}
+
+# ---- deployment + wiring contracts -------------------------------------
+# The exporter is a HOST-PROCESS user unit (not a quadlet): it must read the
+# host's own DRM sysfs and shell to nvidia-smi, which a container could not do
+# without GPU passthrough. These tests pin the cross-file wiring a reader
+# cannot eyeball — a lone change on one side leaves the metric silently absent.
+
+@test "the base unit runs the exporter from the checkout and never binds a wildcard" {
+    local u="$REPO_ROOT/scripts/gpu-exporter.service"
+    [ -f "$u" ]
+    grep -q '^ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/gpu-exporter.py --serve$' "$u" \
+        || { echo "unit does not run the exporter --serve from the checkout"; return 1; }
+    grep -q '^Restart=always$' "$u"
+    grep -q '^WantedBy=default.target$' "$u"
+    # Base unit binds loopback only; the per-host LAN address is added by a
+    # deploy drop-in, so a checked-in unit is never a wildcard.
+    grep -q '^Environment=GPU_EXPORTER_BINDS=127.0.0.1$' "$u"
+    ! grep -qE '0\.0\.0\.0' "$u"
+}
+
+@test "the deployer installs, drops in the LAN bind, and health-gates gpu-exporter" {
+    local handler="$REPO_ROOT/scripts/deploy-remote.sh"
+    grep -q '^deploy_gpu_exporter() {' "$handler"
+    # Per-host LAN from $TARGET (never a hardcoded host, never a wildcard).
+    grep -q 'dankclaw) lan=192.168.0.143' "$handler"
+    grep -q 'dankserver) lan=192.168.0.235' "$handler"
+    # Drop-in binds loopback + this host's LAN as one comma list.
+    grep -q "GPU_EXPORTER_BINDS=127.0.0.1,%s" "$handler"
+    grep -q 'systemctl --user enable gpu-exporter.service' "$handler"
+    grep -q 'systemctl --user restart gpu-exporter.service' "$handler"
+    # Health gate waits on the loopback metrics endpoint.
+    grep -q 'http://127.0.0.1:9401/metrics' "$handler"
+    # Dispatched from the central case.
+    grep -q 'gpu-exporter) deploy_gpu_exporter ;;' "$handler"
+    # Allowlisted on BOTH hosts.
+    grep -q 'dankclaw:gpu-exporter' "$handler"
+    grep -q 'dankserver:gpu-exporter' "$handler"
+}
+
+@test "the planner selects gpu-exporter for its files on each target" {
+    local t f
+    for t in dankclaw dankserver; do
+        for f in scripts/gpu-exporter.py scripts/gpu-exporter.service; do
+            run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target "$t" --paths "$f"
+            [ "$status" -eq 0 ]
+            [ "$output" = "gpu-exporter" ] || { echo "$t/$f -> '$output'"; return 1; }
+        done
+    done
+}
+
+@test "Prometheus scrapes gpu-exporter where node/cadvisor are reachable on each host" {
+    # Cross-file contract, same posture as node-exporter: the local instance is
+    # reached via the podman gateway (this Prometheus is a bridge container and
+    # a host-process bind is refused at the host LAN IP); the Dankserver
+    # instance is a different machine, so plain LAN routing reaches it. If either
+    # address drifts from the exporter's bind, GPU panels go blank with no error.
+    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
+    grep -q 'job_name: "gpu-dankclaw"' "$cfg" || { echo "no gpu-dankclaw job"; return 1; }
+    grep -q 'job_name: "gpu-dankserver"' "$cfg" || { echo "no gpu-dankserver job"; return 1; }
+    grep -q '"host.containers.internal:9401"' "$cfg" \
+        || { echo "gpu-dankclaw does not scrape the podman gateway"; return 1; }
+    grep -q '"192.168.0.235:9401"' "$cfg" \
+        || { echo "gpu-dankserver does not scrape the LAN address"; return 1; }
+    # A vhost target would be red forever (Prometheus cannot solve Anubis).
+    ! grep -qE '"[^"]*franksdank[^"]*:9401"' "$cfg"
+}
+
+@test "the host-system dashboard graphs the gpu_* families the exporter emits" {
+    local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/host-system.json"
+    [ -f "$dash" ]
+    python3 -c "
+import json
+d = json.load(open('$dash'))
+exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
+for want in ('gpu_utilization_percent', 'gpu_memory_used_bytes',
+             'gpu_temperature_celsius', 'gpu_power_watts'):
+    assert any(want in e for e in exprs), ('missing', want, exprs)
+print('gpu dashboard panels ok')
+"
+}

tests/observability_quadlet.bats

@@ -21,7 +21,7 @@ load helpers
 
 UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
 REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
-UNITS=(prometheus grafana loki alloy node-exporter alertmanager)
+UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager)
 
 # container_section <unit> — the [Container] key/value block only
 container_section() {
@@ -33,7 +33,7 @@ service_section() {
     awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
 }
 
-@test "all six observability units exist under dankclaw/observability/quadlet/" {
+@test "all seven observability units exist under dankclaw/observability/quadlet/" {
     local svc
     for svc in "${UNITS[@]}"; do
         [ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
@@ -277,6 +277,7 @@ loki http://127.0.0.1:3100/ready
 alloy http://127.0.0.1:12345/-/healthy
 node-exporter http://127.0.0.1:9100/metrics
 alertmanager http://127.0.0.1:9093/-/healthy
+cadvisor http://127.0.0.1:8080/healthz
 PAIRS
 }
 
@@ -568,3 +569,72 @@ assert any('ALERTS' in e for e in exprs), exprs
     [ "$status" -eq 0 ]
     [ "$output" = "alertmanager" ]
 }
+
+@test "cadvisor exports container metrics only because of --cgroupns=host, and binds exactly its two consumers" {
+    # Podman starts every container in a private cgroup namespace, so a cadvisor
+    # without --cgroupns=host sees only its own cgroup root and exports a single
+    # series (measured 2026-10-20: 1 series before the flag, 996 after). That
+    # flag and the host /sys mount are the entire mechanism — pin them, plus the
+    # publish set: loopback (host probe + health gate) and this host's LAN
+    # address (Prometheus reaches a host-published port there). Never a wildcard.
+    local unit lan
+    for pair in "dankclaw/observability/quadlet:192.168.0.143" "dankserver/observability/quadlet:192.168.0.235"; do
+        unit="$REPO_ROOT/${pair%%:*}/cadvisor.container"
+        lan="${pair##*:}"
+        [ -f "$unit" ] || { echo "missing $unit"; return 1; }
+        grep -q -- '--cgroupns=host' "$unit" || { echo "$unit lacks --cgroupns=host"; return 1; }
+        grep -q -- '--pid=host' "$unit" || { echo "$unit lacks --pid=host"; return 1; }
+        grep -q '^Volume=/sys:/sys:ro$' "$unit" || { echo "$unit lacks the host /sys mount"; return 1; }
+        grep -q '^WantedBy=default.target$' "$unit" || { echo "$unit is not boot-persistent"; return 1; }
+        grep -q '^Restart=always$' "$unit" || { echo "$unit is not self-healing"; return 1; }
+        grep -q '^HealthCmd=' "$unit" || { echo "$unit has no health check"; return 1; }
+        grep -q "^PublishPort=127.0.0.1:8080:8080\$" "$unit" || { echo "$unit must publish loopback :8080"; return 1; }
+        grep -q "^PublishPort=$lan:8080:8080\$" "$unit" || { echo "$unit must publish $lan:8080"; return 1; }
+        ! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$unit" || { echo "$unit publishes on a wildcard"; return 1; }
+    done
+}
+
+@test "Prometheus scrapes cadvisor where it is published and relabels the cgroup path to a container name" {
+    # Cross-file contract: cadvisor publishes the LAN port; a cadvisor scrape
+    # job pointing anywhere else reads nothing (unlike node-exporter's host-net
+    # bind, a published port IS reachable from this bridge container at the LAN
+    # address). And the whole dashboard depends on the id -> container relabel:
+    # cadvisor names nothing itself, the quadlet unit name is only in the cgroup
+    # path. If either half changes alone, the Containers dashboard is blank.
+    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
+    grep -q 'job_name: "cadvisor-dankclaw"' "$cfg" || { echo "no cadvisor-dankclaw job"; return 1; }
+    grep -q 'job_name: "cadvisor-dankserver"' "$cfg" || { echo "no cadvisor-dankserver job"; return 1; }
+    grep -q '"192.168.0.143:8080"' "$cfg" || { echo "cadvisor-dankclaw does not scrape the published LAN port"; return 1; }
+    grep -q '"192.168.0.235:8080"' "$cfg" || { echo "cadvisor-dankserver does not scrape the published LAN port"; return 1; }
+    # The relabel that turns …/app.slice/<name>.service… into a `container`.
+    grep -q 'app\\.slice/(\[\^\.\]+)\\.service' "$cfg" || { echo "no cgroup-path -> container relabel"; return 1; }
+    grep -q 'target_label: container' "$cfg" || { echo "relabel does not set the container label"; return 1; }
+}
+
+@test "the Containers dashboard is provisioned from git" {
+    local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/containers.json"
+    [ -f "$dash" ]
+    python3 -c "
+import json
+d = json.load(open('$dash'))
+assert d['uid'] == 'containers', d['uid']
+names = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
+# Only families cadvisor actually exports for rootless quadlet containers are
+# graphed; network/fs are absent for them, so a network/fs panel would be a
+# permanent empty lie.
+assert any('container_cpu_usage_seconds_total' in e for e in names), names
+assert any('container_memory_working_set_bytes' in e for e in names), names
+assert not any('container_network' in e or 'container_fs_' in e for e in names), names
+"
+}
+
+@test "the planner selects cadvisor for its unit on each target" {
+    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
+        --paths dankclaw/observability/quadlet/cadvisor.container
+    [ "$status" -eq 0 ]
+    [ "$output" = "cadvisor" ]
+    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
+        --paths dankserver/observability/quadlet/cadvisor.container
+    [ "$status" -eq 0 ]
+    [ "$output" = "cadvisor" ]
+}