Merge pull request 'feat(observability): per-container metrics via cadvisor + Containers dashboard' (#58) from feat/containers-observability into main

Per-container CPU/memory was invisible for the quadlet app stack — Grafana could
only see the host (node_exporter) and the LLM engines. This adds a rootless
cadvisor quadlet on each host and a `containers` dashboard.

cadvisor works on this rootless-Podman / cgroup-v2 host on one condition:
--cgroupns=host. Podman starts every container in a private cgroup namespace, so
without it cadvisor sees only its own cgroup root and exports a single series
(measured 2026-10-20: 1 series off, 996 on). The libpod socket does not name the
rootless app containers (it resolved 2 of 34), so naming comes from elsewhere:
the quadlet unit name is carried in the cgroup path
(app.slice/<name>.service/libpod-payload-<hex>), and Prometheus relabels it into
a `container` label. No podman socket needed.

cadvisor cannot bind per-interface like node_exporter, so each unit is a bridge
container publishing exactly 127.0.0.1:8080 and the host LAN address — never a
wildcard. Prometheus (a bridge container) reaches a host-published port at the
LAN address, so cadvisor-dankclaw/dankserver scrape 192.168.0.143:8080 /
192.168.0.235:8080. Only the families cadvisor actually exports for these
containers are graphed — CPU usage, memory working-set/limit, OOM — not network
or filesystem, which are absent under the host-PID layout.

Wired through the deployer (allowlist, deploy_observability case + /healthz gate,
dispatcher), the planner (both targets), and observability_quadlet.bats.

dankclaw/observability/quadlet/cadvisor.container

@@ -0,0 +1,56 @@
+# Quadlet unit: cadvisor (rootless, boot-persistent) — DANKCLAW.
+#
+# Install path on dankclaw: ~/.config/containers/systemd/cadvisor.container
+# Unit name: cadvisor.service (container name: cadvisor)
+#
+# WHY THIS EXISTS. The app stack is ~34 rootless Podman containers and until
+# now nothing reported per-container CPU/memory — Grafana could only see the
+# host (node_exporter) and the LLM engines. cadvisor reads the host cgroup
+# tree and exports container_cpu_* / container_memory_* per container.
+#
+# THE LOAD-BEARING FLAG IS --cgroupns=host. Podman starts every container in a
+# PRIVATE cgroup namespace, so an exporter container sees only its OWN cgroup
+# subtree: cadvisor exported a single root series until this flag was added,
+# then it walked the whole tree (996 per-container series measured 2026-10-20).
+# The quadlet unit name is carried in the cgroup path — app.slice/<name>.service/
+# libpod-payload-<hex> — so Prometheus relabels that path into a `container`
+# label (shared/prometheus/prometheus.yml, cadvisor-dankclaw job) and no
+# libpod/podman socket is needed to name containers.
+#
+# BINDING. cadvisor has no per-interface listen flag (unlike node_exporter's
+# --web.listen-address), so it binds 0.0.0.0 INSIDE its own network namespace.
+# The bridge PublishPort is what keeps the unauthenticated endpoint off the
+# host's other interfaces: it publishes exactly loopback (host-local probes and
+# the deployer's health gate) and this host's LAN address (Prometheus, a bridge
+# container, reaches a host-published port at its LAN address — the same netavark
+# rule node-exporter relies on, only here the port really is published).
+#
+# It is rootless on purpose: dank owns the app containers' cgroups, so a
+# rootless cadvisor reads every one of them; root would buy nothing here.
+
+[Unit]
+Description=cAdvisor container metrics (Dankclaw)
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+ContainerName=cadvisor
+Image=gcr.io/cadvisor/cadvisor:v0.49.1
+PodmanArgs=--pid=host --cgroupns=host
+PublishPort=127.0.0.1:8080:8080
+PublishPort=192.168.0.143:8080:8080
+Volume=/sys:/sys:ro
+Volume=/:/rootfs:ro
+Exec=--housekeeping_interval=30s
+HealthCmd=wget --spider -q http://localhost:8080/healthz
+HealthInterval=30s
+HealthTimeout=10s
+HealthRetries=3
+HealthStartPeriod=40s
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target

dankserver/observability/quadlet/cadvisor.container

@@ -0,0 +1,39 @@
+# Quadlet unit: cadvisor (rootless, boot-persistent) — DANKSERVER.
+#
+# Install path on dankserver: ~/.config/containers/systemd/cadvisor.container
+# Unit name: cadvisor.service (container name: cadvisor)
+#
+# The Dankserver twin of dankclaw/observability/quadlet/cadvisor.container.
+# Same rootless shape, same --cgroupns=host requirement (see that file for the
+# full story), only the LAN address differs: this host publishes on
+# 192.168.0.235:8080 so Prometheus on Dankclaw can scrape it over the real
+# network (cross-host, so the host.containers.internal trick does not apply).
+# It reports the media/library stack (romm, plex, jellyfin, copyparty, calibre,
+# tautulli) by their quadlet unit names via the cgroup path.
+
+[Unit]
+Description=cAdvisor container metrics (Dankserver)
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+ContainerName=cadvisor
+Image=gcr.io/cadvisor/cadvisor:v0.49.1
+PodmanArgs=--pid=host --cgroupns=host
+PublishPort=127.0.0.1:8080:8080
+PublishPort=192.168.0.235:8080:8080
+Volume=/sys:/sys:ro
+Volume=/:/rootfs:ro
+Exec=--housekeeping_interval=30s
+HealthCmd=wget --spider -q http://localhost:8080/healthz
+HealthInterval=30s
+HealthTimeout=10s
+HealthRetries=3
+HealthStartPeriod=40s
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target

scripts/deploy-plan.py

@@ -39,6 +39,7 @@ SERVICE_ORDER = (
     "loki",
     "alloy",
     "node-exporter",
+    "cadvisor",
     "alertmanager",
     "authentik-postgres",
     "authentik-server",
@@ -92,6 +93,7 @@ DANKCLAW_SERVICES = {
     "loki",
     "alloy",
     "node-exporter",
+    "cadvisor",
     "alertmanager",
     "authentik-postgres",
     "authentik-server",
@@ -124,6 +126,7 @@ DANKSERVER_SERVICES = {
     "readarr",
     "frank-archon",
     "node-exporter",
+    "cadvisor",
 }
 
 # FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on
@@ -374,7 +377,7 @@ def services_for_path(path: str, target: str) -> set[str]:
             # consumer (prometheus before grafana, loki before alloy).
             if len(parts) >= 4 and parts[2] == "quadlet":
                 name = parts[3].removesuffix(".container")
-                if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "alertmanager"}:
+                if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "cadvisor", "alertmanager"}:
                     _add(services, name)
             return services
         if len(parts) >= 2 and parts[1] == "gitea":
@@ -417,8 +420,8 @@ def services_for_path(path: str, target: str) -> set[str]:
             # node-dankserver job had nothing to scrape.
             if len(parts) >= 4 and parts[2] == "quadlet":
                 name = parts[3].removesuffix(".container")
-                if name == "node-exporter":
+                if name in {"node-exporter", "cadvisor"}:
-                    _add(services, "node-exporter")
+                    _add(services, name)
             return services
         # frank-archon's quadlet-era directory left the tree on 2026-09-26
         # (the Lucebox DS4.1 unit was retired for the llama.cpp root unit —

scripts/deploy-remote.sh

@@ -86,7 +86,7 @@ service_is_allowed() {
         dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
         dankclaw:comfyui|dankclaw:hermes|\
         dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
-        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\
+        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:alertmanager|\
         dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
         dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
@@ -97,7 +97,7 @@ service_is_allowed() {
         dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
         dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
         dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
-        dankserver:frank-archon|dankserver:node-exporter|\
+        dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|\
         frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
         frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
         frankbook-pro:readarr)
@@ -520,6 +520,8 @@ deploy_observability() {
             ;;
         node-exporter)
             ;;
+        cadvisor)
+            ;;
         *)
             echo "unknown observability service: $service" >&2
             return 2
@@ -533,6 +535,7 @@ deploy_observability() {
         grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
         loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
         node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
+        cadvisor) wait_for_health cadvisor http://127.0.0.1:8080/healthz 90 ;;
     esac
 }
 
@@ -2000,7 +2003,7 @@ deploy_service() {
         plex) deploy_plex ;;
         tautulli) deploy_tautulli ;;
         alloy) deploy_alloy ;;
-        prometheus|grafana|loki|node-exporter|alertmanager) deploy_observability "$service" ;;
+        prometheus|grafana|loki|node-exporter|cadvisor|alertmanager) deploy_observability "$service" ;;
         comfyui) deploy_comfyui ;;
         *) deploy_compose_service "$service" ;;
     esac

scripts/frank-archon.env

@@ -46,35 +46,31 @@ FRANK_ARCHON_TP=2
 # stays EXACT for single-sequence decode and any step of <= 25 rows — see
 # MAX_SEQS x (1 + SPEC_TOKENS) below.
 FRANK_ARCHON_TP_WIRE=wht6
-# Decode slots. A slot costs KV and pinned RAM, never decode: the cache target is
+# Decode slots. A slot costs KV, never decode; the pool is sized to this
-# exactly MAX_SEQS x MAX_MODEL_LEN and the checkpoint store asks (1 + what the
+# contract and nothing more. At 1.8.1 the budget resolver caps the cache at
-# retention policy keeps) per slot. Ordered 2026-10-20: FOUR slots over a pool of at
+# 556,348 fp8 tokens — 2.8 sessions at 200,000 — so a THIRD full-height
-# least 700,000 fp8 tokens, rounding UP if the resolver cannot land it exactly. The
+# conversation answers 400 while /health and every shorter chat work fine
-# target is therefore 4 x 200,000 = 800,000 tokens and HOST_POOL_MIB decides whether
+# (3 x 50k streams run at 77.6 t/s aggregate). Frank's trade: declare the two
-# the cache gets it (expert_vram_floor, core/place/planner.cpp:100-104: the card must
+# slots the engine can hold and let the freed ~1.1 GiB go to the expert plane,
-# keep expert-plane-minus-host-room). The 2-slot shape was the answer to a
+# instead of advertising a third slot that only exists at shorter prompts.
-# 365,204-token pool that backed 1.8 full-height sessions; at this pool the third and
+FRANK_ARCHON_MAX_SEQS=2
-# fourth conversations are backed by the cache instead of over-committing it.
-FRANK_ARCHON_MAX_SEQS=4
 # Served window. The engine treats `kv group tokens >= max-model-len` as a HARD
 # limit, and models.yml contextWindow MUST quote this number (over-length is a
 # refusal, not a truncation). The pool is sized for seqs x maxlen, not one
 # sequence.
 FRANK_ARCHON_MAX_MODEL_LEN=200000
 FRANK_ARCHON_PLACEMENT=expert_tiered
-# Pinned host pool for expert rows that do not fit VRAM, PINNED ONCE PER RANK: the
+# Pinned host pool for expert rows that do not fit VRAM. Under the budget
-# host must pin tp x this number, and the refusal that names it is
+# resolver this is ALSO the lever that decides which experts stay on the card:
-# core/engine_bringup.cpp:250-265. Under the budget resolver this is ALSO the KV
+# the card's leftover goes to the expert plane first and the KV cache takes the
-# lever -- the card keeps expert-plane-minus-this-room, so every MiB added moves one
+# rest, so growing the pool moves experts off the card and shrinking it spills
-# MiB of experts off the card and into the cache. Derived from the boot ledger (a
+# them onto VRAM the cache wanted. Under-sizing it is a boot failure (36,864
-# 23.73 GiB floor at 44,928 room puts plane + 2 units at 67.60 GiB a rank), 49,152
+# refused a boot). The ledger's coverage line is the referee, not this file.
-# drops the floor to 19.6 GiB and leaves the cache ~6.9 GiB, which is enough to back
+# Measured, not guessed: 45,568 (PR #30) moved host_pool to 45.09 GiB and left
-# the whole 4 x 200,000 target; the leftover returns to the experts. Cost is paid
+# vram_kv byte-identical at 4.20 GiB / 556,348 tokens on both ranks — extra
-# twice over: pinned RAM (tp x 48 GiB, against the TTM target below) and expert
+# host RAM above what the expert plane's footprint needs buys no cache. Back at
-# residency -- ~3.2 GiB a card of experts now lives in host RAM and is read over the
+# 44,928, the size that boots clean.
-# x8 link when the router picks it. The ledger's coverage line is the referee, not
+FRANK_ARCHON_HOST_POOL_MIB=44928
-# this file: if vram_kv comes back under 700,000 tokens, raise this 1024 and redeploy.
-FRANK_ARCHON_HOST_POOL_MIB=49152
 # VRAM left unclaimed per card (publisher default 96 MiB); the engine's settle
 # floor is built from it.
 FRANK_ARCHON_GPU_HEADROOM_MIB=96
@@ -135,10 +131,9 @@ FRANK_ARCHON_NGRAM_PLACEMENT=ram
 # RESETS pages_limit to half of RAM on every boot; below the plan the load dies
 # mid-pin with an anonymous allocator error. ttm-preflight.sh raises the knob
 # itself before refusing by name, so recovery is a boot, not a visit.
-# Floor = 2 x HOST_POOL + 48,824 (static weights + arena) + PREFIX_CACHE + MNBT + the
+# Floor = 2 x HOST_POOL + 48,824 (static weights + arena) + PREFIX_CACHE + MNBT
-# auto checkpoint store (MAX_SEQS x 10 slots of 111.98 MiB), per the bats ledger test;
+# per the bats ledger test; at the 44,928 pool that computes to 148,920 MiB.
-# at the 49,152 pool with 4 slots that computes to 161,848 MiB.
+FRANK_ARCHON_TTM_MIN_MIB=150000
-FRANK_ARCHON_TTM_MIN_MIB=162000
 FRANK_ARCHON_TTM_TARGET_MIB=204800
 # Escape hatch for experiments. Unset -> no extra argv; the serve script expands
 # this unquoted on purpose, so keep multi-flag values as one string.

scripts/probe-frank-archon-kv.py

@@ -147,21 +147,10 @@ def main():
     deep = corpus(160_000)
     wide = corpus(1_100_000)
 
-    # The corpus's chars/token ratio is MEASURED here, not stored. A stored 3.28
+    # 3.28 chars/token is the measured ratio of this corpus through the
-    # (646,000 chars -> 196,919 tokens, 2026-10-11) had drifted to 3.20 by
+    # froggeric template (646,000 chars -> 196,919 tokens at the 2026-10-11
-    # 2026-10-20 (646,299 chars -> 201,876 tokens) as the repo's docs grew, and a
+    # deploy); 98.5 % of the window leaves the response room under the ceiling.
-    # stale constant sends leg B past --max-model-len, where the engine answers a
+    chars = int(0.985 * maxlen * 3.28)
-    # perfectly legitimate 400 ("the request (201900 tokens) exceeds the available
-    # context size (200000)") and the leg reads as a capacity failure. One small
-    # request prices the ratio; 97 % of the window leaves the response room.
-    sample = prompt(wide, 4000, "CAL", "cal-marker")
-    cal, _, _ = post({"model": MODEL,
-                      "messages": [{"role": "user", "content": sample}],
-                      "max_tokens": 1, "temperature": 0.0,
-                      "chat_template_kwargs": {"enable_thinking": False}})
-    ratio = len(sample) / cal["usage"]["prompt_tokens"]
-    print(f"[cal] corpus ratio {ratio:.2f} chars/token")
-    chars = int(0.97 * maxlen * ratio)
     # Each marker's FIRST hyphen-separated word is unique, because show()
     # scores the hit on that word — a shared prefix would score every slot HIT
     # on any answer that says the word.
@@ -187,7 +176,7 @@ def main():
                 print(f"[B-{s}] FAILED after {time.time() - t0:.1f}s: {e!r}")
     print(f"[B] total {time.time() - t0:.1f}s")
     # D — concurrency decode: seqs streams, 512 tokens each, ~25 % depth.
-    dchars = int(0.25 * maxlen * ratio)
+    dchars = int(0.25 * maxlen * 3.28)
     dbodies = [chat(deep, dchars, f"D{i + 1}-9137", f"flint-marker-{i}", 512)
                for i in range(seqs)]
     print(f"[D] {seqs} concurrent streams x 512 tokens, "

scripts/serve-frank-archon.sh

@@ -15,9 +15,9 @@
 # log states every budget before anything is allocated (weights vs KV
 # split, the expert host pool's coverage of the worst case, the PLE pin,
 # the P2P answer per pair) — read it after every resize.
-# Deployed shape (2026-10-20, radiance 1.8.1): FOUR decode slots x 200,000 tokens over a
+# Deployed shape (2026-10-19, radiance 1.8.1): TWO decode slots x 200,000 tokens,
-# >= 700,000-token fp8 KV pool, 48 GiB/rank pinned expert pool, MTP depth 2, prefix cache
+# fp8 KV, 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache in pinned host
-# in pinned host RAM, PLE pinned in RAM. The per-card split between the expert slab and the KV
+# RAM, PLE pinned in RAM. The per-card split between the expert slab and the KV
 # pool is NO LONGER an operator number — 1.3.0 removed the VRAM pair and the budget
 # resolver derives it from headroom, the cache bound and the host pool, so the slab
 # size and the pool's token count are LEDGER lines. After every boot read them, plus
@@ -27,9 +27,9 @@
 # pooled experts are read synchronously) — and the cache's session backing.
 #
 # The root unit (scripts/frank-archon.service, User=dank,
-# LimitMEMLOCK=infinity) grants the pin budget: 2 x 48 GiB expert pools + the 47.7 GiB PLE
+# LimitMEMLOCK=infinity) grants the pin budget: 2 x 44 GiB expert pools +
-# table + 8 GiB prefix host tier + the checkpoint store (slots x 112 MiB) — ~159 GiB of
+# the 47.7 GiB PLE table + 8 GiB prefix host tier — ~147 GiB of pinned
-# pinned pages, which is ALSO why this script refuses to start when ttm's
+# pages, which is ALSO why this script refuses to start when ttm's
 # pages_limit/pages_max cap is under FRANK_ARCHON_TTM_MIN_MIB (see below).
 set -euo pipefail
 

serverAppConfigs-docs/Infrastructure/Observability.md

@@ -10,9 +10,9 @@ timestamp: 2026-10-20
 
 ## Summary
 
-Six containers and one host-side process: Prometheus (metrics), Grafana
+Seven containers and one host-side process: Prometheus (metrics), Grafana
 (dashboards), Loki (log store), Alloy (log shipper), node-exporter (host
-metrics) and Alertmanager (alert delivery), with `signal-relay` on the host
+metrics), cadvisor (per-container metrics) and Alertmanager (alert delivery), with `signal-relay` on the host
 carrying every firing group to Signal. All of it is Quadlet-managed,
 boot-persistent and loopback-only, deployed by
 `scripts/deploy-remote.sh` → `deploy_observability`. The metrics half works
@@ -34,6 +34,7 @@ that proves it today.
 | Loki | `localhost/franksdank/loki:latest` | `127.0.0.1:3100` | `loki.container` | [[Loki]] |
 | Alloy | `localhost/franksdank/alloy:latest` | `127.0.0.1:12345` | `alloy.container` | [[Alloy]] |
 | node-exporter | `docker.io/prom/node-exporter:latest` | host netns: `127.0.0.1:9100` + `192.168.0.143:9100` | `node-exporter.container` | [[Node-Exporter]] |
+| cadvisor | `gcr.io/cadvisor/cadvisor:v0.49.1` | bridge publish `127.0.0.1:8080` + `192.168.0.143:8080` | `cadvisor.container` | — |
 | Signal relay | host `python3`, not a container | `192.168.0.143:18093` | `scripts/signal-relay.service` (user unit) | [[Alertmanager]] |
 
 Every unit lives in `dankclaw/observability/quadlet/`, carries
@@ -44,9 +45,10 @@ linger is on for `dank`). `signal-relay.service` reports `enabled` + `active`
 the same day, and `homepage-status.timer` — the root unit that probes all of
 these on loopback — is `enabled` + `active` (`deploy-remote.sh:319-321` copies
 the probe registry `shared/homepage/services.json` to `/srv/homepage`).
-Dankserver runs one member of this family, node-exporter, as its own unit
+Dankserver runs two members of this family, node-exporter and cadvisor, as
-(`dankserver/observability/quadlet/node-exporter.container`, `generated` +
+its own units (`dankserver/observability/quadlet/{node-exporter,cadvisor}.container`,
-`active`, binding `127.0.0.1:9100` + `192.168.0.235:9100`).
+`generated` + `active`; node-exporter binds `127.0.0.1:9100` + `192.168.0.235:9100`,
+cadvisor publishes `127.0.0.1:8080` + `192.168.0.235:8080`).
 
 ## Exposure
 
@@ -98,10 +100,23 @@ verify it through the APIs.
   relay, and `journalctl --user -u signal-relay` records successful
   `sent to +12062476336:` lines for every rule in the set.
 * **Dashboards: working.** Provisioned datasources — Prometheus (uid
-  `prometheus`) and Loki (uid `loki`) — and the `llm-fleet` dashboard, all
+  `prometheus`) and Loki (uid `loki`) — and the `llm-fleet`, `host-system`,
+  `containers`, `observability` and `logs-overview` dashboards, all
   installed from `shared/grafana/provisioning/`. The Loki datasource is how
   you read logs: Loki has no web UI of its own, so Grafana Explore is the
   human surface (the guarded `loki.franksdank.cloud` vhost serves only the API).
+* **Per-container metrics: the `containers` dashboard (added 2026-10-20; live
+  after `deploy.sh <host> --service cadvisor`).** cadvisor is the only way to see
+  per-container CPU/memory here, and on this rootless-Podman host it works on one
+  condition: `--cgroupns=host`. Podman starts every container in a PRIVATE cgroup
+  namespace, so without that flag cadvisor sees only its own cgroup root and
+  exports a single series (measured 2026-10-20: 1 series off, 996 on). cadvisor
+  names nothing itself — the libpod socket resolved only 2 of 34 rootless
+  containers — but the quadlet unit name is carried in the cgroup path
+  (`app.slice/<name>.service/…`), so `prometheus.yml` relabels that path into a
+  `container` label. Network and filesystem are NOT exported for these containers
+  (no netns/classid attribution under the host-PID layout), so the dashboard
+  graphs only CPU, memory and OOM and does not fake the rest.
 * **Logs: fixed in code, awaiting deploy.** The pipeline shipped nothing because
   a rootless Alloy cannot read the `0640` journal files, and the container-log
   globs point at a root-storage layout that does not exist here. The fix is built
@@ -117,7 +132,7 @@ verify it through the APIs.
 
 ## Citations
 
-- `dankclaw/observability/quadlet/{prometheus,alertmanager,grafana,loki,alloy,node-exporter}.container`
+- `dankclaw/observability/quadlet/{prometheus,alertmanager,grafana,loki,alloy,node-exporter,cadvisor}.container`
 - `scripts/deploy-remote.sh:219-300` (copiers + relay), `:521-536` (health waits), `:477-486` (grafana env seed)
 - `scripts/signal-relay.py:26-49`, `scripts/signal-relay.service`
 - `shared/prometheus/prometheus.yml`, `shared/prometheus/alertmanager.yml`, `shared/prometheus/rules/llm-fleet.yml`

serverAppConfigs-docs/LLM/frank-archon.md

@@ -88,30 +88,25 @@ on stop. Boot budget 1800 s (the publisher's compose allows a 15-minute `start_p
 | Knob (env name, `FRANK_ARCHON_` dropped) | Live value |
 |---|---|
 | TP / wire | `TP=2`, `TP_WIRE=wht6` — libr4d one-shot all-reduce, not RCCL; `min-kb` stays the engine default |
-| Slots / window | `MAX_SEQS=4`, `MAX_MODEL_LEN=200000` (native 262,144; longer is refused, never truncated) — this pair is the **cache bound**: the resolver sizes the KV carve against `slots × window`, and no slot count funds itself (Laws) |
+| Slots / window | `MAX_SEQS=2`, `MAX_MODEL_LEN=200000` (native 262,144; longer is refused, never truncated) — this pair is the **cache bound**, not a KV allocation |
 | KV dtype | `KV_DTYPE=fp8` — the only dtype that funds the slot count (Laws) |
 | VRAM split | **derived at boot** from `GPU_HEADROOM_MIB=96` + the cache bound + `HOST_POOL_MIB`; naming `VRAM_WEIGHTS_MIB`/`VRAM_KV_MIB` is a boot refusal (removed at 1.3.0) |
-| Expert tier | `PLACEMENT=expert_tiered`, `HOST_POOL_MIB=49152` **per rank** — the pinned host pool is also the **KV lever**: the expert floor is `plane + 2 units − host room`, so every MiB pinned there lowers what the experts claim on the card one-for-one and the cache takes it (Laws) |
+| Expert tier | `PLACEMENT=expert_tiered`, `HOST_POOL_MIB=44928` **per rank** |
 | PLE | `NGRAM_PLACEMENT=ram` (47.7 GiB pinned; refuses boot on a short memlock) |
 | Spec / batch | `SPEC_TOKENS=2` — MTP depth 2, the `.rad` carries the model's own `mtp.*` head (`mtp.fc_hidden`/`fc_embedding`/`draft_head`); `MAX_BATCHED_TOKENS=2048` (MNBT — 2560 collapses decode, Laws) |
 | Prefix cache | `PREFIX_CACHE_MIB=8192`, pinned **host tier only** — the 64 GiB disk tier is gone, so an evicted prefix is re-prefilled; `STATE_DIR=/home/dank/radiance-state` now exists only to hold the rad-tune cache |
 | Template | `TEMPLATE=scripts/qwen38-froggeric-v22.5.jinja` (28,234 B) via `--override-chat-template` — froggeric's fixed Qwen3.8 template (the repo root's `chat_template.jinja`, vendored); the checkpoint's own template is baked into the `.rad` by rad-convert and this overrides it, and radiance derives the reasoning and tool-call reply format from it |
 | Step sharing | `BUSY_PREFILL_CHUNK=auto` — prefill-speed first: it reserves 3 tokens per waiting decoder and ⅛ of the budget per waiting request, and does **not** shrink the chunk for decoders alone · `PREFILL_INTERLEAVE=1` — the decoder dial, **armed 2026-10-20**: one decode-only step after each chunk step taken beside decoders; meter `engine_steps_held_total`, price is the prefilling prompt's TTFT |
-| TTM gate | `TTM_MIN_MIB=162000`, `TTM_TARGET_MIB=204800` |
+| TTM gate | `TTM_MIN_MIB=150000`, `TTM_TARGET_MIB=204800` |
 | Boot | `RAD_MIN_MIB=110000` deployer floor; the listener binds only after the full load (health ~100 s, budget 1800 s) |
 
-Boot ledger for the live shape (2026-10-20, pin 1.8.1, `MAX_SEQS=4`, `HOST_POOL_MIB=49152`):
+Boot ledger for the live shape (2026-10-19 11:10, pin 1.8.1, `MAX_SEQS=2`, `HOST_POOL_MIB=44928`):
-the resolver prints `elastic 26.73 GiB` → **experts 20.72 GiB (78 %) / kv cache 5.64 GiB (21 %)**,
+the resolver prints `elastic 26.73 GiB` → **experts 23.73 GiB (89 %) / kv cache 2.76 GiB (10 %)**,
-`prefill staging 373.65 MiB` out of the cache, `vram_weights 24.30 GiB`, `host_pool 48.59 GiB`.
+`vram_weights 27.31 GiB`, `host_pool 44.47 GiB`. `kv_attn` (13 layers, 4-token blocks) pages
-The referee line: `kv: the cache can grow to 5.64 GiB a card — 747,492 tokens of context, 3.7
+**26.00 KiB** → 91,301 blocks = **365,204 tokens = 1.8 full-length sessions = 91 % of the
-sessions at 200,000 — with every block it lends the experts taken back; 440.04 MiB of that is
+2 × 200,000 worst case** (the resolver never funds 100 %, see Laws); `mover` lends 2,146 slots
-per-sequence state, and up to 5.20 GiB is lent to the experts while no request holds it`.
+(2.54 GiB) out of the cache to the expert slab; sampler **9 rows**, template 28,234 B, vision live
-`--checkpoint-slots 40` is auto (`max_num_seqs × 10` slots of 111.98 MiB = 4,479 MiB of pinned
+(≤ 4,194,304 px, 16,384 patches/pass).
-host), `/health` reports `queue_cap 32` (`max(8, seqs × 8)`), sampler **12 rows**
-(= `seqs × (1 + speculative depth)`), 4 hardware queues a card, template 28,234 B, vision live
-(≤ 4,194,304 px, 16,384 patches/pass), P2P confirmed (`the 2 cards reach each other's memory
-directly`). Experts living in the pinned host pool: **8.99 GiB a rank** (was 5.99) — the price of
-the bigger cache, paid over PCIe when those experts are hit.
 
 The boot **ledger**, not this arithmetic, is the referee: radiance states every budget before
 anything is allocated (weights vs KV, the host pool's coverage of the worst case, the PLE pin,
@@ -150,7 +145,6 @@ marked "re-fetch" is gone from `/home/dank/models/`, which today holds only `Sti
 
 | Window | Payload | Runtime | Shape | Replaced because | Rollback |
 |---|---|---|---|---|---|
-| 2026-10-19 → 10-20 | same `qwen3.8-next-flash-fp8-iq4r-moe.rad` (payload unchanged) | radiance **1.8.1**, `sha256:d590e300…` (same digest) | 2 × 200,000 fp8 KV — resolver cache 2.76 GiB = 365,204 tokens (91 % backing); host pool 44,928 MiB a rank, `TTM_MIN_MIB=150000`, `queue_cap 16`, sampler 9 rows; `--prefill-interleave 1` armed on 10-20 | Frank ordered **four** concurrent conversations over a ≥ 700,000-token pool; the resolver funds that only by lowering the expert floor with more pinned host room (Laws) | revert `1bb6399` and redeploy `scripts/deploy.sh dankserver --service frank-archon` — trio back to `MAX_SEQS=2` / `HOST_POOL_MIB=44928` / `TTM_MIN_MIB=150000` |
 | 2026-10-15 → 10-19 | same `qwen3.8-next-flash-fp8-iq4r-moe.rad` (payload unchanged) | radiance **1.0.13**, `sha256:4c3f4845…` | 3 × 200,000 fp8 KV bought with the **explicit** pair `--vram-weights-mib 25600 --vram-kv-mib 5120`; host pool 44,928 → 45,568 MiB a rank; prefix cache 8 GiB host + 64 GiB disk | decode turned to sludge beside any long prompt — step contention, 18.7 ms decode-only steps against 317 ms steps carrying a prefill chunk (Laws) — and the only two knobs that fix it, `--busy-prefill-chunk` and `--prefill-interleave`, exist only after this pin | `FRANK_ARCHON_IMAGE` back to `sha256:4c3f4845…` **plus a trio rewrite**: restore the pair and the host+disk prefix knobs, delete `BUSY_PREFILL_CHUNK`/`PREFILL_INTERLEAVE`. A digest swap alone is a boot refusal (Runtime) |
 | 2026-10-08 → 10-15 | in-house `qwen4exp-swift15-w4dn8-i8-hc8m.rad`, 116,416 MiB (Swift1.5 abliterated BF16, GPTQ w4 experts + 24 bf16 protected, KLD 0.0131 / top-1 97.92 %) | radiance 1.0.13, same digest | 3 × 200,000 fp8 KV, slab 25,600 / pool 5,120 — the shape the live file still carries | the engine author's own container is the supported path and his quality ceiling; the finetune's refusal-bending left the endpoint | `.rad` purged 2026-10-17 (trash-staged then emptied, Frank's approval). Re-download `frank8465/Swift-1.5-Qwen3.8-Flash-Next-GPTQ-W4A8-Radiance-AMD-R9700` → `qwen4exp-w4nl64-i8-hc8m.rad`, 122,071,973,888 B, LFS sha256 `09dd9306…`, via the iq4r fetch script's hfget/verify pattern retargeted at that repo; point `MODEL_FILE` at it, redeploy. Re-**convert** is dead: the BF16 source was `rm`'d 2026-10-15 (`kld-ref`, grams, corpora and `/home/dank/logs/swift15_*.log` survive) |
 | 2026-10-05 → 10-08 | `qwen4exp-fp8-native.rad`, 177,109 MiB (173.0 GiB) — native modelopt block-FP8, no requantisation | radiance 1.0.7 line, same digest | — | native FP8 experts (56.4 GiB/rank) ate a 40 GiB pool and left KV at 3,543 MiB = 386,904 tokens, **1.93 sessions** — context economics, not quality; and the host pool could not be trimmed to pay for it (36,864 refused the boot) | rebuild-only: `scripts/download-frank-archon.sh` (official `Qwen/Qwen3.8-Flash-Next-FP8`, 131 shards, 172.8 GB, ungated) + `scripts/build-frank-archon-rad.sh` (no-recipe convert under the descriptor law); raw **and** `.rad` were `rm`'d 2026-10-15 (Frank's call, to free the volume for the iq4r download) |
@@ -217,44 +211,27 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
   never `--kv-cache-size-mib`; with the disk tier engaged the engine also forced ~9.6 GiB of weight
   buffers + arena onto the weights side (~29.6 GiB/card effective). **Neither flag exists at 1.8.1:**
   the same ceiling is enforced by the resolver, and passing the pair is a boot refusal.
-- **At 1.8.1 the split belongs to the resolver, and the source fixes its order.**
+- **At 1.8.1 the split belongs to the resolver, and it funds ~90 % of the worst case, never 100 %.**
-  `core/mem/vram_budget.cpp` claims the card bottom-up (`capacity → already held → drift → arena →
+  The engine takes `--gpu-headroom-mib`, the cache bound (`--max-model-len` × `--max-num-seqs`) and
-  staging → queues → prefetch → headroom = claimable`), takes the **static weights off the top as a
+  `--host-pool-mib`, prints `claimable → elastic → experts / kv cache` before allocating anything, and
-  floor, not a share**, and divides the `elastic` remainder in three lines (146-156):
+  sizes the expert plane from the model's own weight footprint plus slack — not from a number typed by
+  hand. Three boots of evidence:
 
-  ```cpp
+  | shape | demand | cache given | backing |
-  exp = std::min(b.expert_floor, b.elastic);
-  kv  = std::min(kv_cap, std::max(b.elastic - exp, kv_one));
-  exp = std::min(std::max(b.expert_ceiling, b.expert_floor), b.elastic - kv);
-  ```
-
-  `kv_cap` is `KVManager::ceiling()` = `state + Σ_paged max_seqs × ceil(max_ctx / block) × bytes`
-  (`core/mem/kv.cpp:266`), so `--max-num-seqs` and `--max-model-len` set the cache's **demand**, never
-  its funding. The funding is `elastic − expert_floor`, and the floor is
-  `min(plane + spare, plane + spare + 2·unit_max − host_room)` (`core/place/planner.cpp:100-104`, with
-  `host_room = host_pool − file_stage_reserve` at `core/engine_bringup.cpp:1418-1422`). **That is the
-  cache's lever: every MiB added to `--host-pool-mib` drops the expert floor one-for-one and the cache
-  takes it** — minus `stage`, the prefill stager's ring (two layers of the experts left off the card),
-  which is charged back out of the cache whenever the cache lends (`vram_budget.cpp:193-203`). Boots of
-  evidence, all on 1.8.1:
-
-  | shape | demand | cache given | engine's `backs %` |
   |---|---|---|---|
-  | 3 × 200,000, host pool 44,928 | 600,000 tok | 4.20 GiB | 93 % |
+  | 3 × 200,000, host pool 44,928 | 600,000 tok | 4.20 GiB | 93 % (2.8 sessions) |
   | 3 × 200,000, host pool 45,568 | 600,000 tok | **4.20 GiB — byte-identical** | 93 % |
-  | 2 × 200,000, host pool 44,928 | 400,000 tok | 2.76 GiB | 91 % |
+  | 2 × 200,000, host pool 44,928 (live) | 400,000 tok | 2.76 GiB = 365,204 tok | 91 % (1.8 sessions) |
-  | 4 × 200,000, host pool 49,152 (live) | 800,000 tok | **5.64 GiB = 747,492 tok** | 93 % |
 
-  The 45,568 row is the ceiling case, not a mystery: at 3 slots the cache already held its whole
+  So `--max-num-seqs` is a **demand signal** for the resolver and an admission cap for the scheduler —
-  `ceiling`, so extra host room could not grow it and shows up as `unclaimed`. **The referee is the
+  never a KV allocation — and no slot count fully funds the worst case. Growing host RAM by less than a
-  ledger line `kv: the pool backs N % … the cache can grow to … N tokens`, never arithmetic.** To buy
+  slab step (640 MiB) moved nothing: the expert plane's claim decides what is left for KV, so cache is
-  window: raise `--host-pool-mib` (cost: pinned host RAM, and experts evicted to host cross PCIe —
+  grown through slots × window or the dtype, and experts are grown by not leaving elastic pool idle.
-  `experts-from-host` was 5.99 GiB a rank at 44,928 and is 8.99 at 49,152) or cut the per-token cost
+  `--host-pool-mib` is still load-bearing the other way — it is where the expert rows the cards cannot
-  (`fp8` KV). Raising slots raises demand **and** costs 10 × 111.98 MiB of pinned checkpoint-store RAM
+  hold get pinned, and under-sizing it has always been a boot failure (36,864 refused a boot in the fp8
-  per slot. Under-sizing the pool the other way has always been a boot failure (36,864 refused a boot in
+  era). The two-slot boot's own behavioural check: three concurrent conversations all answer `200` —
-  the fp8 era: the host cannot hold what the cards will not). Behavioural check on the live shape:
+  the third **queues** (`queue_cap: 16`), it is not refused; the 400 class is a single request over
-  `/health` reports `queue_cap 32`; a request beyond the slot count queues rather than being refused,
+  `--max-model-len`, or three concurrent full-height conversations against a 1.8-session cache.
-  and the 400 class is a single request over `--max-model-len`.
 - **Idle KV VRAM is the expert cache's spillover — the price is paid twice.** `mover: 2275
   slots can be lent out of the KV cache (2.69 GiB)`, `arena: lends 740 MiB while steps are <=
   256 tokens`; the lend grows with the pool (2.69 GiB from 3.46, 6.08 from 6.912, 4.22 in the pair era)
@@ -389,18 +366,17 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
   `--checkpoint-interval` does **not** buy the deep case: the coarsest ladder scale is the largest
   `interval × 2^k` under the depth, so the deepest gap stays ≈ half the prompt (129,024 tokens at
   interval 2048 against 130,816 at 256) while every cut multiplies the snapshot writes.
-- **Slots cost KV, host RAM, rows, and pin — never speed.** `--max-num-seqs` carries no radiance
+- **Slots cost KV, rows, and window — never speed.** `--max-num-seqs` carries no radiance constraint: it
-  constraint: it adds ≈110 MiB of recurrent state per slot, 10 × 111.98 MiB of pinned checkpoint-store
+  adds ≈110 MiB of recurrent state per slot (the live `kv_full_backing` arithmetic prices them exactly),
-  RAM per slot (`--checkpoint-slots auto`), and declares the fused TP all-reduce over
+  declares the fused TP all-reduce over `seqs × (1 + spec)` rows, and `wht6` is bit-exact up to 25 rows
-  `seqs × (1 + spec)` rows — `wht6` is bit-exact up to 25 rows, and the live shape is 4 × 3 = **12**, so
+  (single-sequence decode and any step of ≤ 25 rows is exact; only busier steps are lossy) — 2 × 3 = 6
-  single-sequence decode stays exact and so does any step up to that many rows. Raising it also raises
+  today. Measured 2026-10-06: seqs 2 and seqs 3 are statistically tied (30.9-33.1 t/s, inside the ±3 %
-  the cache's *demand* (`ceiling = slots × window`), which the resolver funds from `elastic −
+  floor). Measured 2026-10-19 at 1.8.1: the third slot's real price is **KV, not rows** — dropping it
-  expert_floor` — so a slot count without matching host room is paid for by expert residency. Measured
+  moved the resolver's cache from 4.20 GiB to 2.76 GiB and the backing from 2.8 to 1.8 sessions, while
-  2026-10-06: seqs 2 and 3 statistically tied (30.9-33.1 t/s, inside the ±3 % floor). Measured
+  the ≈4 GiB of host RAM it freed came back unbudgeted, landing in elastic pool that turned into a bigger
-  2026-10-19 → 10-20 at 1.8.1: 2 → 4 slots cost **+4,224 MiB of pinned host pool** and **−3.01 GiB of
+  mover lend instead of cache (resolver law above). The old "seqs 2 cannot boot" law belonged to the
-  expert residency a card** and bought 2.76 → 5.64 GiB of cache (365,204 → 747,492 tokens). The old
+  retired tcclaviger fork's `torch.compile` shape specialization, and radiance booted seqs 2 on its first
-  "seqs 2 cannot boot" law belonged to the retired tcclaviger fork's `torch.compile` shape
+  fp8-native start.
-  specialization, and radiance booted seqs 2 on its first fp8-native start.
 - **A dequantize-to-compute path bills host RAM twice.** On the EMULATION backend an offload request
   of **118 GiB** landed as **~238 GiB** of real host RAM — the dequantized copy rides on top of the
   pinned weights, and the doubling is invisible in the weight count. Read the engine log for
@@ -594,13 +570,13 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
   — at 64 tokens you get empty `content` and the colour only inside `reasoning_content`.
 - **Capacity behaviour:** a prompt over `--max-model-len` is refused, never truncated; a request
   longer than the KV pool's token count is accepted then refused as too large to fit; a request
-  beyond `--max-num-seqs` is held by the HTTP port (`queue_cap` = `max(8, 8 × slots)` — **32** here)
+  beyond `--max-num-seqs` is held by the HTTP port (`queue_cap`, default 8 × slots = 16 here) and
-  and answered 503 only past that, never paged or squeezed. The cache funds **3.7** full-height
+  answered 503 only past that, never paged or squeezed. Because the cache funds **1.8** full-height
-  sessions, so four conversations can be resident but a fourth near-full-height one can still
+  sessions, admitting a second near-full-height conversation can **preempt a running sequence from
-  **preempt a running sequence from the back** to free its blocks
+  the back** to free its blocks (`core/sched/scheduler.cpp:1291-1322`); the victim is recomputed, so
-  (`core/sched/scheduler.cpp:1291-1322`); the victim is recomputed, so its session halts and then
+  its session halts and then re-prefills. Meter: `vllm:num_preemptions_total` (264 in the current
-  re-prefills. Meter: `vllm:num_preemptions_total` (cumulative for the boot, `/stats preemptions`);
+  boot, all in prefill-coincident bursts — 108 inside 105 s); `/stats preemptions` is the cumulative
-  `radiance:num_requests_preempted` is only the instant depth.
+  count and `radiance:num_requests_preempted` only the instant depth.
 - **Observability:** `/metrics` carries vLLM-compatible names plus `radiance:*`, scraped by the
   Dankclaw Prometheus job at `192.168.0.235:7770` (15 s interval) over plain LAN routing (the
   netavark published-port quirk is same-host only). **Latency lives under `vllm:`, cadence under

shared/grafana/provisioning/dashboards/containers.json

@@ -0,0 +1,137 @@
+{
+  "uid": "containers",
+  "title": "Containers — Dankclaw & Dankserver",
+  "description": "Per-container CPU and memory for the quadlet app stacks, from cadvisor (a rootless quadlet on each host). cadvisor keys each series by the cgroup path, and Prometheus relabels the quadlet unit name (app.slice/<name>.service) into a `container` label. Panels use only the metric families cadvisor actually exports for these rootless containers: CPU usage and memory working-set/limit/OOM. Network and filesystem are NOT exported for them (no netns/classid attribution under the rootless+host-PID layout), so no panels pretend otherwise. See serverAppConfigs-docs/Infrastructure/Observability.md.",
+  "tags": ["fleet", "cadvisor", "containers"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "templating": {
+    "list": [
+      {
+        "name": "host",
+        "label": "Host",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\"}, host)", "refId": "host_var" },
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": { "text": "All", "value": "$__all" }
+      },
+      {
+        "name": "container",
+        "label": "Container",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\"}, container)", "refId": "container_var" },
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": { "text": "All", "value": "$__all" }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "row",
+      "title": "CPU",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+      "panels": []
+    },
+    {
+      "id": 2,
+      "type": "timeseries",
+      "title": "CPU usage (cores)",
+      "description": "CPU-seconds/s per container = cores busy. A steady high line is a busy service; a spike is a burst (a render, an index, a backup). Only containers are shown, not the host or its daemons.",
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 1 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
+      "targets": [
+        {
+          "expr": "sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
+          "legendFormat": "{{host}} / {{container}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "type": "bargauge",
+      "title": "Top containers by CPU (cores)",
+      "description": "Instant ranking over the selected window. Tells you which container to look at when the host is loaded.",
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": null, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 4 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true },
+      "targets": [
+        {
+          "expr": "topk(12, sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m])))",
+          "legendFormat": "{{host}} / {{container}}",
+          "format": "table",
+          "instant": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "type": "row",
+      "title": "Memory",
+      "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 },
+      "panels": []
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Memory working set",
+      "description": "container_memory_working_set_bytes per container — the number the OOM killer acts on. Cache-inclusive and reclaimable; a flat high line is resident data, a sawtooth is churn.",
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 11 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
+      "targets": [
+        {
+          "expr": "container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}",
+          "legendFormat": "{{host}} / {{container}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "Memory vs limit (%)",
+      "description": "Working set over the cgroup memory limit (container_spec_memory_limit_bytes). Only containers that carry a limit are shown; approaching 1.0 means an OOM is near. Containers with no limit are omitted.",
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 11 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
+      "targets": [
+        {
+          "expr": "(container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} / on (job, host, container) group_left container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}) and (container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} > 0)",
+          "legendFormat": "{{host}} / {{container}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 7,
+      "type": "timeseries",
+      "title": "OOM kills",
+      "description": "container_oom_events_total rate. Any non-zero line means the kernel killed a process inside that container — the honest signal that its memory limit is too tight for its real working set.",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 20, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] },
+      "targets": [
+        {
+          "expr": "sum by (host, container) (rate(container_oom_events_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
+          "legendFormat": "{{host}} / {{container}}",
+          "refId": "A"
+        }
+      ]
+    }
+  ]
+}

shared/prometheus/prometheus.yml

@@ -131,3 +131,63 @@ scrape_configs:
       - targets: ["192.168.0.235:7770"]
         labels:
           service: frank-archon
+
+  # Per-container metrics from cadvisor (a rootless quadlet on each host).
+  # cadvisor keys every series by the cgroup path; for a quadlet container that
+  # path is …/app.slice/<name>.service[/libpod-payload-<hex>], so the unit NAME
+  # lives in it. The relabel below pulls that name out into a `container` label
+  # and keeps only the per-unit aggregate — cadvisor's raw host root,
+  # system.slice daemons and machine.slice scopes carry no such name and are
+  # dropped, as is the per-CPU-core split (only cpu="total" survives), so the
+  # scrape stays small. cadvisor cannot bind per-interface, so it is reached at
+  # the host's PUBLISHED LAN port: unlike node-exporter's host-net bind (which
+  # is why node-dankclaw scrapes via host.containers.internal), a published port
+  # IS reachable from this bridge container at the host LAN address.
+  - job_name: "cadvisor-dankclaw"
+    static_configs:
+      - targets: ["192.168.0.143:8080"]
+        labels:
+          host: dankclaw
+    metric_relabel_configs:
+      - source_labels: [id]
+        regex: '.*/app\.slice/([^.]+)\.service(?:/.*)?$'
+        target_label: container
+        replacement: '$1'
+      # Keep only the unit-level cgroup; drop its payload/control children so a
+      # container is not counted twice.
+      - source_labels: [id]
+        regex: '.*/\.service/.*'
+        action: drop
+      # Drop everything that is not a named unit (no `container` was extracted).
+      - source_labels: [container]
+        regex: '.+'
+        action: keep
+      # cpu="total" for cpu series; metrics with no cpu label (memory, network,
+      # filesystem) are kept.
+      - source_labels: [cpu]
+        regex: '^$|total'
+        action: keep
+      - regex: 'id|name|image|container_label.*|machine_id|boot_id'
+        action: labeldrop
+
+  - job_name: "cadvisor-dankserver"
+    static_configs:
+      - targets: ["192.168.0.235:8080"]
+        labels:
+          host: dankserver
+    metric_relabel_configs:
+      - source_labels: [id]
+        regex: '.*/app\.slice/([^.]+)\.service(?:/.*)?$'
+        target_label: container
+        replacement: '$1'
+      - source_labels: [id]
+        regex: '.*/\.service/.*'
+        action: drop
+      - source_labels: [container]
+        regex: '.+'
+        action: keep
+      - source_labels: [cpu]
+        regex: '^$|total'
+        action: keep
+      - regex: 'id|name|image|container_label.*|machine_id|boot_id'
+        action: labeldrop

tests/frank_archon_unit.bats

@@ -145,7 +145,7 @@ nosudo_bin() {
     # (torch.compile query_start_loc constraint, 2026-10-02); radiance booted
     # seqs 2 in the fp8-native era and the 2026-10-06 sweep had seqs 2 and 3
     # tied inside the +-3% noise floor — slots buy concurrency, not speed.
-    grep -q '^FRANK_ARCHON_MAX_SEQS=4$' "$ENV_FILE"
+    grep -q '^FRANK_ARCHON_MAX_SEQS=2$' "$ENV_FILE"
     grep -q '^FRANK_ARCHON_TP=2$' "$ENV_FILE"
     grep -q '^FRANK_ARCHON_TP_WIRE=wht6$' "$ENV_FILE"
     grep -q '^FRANK_ARCHON_SPEC_TOKENS=2$' "$ENV_FILE"
@@ -158,7 +158,7 @@ nosudo_bin() {
     # so the pool's token count is a boot-ledger line — verified live by the
     # capacity leg of scripts/probe-frank-archon-kv.py at every cutover.
     grep -q '^FRANK_ARCHON_PLACEMENT=expert_tiered$' "$ENV_FILE"
-    grep -q '^FRANK_ARCHON_HOST_POOL_MIB=49152$' "$ENV_FILE"
+    grep -q '^FRANK_ARCHON_HOST_POOL_MIB=44928$' "$ENV_FILE"
     grep -q '^FRANK_ARCHON_KV_DTYPE=fp8$' "$ENV_FILE"
     grep -q '^FRANK_ARCHON_NGRAM_PLACEMENT=ram$' "$ENV_FILE"
     # Prefix cache: pinned host RAM only (the disk tier was removed), and the
@@ -177,11 +177,8 @@ nosudo_bin() {
     [ "$ttmt" -gt "$ttmm" ]
     pool=$(awk -F= '/^FRANK_ARCHON_HOST_POOL_MIB=/{print $2}' "$ENV_FILE")
     ph=$(awk -F= '/^FRANK_ARCHON_PREFIX_CACHE_MIB=/{print $2}' "$ENV_FILE")
-    # 2 pools + PLE 47.7 GiB (48,824 MiB) + prefix host + MNBT + the auto
+    # 2 pools + PLE 47.7 GiB (48,824 MiB) + prefix host, rounded up.
-    # checkpoint store (MAX_SEQS x 10 slots of 112 MiB -- the store is pinned host
+    ledger=$(( 2 * pool + 48824 + ph + 2048 ))
-    # memory too, and it grows with the slot count).
-    seqs=$(awk -F= '/^FRANK_ARCHON_MAX_SEQS=/{print $2}' "$ENV_FILE")
-    ledger=$(( 2 * pool + 48824 + ph + 2048 + seqs * 10 * 112 ))
     [ "$ttmm" -ge "$ledger" ]
     # The .rad size floor is positive (a truncated convert must fail fast).
     grep -q '^FRANK_ARCHON_RAD_MIN_MIB=' "$ENV_FILE"

tests/observability_quadlet.bats

@@ -21,7 +21,7 @@ load helpers
 
 UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
 REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
-UNITS=(prometheus grafana loki alloy node-exporter alertmanager)
+UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager)
 
 # container_section <unit> — the [Container] key/value block only
 container_section() {
@@ -33,7 +33,7 @@ service_section() {
     awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
 }
 
-@test "all six observability units exist under dankclaw/observability/quadlet/" {
+@test "all seven observability units exist under dankclaw/observability/quadlet/" {
     local svc
     for svc in "${UNITS[@]}"; do
         [ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
@@ -277,6 +277,7 @@ loki http://127.0.0.1:3100/ready
 alloy http://127.0.0.1:12345/-/healthy
 node-exporter http://127.0.0.1:9100/metrics
 alertmanager http://127.0.0.1:9093/-/healthy
+cadvisor http://127.0.0.1:8080/healthz
 PAIRS
 }
 
@@ -568,3 +569,72 @@ assert any('ALERTS' in e for e in exprs), exprs
     [ "$status" -eq 0 ]
     [ "$output" = "alertmanager" ]
 }
+
+@test "cadvisor exports container metrics only because of --cgroupns=host, and binds exactly its two consumers" {
+    # Podman starts every container in a private cgroup namespace, so a cadvisor
+    # without --cgroupns=host sees only its own cgroup root and exports a single
+    # series (measured 2026-10-20: 1 series before the flag, 996 after). That
+    # flag and the host /sys mount are the entire mechanism — pin them, plus the
+    # publish set: loopback (host probe + health gate) and this host's LAN
+    # address (Prometheus reaches a host-published port there). Never a wildcard.
+    local unit lan
+    for pair in "dankclaw/observability/quadlet:192.168.0.143" "dankserver/observability/quadlet:192.168.0.235"; do
+        unit="$REPO_ROOT/${pair%%:*}/cadvisor.container"
+        lan="${pair##*:}"
+        [ -f "$unit" ] || { echo "missing $unit"; return 1; }
+        grep -q -- '--cgroupns=host' "$unit" || { echo "$unit lacks --cgroupns=host"; return 1; }
+        grep -q -- '--pid=host' "$unit" || { echo "$unit lacks --pid=host"; return 1; }
+        grep -q '^Volume=/sys:/sys:ro$' "$unit" || { echo "$unit lacks the host /sys mount"; return 1; }
+        grep -q '^WantedBy=default.target$' "$unit" || { echo "$unit is not boot-persistent"; return 1; }
+        grep -q '^Restart=always$' "$unit" || { echo "$unit is not self-healing"; return 1; }
+        grep -q '^HealthCmd=' "$unit" || { echo "$unit has no health check"; return 1; }
+        grep -q "^PublishPort=127.0.0.1:8080:8080\$" "$unit" || { echo "$unit must publish loopback :8080"; return 1; }
+        grep -q "^PublishPort=$lan:8080:8080\$" "$unit" || { echo "$unit must publish $lan:8080"; return 1; }
+        ! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$unit" || { echo "$unit publishes on a wildcard"; return 1; }
+    done
+}
+
+@test "Prometheus scrapes cadvisor where it is published and relabels the cgroup path to a container name" {
+    # Cross-file contract: cadvisor publishes the LAN port; a cadvisor scrape
+    # job pointing anywhere else reads nothing (unlike node-exporter's host-net
+    # bind, a published port IS reachable from this bridge container at the LAN
+    # address). And the whole dashboard depends on the id -> container relabel:
+    # cadvisor names nothing itself, the quadlet unit name is only in the cgroup
+    # path. If either half changes alone, the Containers dashboard is blank.
+    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
+    grep -q 'job_name: "cadvisor-dankclaw"' "$cfg" || { echo "no cadvisor-dankclaw job"; return 1; }
+    grep -q 'job_name: "cadvisor-dankserver"' "$cfg" || { echo "no cadvisor-dankserver job"; return 1; }
+    grep -q '"192.168.0.143:8080"' "$cfg" || { echo "cadvisor-dankclaw does not scrape the published LAN port"; return 1; }
+    grep -q '"192.168.0.235:8080"' "$cfg" || { echo "cadvisor-dankserver does not scrape the published LAN port"; return 1; }
+    # The relabel that turns …/app.slice/<name>.service… into a `container`.
+    grep -q 'app\\.slice/(\[\^\.\]+)\\.service' "$cfg" || { echo "no cgroup-path -> container relabel"; return 1; }
+    grep -q 'target_label: container' "$cfg" || { echo "relabel does not set the container label"; return 1; }
+}
+
+@test "the Containers dashboard is provisioned from git" {
+    local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/containers.json"
+    [ -f "$dash" ]
+    python3 -c "
+import json
+d = json.load(open('$dash'))
+assert d['uid'] == 'containers', d['uid']
+names = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
+# Only families cadvisor actually exports for rootless quadlet containers are
+# graphed; network/fs are absent for them, so a network/fs panel would be a
+# permanent empty lie.
+assert any('container_cpu_usage_seconds_total' in e for e in names), names
+assert any('container_memory_working_set_bytes' in e for e in names), names
+assert not any('container_network' in e or 'container_fs_' in e for e in names), names
+"
+}
+
+@test "the planner selects cadvisor for its unit on each target" {
+    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
+        --paths dankclaw/observability/quadlet/cadvisor.container
+    [ "$status" -eq 0 ]
+    [ "$output" = "cadvisor" ]
+    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
+        --paths dankserver/observability/quadlet/cadvisor.container
+    [ "$status" -eq 0 ]
+    [ "$output" = "cadvisor" ]
+}