docs(probe): price the corpus chars/token ratio at run time

The stored 3.28 chars/token (measured 2026-10-11) had drifted to 3.20 as the repo's docs grew, so leg B sent ~201,900-token prompts and took a legitimate 400 from the engine's own window check — which reads as a capacity-contract failure on a healthy endpoint. One ~4k-token request now prices the ratio, and the leg sizes to 97 % of the deployed window.
docs(vault): the four-slot ledger, and the resolver's funding law from the source
2026-10-20 23:26:53 -07:00 · 2026-10-20 23:26:53 -07:00
12 changed files with 102 additions and 450 deletions
@@ -1,56 +0,0 @@
-# Quadlet unit: cadvisor (rootless, boot-persistent) — DANKCLAW.
-#
-# Install path on dankclaw: ~/.config/containers/systemd/cadvisor.container
-# Unit name: cadvisor.service (container name: cadvisor)
-#
-# WHY THIS EXISTS. The app stack is ~34 rootless Podman containers and until
-# now nothing reported per-container CPU/memory — Grafana could only see the
-# host (node_exporter) and the LLM engines. cadvisor reads the host cgroup
-# tree and exports container_cpu_* / container_memory_* per container.
-#
-# THE LOAD-BEARING FLAG IS --cgroupns=host. Podman starts every container in a
-# PRIVATE cgroup namespace, so an exporter container sees only its OWN cgroup
-# subtree: cadvisor exported a single root series until this flag was added,
-# then it walked the whole tree (996 per-container series measured 2026-10-20).
-# The quadlet unit name is carried in the cgroup path — app.slice/<name>.service/
-# libpod-payload-<hex> — so Prometheus relabels that path into a `container`
-# label (shared/prometheus/prometheus.yml, cadvisor-dankclaw job) and no
-# libpod/podman socket is needed to name containers.
-#
-# BINDING. cadvisor has no per-interface listen flag (unlike node_exporter's
-# --web.listen-address), so it binds 0.0.0.0 INSIDE its own network namespace.
-# The bridge PublishPort is what keeps the unauthenticated endpoint off the
-# host's other interfaces: it publishes exactly loopback (host-local probes and
-# the deployer's health gate) and this host's LAN address (Prometheus, a bridge
-# container, reaches a host-published port at its LAN address — the same netavark
-# rule node-exporter relies on, only here the port really is published).
-#
-# It is rootless on purpose: dank owns the app containers' cgroups, so a
-# rootless cadvisor reads every one of them; root would buy nothing here.
-
-[Unit]
-Description=cAdvisor container metrics (Dankclaw)
-After=network-online.target
-Wants=network-online.target
-
-[Container]
-ContainerName=cadvisor
-Image=gcr.io/cadvisor/cadvisor:v0.49.1
-PodmanArgs=--pid=host --cgroupns=host
-PublishPort=127.0.0.1:8080:8080
-PublishPort=192.168.0.143:8080:8080
-Volume=/sys:/sys:ro
-Volume=/:/rootfs:ro
-Exec=--housekeeping_interval=30s
-HealthCmd=wget --spider -q http://localhost:8080/healthz
-HealthInterval=30s
-HealthTimeout=10s
-HealthRetries=3
-HealthStartPeriod=40s
-
-[Service]
-Restart=always
-RestartSec=10
-
-[Install]
-WantedBy=default.target
@@ -1,39 +0,0 @@
-# Quadlet unit: cadvisor (rootless, boot-persistent) — DANKSERVER.
-#
-# Install path on dankserver: ~/.config/containers/systemd/cadvisor.container
-# Unit name: cadvisor.service (container name: cadvisor)
-#
-# The Dankserver twin of dankclaw/observability/quadlet/cadvisor.container.
-# Same rootless shape, same --cgroupns=host requirement (see that file for the
-# full story), only the LAN address differs: this host publishes on
-# 192.168.0.235:8080 so Prometheus on Dankclaw can scrape it over the real
-# network (cross-host, so the host.containers.internal trick does not apply).
-# It reports the media/library stack (romm, plex, jellyfin, copyparty, calibre,
-# tautulli) by their quadlet unit names via the cgroup path.
-
-[Unit]
-Description=cAdvisor container metrics (Dankserver)
-After=network-online.target
-Wants=network-online.target
-
-[Container]
-ContainerName=cadvisor
-Image=gcr.io/cadvisor/cadvisor:v0.49.1
-PodmanArgs=--pid=host --cgroupns=host
-PublishPort=127.0.0.1:8080:8080
-PublishPort=192.168.0.235:8080:8080
-Volume=/sys:/sys:ro
-Volume=/:/rootfs:ro
-Exec=--housekeeping_interval=30s
-HealthCmd=wget --spider -q http://localhost:8080/healthz
-HealthInterval=30s
-HealthTimeout=10s
-HealthRetries=3
-HealthStartPeriod=40s
-
-[Service]
-Restart=always
-RestartSec=10
-
-[Install]
-WantedBy=default.target
@@ -39,7 +39,6 @@ SERVICE_ORDER = (
    "loki",
    "alloy",
    "node-exporter",
-    "cadvisor",
    "alertmanager",
    "authentik-postgres",
    "authentik-server",
@@ -93,7 +92,6 @@ DANKCLAW_SERVICES = {
    "loki",
    "alloy",
    "node-exporter",
-    "cadvisor",
    "alertmanager",
    "authentik-postgres",
    "authentik-server",
@@ -126,7 +124,6 @@ DANKSERVER_SERVICES = {
    "readarr",
    "frank-archon",
    "node-exporter",
-    "cadvisor",
 }

 # FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on
@@ -377,7 +374,7 @@ def services_for_path(path: str, target: str) -> set[str]:
            # consumer (prometheus before grafana, loki before alloy).
            if len(parts) >= 4 and parts[2] == "quadlet":
                name = parts[3].removesuffix(".container")
-                if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "cadvisor", "alertmanager"}:
+                if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "alertmanager"}:
                    _add(services, name)
            return services
        if len(parts) >= 2 and parts[1] == "gitea":
@@ -420,8 +417,8 @@ def services_for_path(path: str, target: str) -> set[str]:
            # node-dankserver job had nothing to scrape.
            if len(parts) >= 4 and parts[2] == "quadlet":
                name = parts[3].removesuffix(".container")
-                if name in {"node-exporter", "cadvisor"}:
-                    _add(services, name)
+                if name == "node-exporter":
+                    _add(services, "node-exporter")
            return services
        # frank-archon's quadlet-era directory left the tree on 2026-09-26
        # (the Lucebox DS4.1 unit was retired for the llama.cpp root unit —
@@ -86,7 +86,7 @@ service_is_allowed() {
        dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
        dankclaw:comfyui|dankclaw:hermes|\
        dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
-        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:alertmanager|\
+        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\
        dankclaw:authentik-postgres|dankclaw:authentik-server|\
        dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
        dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
@@ -97,7 +97,7 @@ service_is_allowed() {
        dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
        dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
        dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
-        dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|\
+        dankserver:frank-archon|dankserver:node-exporter|\
        frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
        frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
        frankbook-pro:readarr)
@@ -520,8 +520,6 @@ deploy_observability() {
            ;;
        node-exporter)
            ;;
-        cadvisor)
-            ;;
        *)
            echo "unknown observability service: $service" >&2
            return 2
@@ -535,7 +533,6 @@ deploy_observability() {
        grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
        loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
        node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
-        cadvisor) wait_for_health cadvisor http://127.0.0.1:8080/healthz 90 ;;
    esac
 }

@@ -2003,7 +2000,7 @@ deploy_service() {
        plex) deploy_plex ;;
        tautulli) deploy_tautulli ;;
        alloy) deploy_alloy ;;
-        prometheus|grafana|loki|node-exporter|cadvisor|alertmanager) deploy_observability "$service" ;;
+        prometheus|grafana|loki|node-exporter|alertmanager) deploy_observability "$service" ;;
        comfyui) deploy_comfyui ;;
        *) deploy_compose_service "$service" ;;
    esac
@@ -48,7 +48,7 @@ FRANK_ARCHON_TP=2
 FRANK_ARCHON_TP_WIRE=wht6
 # Decode slots. A slot costs KV and pinned RAM, never decode: the cache target is
 # exactly MAX_SEQS x MAX_MODEL_LEN and the checkpoint store asks (1 + what the
-# retention policy keeps) per slot. Ordered 2026-10-21: FOUR slots over a pool of at
+# retention policy keeps) per slot. Ordered 2026-10-20: FOUR slots over a pool of at
 # least 700,000 fp8 tokens, rounding UP if the resolver cannot land it exactly. The
 # target is therefore 4 x 200,000 = 800,000 tokens and HOST_POOL_MIB decides whether
 # the cache gets it (expert_vram_floor, core/place/planner.cpp:100-104: the card must
@@ -147,10 +147,21 @@ def main():
    deep = corpus(160_000)
    wide = corpus(1_100_000)

-    # 3.28 chars/token is the measured ratio of this corpus through the
-    # froggeric template (646,000 chars -> 196,919 tokens at the 2026-10-11
-    # deploy); 98.5 % of the window leaves the response room under the ceiling.
-    chars = int(0.985 * maxlen * 3.28)
+    # The corpus's chars/token ratio is MEASURED here, not stored. A stored 3.28
+    # (646,000 chars -> 196,919 tokens, 2026-10-11) had drifted to 3.20 by
+    # 2026-10-20 (646,299 chars -> 201,876 tokens) as the repo's docs grew, and a
+    # stale constant sends leg B past --max-model-len, where the engine answers a
+    # perfectly legitimate 400 ("the request (201900 tokens) exceeds the available
+    # context size (200000)") and the leg reads as a capacity failure. One small
+    # request prices the ratio; 97 % of the window leaves the response room.
+    sample = prompt(wide, 4000, "CAL", "cal-marker")
+    cal, _, _ = post({"model": MODEL,
+                      "messages": [{"role": "user", "content": sample}],
+                      "max_tokens": 1, "temperature": 0.0,
+                      "chat_template_kwargs": {"enable_thinking": False}})
+    ratio = len(sample) / cal["usage"]["prompt_tokens"]
+    print(f"[cal] corpus ratio {ratio:.2f} chars/token")
+    chars = int(0.97 * maxlen * ratio)
    # Each marker's FIRST hyphen-separated word is unique, because show()
    # scores the hit on that word — a shared prefix would score every slot HIT
    # on any answer that says the word.
@@ -176,7 +187,7 @@ def main():
                print(f"[B-{s}] FAILED after {time.time() - t0:.1f}s: {e!r}")
    print(f"[B] total {time.time() - t0:.1f}s")
    # D — concurrency decode: seqs streams, 512 tokens each, ~25 % depth.
-    dchars = int(0.25 * maxlen * 3.28)
+    dchars = int(0.25 * maxlen * ratio)
    dbodies = [chat(deep, dchars, f"D{i + 1}-9137", f"flint-marker-{i}", 512)
               for i in range(seqs)]
    print(f"[D] {seqs} concurrent streams x 512 tokens, "
@@ -15,7 +15,7 @@
 # log states every budget before anything is allocated (weights vs KV
 # split, the expert host pool's coverage of the worst case, the PLE pin,
 # the P2P answer per pair) — read it after every resize.
-# Deployed shape (2026-10-21, radiance 1.8.1): FOUR decode slots x 200,000 tokens over a
+# Deployed shape (2026-10-20, radiance 1.8.1): FOUR decode slots x 200,000 tokens over a
 # >= 700,000-token fp8 KV pool, 48 GiB/rank pinned expert pool, MTP depth 2, prefix cache
 # in pinned host RAM, PLE pinned in RAM. The per-card split between the expert slab and the KV
 # pool is NO LONGER an operator number — 1.3.0 removed the VRAM pair and the budget
@@ -10,9 +10,9 @@ timestamp: 2026-10-20

 ## Summary

-Seven containers and one host-side process: Prometheus (metrics), Grafana
+Six containers and one host-side process: Prometheus (metrics), Grafana
 (dashboards), Loki (log store), Alloy (log shipper), node-exporter (host
-metrics), cadvisor (per-container metrics) and Alertmanager (alert delivery), with `signal-relay` on the host
+metrics) and Alertmanager (alert delivery), with `signal-relay` on the host
 carrying every firing group to Signal. All of it is Quadlet-managed,
 boot-persistent and loopback-only, deployed by
 `scripts/deploy-remote.sh` → `deploy_observability`. The metrics half works
@@ -34,7 +34,6 @@ that proves it today.
 | Loki | `localhost/franksdank/loki:latest` | `127.0.0.1:3100` | `loki.container` | [[Loki]] |
 | Alloy | `localhost/franksdank/alloy:latest` | `127.0.0.1:12345` | `alloy.container` | [[Alloy]] |
 | node-exporter | `docker.io/prom/node-exporter:latest` | host netns: `127.0.0.1:9100` + `192.168.0.143:9100` | `node-exporter.container` | [[Node-Exporter]] |
-| cadvisor | `gcr.io/cadvisor/cadvisor:v0.49.1` | bridge publish `127.0.0.1:8080` + `192.168.0.143:8080` | `cadvisor.container` | — |
 | Signal relay | host `python3`, not a container | `192.168.0.143:18093` | `scripts/signal-relay.service` (user unit) | [[Alertmanager]] |

 Every unit lives in `dankclaw/observability/quadlet/`, carries
@@ -45,10 +44,9 @@ linger is on for `dank`). `signal-relay.service` reports `enabled` + `active`
 the same day, and `homepage-status.timer` — the root unit that probes all of
 these on loopback — is `enabled` + `active` (`deploy-remote.sh:319-321` copies
 the probe registry `shared/homepage/services.json` to `/srv/homepage`).
-Dankserver runs two members of this family, node-exporter and cadvisor, as
-its own units (`dankserver/observability/quadlet/{node-exporter,cadvisor}.container`,
-`generated` + `active`; node-exporter binds `127.0.0.1:9100` + `192.168.0.235:9100`,
-cadvisor publishes `127.0.0.1:8080` + `192.168.0.235:8080`).
+Dankserver runs one member of this family, node-exporter, as its own unit
+(`dankserver/observability/quadlet/node-exporter.container`, `generated` +
+`active`, binding `127.0.0.1:9100` + `192.168.0.235:9100`).

 ## Exposure

@@ -100,23 +98,10 @@ verify it through the APIs.
  relay, and `journalctl --user -u signal-relay` records successful
  `sent to +12062476336:` lines for every rule in the set.
 * **Dashboards: working.** Provisioned datasources — Prometheus (uid
-  `prometheus`) and Loki (uid `loki`) — and the `llm-fleet`, `host-system`,
-  `containers`, `observability` and `logs-overview` dashboards, all
+  `prometheus`) and Loki (uid `loki`) — and the `llm-fleet` dashboard, all
  installed from `shared/grafana/provisioning/`. The Loki datasource is how
  you read logs: Loki has no web UI of its own, so Grafana Explore is the
  human surface (the guarded `loki.franksdank.cloud` vhost serves only the API).
-* **Per-container metrics: the `containers` dashboard (added 2026-10-20; live
-  after `deploy.sh <host> --service cadvisor`).** cadvisor is the only way to see
-  per-container CPU/memory here, and on this rootless-Podman host it works on one
-  condition: `--cgroupns=host`. Podman starts every container in a PRIVATE cgroup
-  namespace, so without that flag cadvisor sees only its own cgroup root and
-  exports a single series (measured 2026-10-20: 1 series off, 996 on). cadvisor
-  names nothing itself — the libpod socket resolved only 2 of 34 rootless
-  containers — but the quadlet unit name is carried in the cgroup path
-  (`app.slice/<name>.service/…`), so `prometheus.yml` relabels that path into a
-  `container` label. Network and filesystem are NOT exported for these containers
-  (no netns/classid attribution under the host-PID layout), so the dashboard
-  graphs only CPU, memory and OOM and does not fake the rest.
 * **Logs: fixed in code, awaiting deploy.** The pipeline shipped nothing because
  a rootless Alloy cannot read the `0640` journal files, and the container-log
  globs point at a root-storage layout that does not exist here. The fix is built
@@ -132,7 +117,7 @@ verify it through the APIs.

 ## Citations

- `dankclaw/observability/quadlet/{prometheus,alertmanager,grafana,loki,alloy,node-exporter,cadvisor}.container`
+- `dankclaw/observability/quadlet/{prometheus,alertmanager,grafana,loki,alloy,node-exporter}.container`
 - `scripts/deploy-remote.sh:219-300` (copiers + relay), `:521-536` (health waits), `:477-486` (grafana env seed)
 - `scripts/signal-relay.py:26-49`, `scripts/signal-relay.service`
 - `shared/prometheus/prometheus.yml`, `shared/prometheus/alertmanager.yml`, `shared/prometheus/rules/llm-fleet.yml`
@@ -88,25 +88,30 @@ on stop. Boot budget 1800 s (the publisher's compose allows a 15-minute `start_p
 | Knob (env name, `FRANK_ARCHON_` dropped) | Live value |
 |---|---|
 | TP / wire | `TP=2`, `TP_WIRE=wht6` — libr4d one-shot all-reduce, not RCCL; `min-kb` stays the engine default |
-| Slots / window | `MAX_SEQS=2`, `MAX_MODEL_LEN=200000` (native 262,144; longer is refused, never truncated) — this pair is the **cache bound**, not a KV allocation |
+| Slots / window | `MAX_SEQS=4`, `MAX_MODEL_LEN=200000` (native 262,144; longer is refused, never truncated) — this pair is the **cache bound**: the resolver sizes the KV carve against `slots × window`, and no slot count funds itself (Laws) |
 | KV dtype | `KV_DTYPE=fp8` — the only dtype that funds the slot count (Laws) |
 | VRAM split | **derived at boot** from `GPU_HEADROOM_MIB=96` + the cache bound + `HOST_POOL_MIB`; naming `VRAM_WEIGHTS_MIB`/`VRAM_KV_MIB` is a boot refusal (removed at 1.3.0) |
-| Expert tier | `PLACEMENT=expert_tiered`, `HOST_POOL_MIB=44928` **per rank** |
+| Expert tier | `PLACEMENT=expert_tiered`, `HOST_POOL_MIB=49152` **per rank** — the pinned host pool is also the **KV lever**: the expert floor is `plane + 2 units − host room`, so every MiB pinned there lowers what the experts claim on the card one-for-one and the cache takes it (Laws) |
 | PLE | `NGRAM_PLACEMENT=ram` (47.7 GiB pinned; refuses boot on a short memlock) |
 | Spec / batch | `SPEC_TOKENS=2` — MTP depth 2, the `.rad` carries the model's own `mtp.*` head (`mtp.fc_hidden`/`fc_embedding`/`draft_head`); `MAX_BATCHED_TOKENS=2048` (MNBT — 2560 collapses decode, Laws) |
 | Prefix cache | `PREFIX_CACHE_MIB=8192`, pinned **host tier only** — the 64 GiB disk tier is gone, so an evicted prefix is re-prefilled; `STATE_DIR=/home/dank/radiance-state` now exists only to hold the rad-tune cache |
 | Template | `TEMPLATE=scripts/qwen38-froggeric-v22.5.jinja` (28,234 B) via `--override-chat-template` — froggeric's fixed Qwen3.8 template (the repo root's `chat_template.jinja`, vendored); the checkpoint's own template is baked into the `.rad` by rad-convert and this overrides it, and radiance derives the reasoning and tool-call reply format from it |
 | Step sharing | `BUSY_PREFILL_CHUNK=auto` — prefill-speed first: it reserves 3 tokens per waiting decoder and ⅛ of the budget per waiting request, and does **not** shrink the chunk for decoders alone · `PREFILL_INTERLEAVE=1` — the decoder dial, **armed 2026-10-20**: one decode-only step after each chunk step taken beside decoders; meter `engine_steps_held_total`, price is the prefilling prompt's TTFT |
-| TTM gate | `TTM_MIN_MIB=150000`, `TTM_TARGET_MIB=204800` |
+| TTM gate | `TTM_MIN_MIB=162000`, `TTM_TARGET_MIB=204800` |
 | Boot | `RAD_MIN_MIB=110000` deployer floor; the listener binds only after the full load (health ~100 s, budget 1800 s) |

-Boot ledger for the live shape (2026-10-19 11:10, pin 1.8.1, `MAX_SEQS=2`, `HOST_POOL_MIB=44928`):
-the resolver prints `elastic 26.73 GiB` → **experts 23.73 GiB (89 %) / kv cache 2.76 GiB (10 %)**,
-`vram_weights 27.31 GiB`, `host_pool 44.47 GiB`. `kv_attn` (13 layers, 4-token blocks) pages
-**26.00 KiB** → 91,301 blocks = **365,204 tokens = 1.8 full-length sessions = 91 % of the
-2 × 200,000 worst case** (the resolver never funds 100 %, see Laws); `mover` lends 2,146 slots
-(2.54 GiB) out of the cache to the expert slab; sampler **9 rows**, template 28,234 B, vision live
-(≤ 4,194,304 px, 16,384 patches/pass).
+Boot ledger for the live shape (2026-10-20, pin 1.8.1, `MAX_SEQS=4`, `HOST_POOL_MIB=49152`):
+the resolver prints `elastic 26.73 GiB` → **experts 20.72 GiB (78 %) / kv cache 5.64 GiB (21 %)**,
+`prefill staging 373.65 MiB` out of the cache, `vram_weights 24.30 GiB`, `host_pool 48.59 GiB`.
+The referee line: `kv: the cache can grow to 5.64 GiB a card — 747,492 tokens of context, 3.7
+sessions at 200,000 — with every block it lends the experts taken back; 440.04 MiB of that is
+per-sequence state, and up to 5.20 GiB is lent to the experts while no request holds it`.
+`--checkpoint-slots 40` is auto (`max_num_seqs × 10` slots of 111.98 MiB = 4,479 MiB of pinned
+host), `/health` reports `queue_cap 32` (`max(8, seqs × 8)`), sampler **12 rows**
+(= `seqs × (1 + speculative depth)`), 4 hardware queues a card, template 28,234 B, vision live
+(≤ 4,194,304 px, 16,384 patches/pass), P2P confirmed (`the 2 cards reach each other's memory
+directly`). Experts living in the pinned host pool: **8.99 GiB a rank** (was 5.99) — the price of
+the bigger cache, paid over PCIe when those experts are hit.

 The boot **ledger**, not this arithmetic, is the referee: radiance states every budget before
 anything is allocated (weights vs KV, the host pool's coverage of the worst case, the PLE pin,
@@ -145,6 +150,7 @@ marked "re-fetch" is gone from `/home/dank/models/`, which today holds only `Sti

 | Window | Payload | Runtime | Shape | Replaced because | Rollback |
 |---|---|---|---|---|---|
+| 2026-10-19 → 10-20 | same `qwen3.8-next-flash-fp8-iq4r-moe.rad` (payload unchanged) | radiance **1.8.1**, `sha256:d590e300…` (same digest) | 2 × 200,000 fp8 KV — resolver cache 2.76 GiB = 365,204 tokens (91 % backing); host pool 44,928 MiB a rank, `TTM_MIN_MIB=150000`, `queue_cap 16`, sampler 9 rows; `--prefill-interleave 1` armed on 10-20 | Frank ordered **four** concurrent conversations over a ≥ 700,000-token pool; the resolver funds that only by lowering the expert floor with more pinned host room (Laws) | revert `1bb6399` and redeploy `scripts/deploy.sh dankserver --service frank-archon` — trio back to `MAX_SEQS=2` / `HOST_POOL_MIB=44928` / `TTM_MIN_MIB=150000` |
 | 2026-10-15 → 10-19 | same `qwen3.8-next-flash-fp8-iq4r-moe.rad` (payload unchanged) | radiance **1.0.13**, `sha256:4c3f4845…` | 3 × 200,000 fp8 KV bought with the **explicit** pair `--vram-weights-mib 25600 --vram-kv-mib 5120`; host pool 44,928 → 45,568 MiB a rank; prefix cache 8 GiB host + 64 GiB disk | decode turned to sludge beside any long prompt — step contention, 18.7 ms decode-only steps against 317 ms steps carrying a prefill chunk (Laws) — and the only two knobs that fix it, `--busy-prefill-chunk` and `--prefill-interleave`, exist only after this pin | `FRANK_ARCHON_IMAGE` back to `sha256:4c3f4845…` **plus a trio rewrite**: restore the pair and the host+disk prefix knobs, delete `BUSY_PREFILL_CHUNK`/`PREFILL_INTERLEAVE`. A digest swap alone is a boot refusal (Runtime) |
 | 2026-10-08 → 10-15 | in-house `qwen4exp-swift15-w4dn8-i8-hc8m.rad`, 116,416 MiB (Swift1.5 abliterated BF16, GPTQ w4 experts + 24 bf16 protected, KLD 0.0131 / top-1 97.92 %) | radiance 1.0.13, same digest | 3 × 200,000 fp8 KV, slab 25,600 / pool 5,120 — the shape the live file still carries | the engine author's own container is the supported path and his quality ceiling; the finetune's refusal-bending left the endpoint | `.rad` purged 2026-10-17 (trash-staged then emptied, Frank's approval). Re-download `frank8465/Swift-1.5-Qwen3.8-Flash-Next-GPTQ-W4A8-Radiance-AMD-R9700` → `qwen4exp-w4nl64-i8-hc8m.rad`, 122,071,973,888 B, LFS sha256 `09dd9306…`, via the iq4r fetch script's hfget/verify pattern retargeted at that repo; point `MODEL_FILE` at it, redeploy. Re-**convert** is dead: the BF16 source was `rm`'d 2026-10-15 (`kld-ref`, grams, corpora and `/home/dank/logs/swift15_*.log` survive) |
 | 2026-10-05 → 10-08 | `qwen4exp-fp8-native.rad`, 177,109 MiB (173.0 GiB) — native modelopt block-FP8, no requantisation | radiance 1.0.7 line, same digest | — | native FP8 experts (56.4 GiB/rank) ate a 40 GiB pool and left KV at 3,543 MiB = 386,904 tokens, **1.93 sessions** — context economics, not quality; and the host pool could not be trimmed to pay for it (36,864 refused the boot) | rebuild-only: `scripts/download-frank-archon.sh` (official `Qwen/Qwen3.8-Flash-Next-FP8`, 131 shards, 172.8 GB, ungated) + `scripts/build-frank-archon-rad.sh` (no-recipe convert under the descriptor law); raw **and** `.rad` were `rm`'d 2026-10-15 (Frank's call, to free the volume for the iq4r download) |
@@ -211,27 +217,44 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
  never `--kv-cache-size-mib`; with the disk tier engaged the engine also forced ~9.6 GiB of weight
  buffers + arena onto the weights side (~29.6 GiB/card effective). **Neither flag exists at 1.8.1:**
  the same ceiling is enforced by the resolver, and passing the pair is a boot refusal.
- **At 1.8.1 the split belongs to the resolver, and it funds ~90 % of the worst case, never 100 %.**
-  The engine takes `--gpu-headroom-mib`, the cache bound (`--max-model-len` × `--max-num-seqs`) and
-  `--host-pool-mib`, prints `claimable → elastic → experts / kv cache` before allocating anything, and
-  sizes the expert plane from the model's own weight footprint plus slack — not from a number typed by
-  hand. Three boots of evidence:
+- **At 1.8.1 the split belongs to the resolver, and the source fixes its order.**
+  `core/mem/vram_budget.cpp` claims the card bottom-up (`capacity → already held → drift → arena →
+  staging → queues → prefetch → headroom = claimable`), takes the **static weights off the top as a
+  floor, not a share**, and divides the `elastic` remainder in three lines (146-156):

-  | shape | demand | cache given | backing |
+  ```cpp
+  exp = std::min(b.expert_floor, b.elastic);
+  kv  = std::min(kv_cap, std::max(b.elastic - exp, kv_one));
+  exp = std::min(std::max(b.expert_ceiling, b.expert_floor), b.elastic - kv);
+  ```
+
+  `kv_cap` is `KVManager::ceiling()` = `state + Σ_paged max_seqs × ceil(max_ctx / block) × bytes`
+  (`core/mem/kv.cpp:266`), so `--max-num-seqs` and `--max-model-len` set the cache's **demand**, never
+  its funding. The funding is `elastic − expert_floor`, and the floor is
+  `min(plane + spare, plane + spare + 2·unit_max − host_room)` (`core/place/planner.cpp:100-104`, with
+  `host_room = host_pool − file_stage_reserve` at `core/engine_bringup.cpp:1418-1422`). **That is the
+  cache's lever: every MiB added to `--host-pool-mib` drops the expert floor one-for-one and the cache
+  takes it** — minus `stage`, the prefill stager's ring (two layers of the experts left off the card),
+  which is charged back out of the cache whenever the cache lends (`vram_budget.cpp:193-203`). Boots of
+  evidence, all on 1.8.1:
+
+  | shape | demand | cache given | engine's `backs %` |
  |---|---|---|---|
-  | 3 × 200,000, host pool 44,928 | 600,000 tok | 4.20 GiB | 93 % (2.8 sessions) |
+  | 3 × 200,000, host pool 44,928 | 600,000 tok | 4.20 GiB | 93 % |
  | 3 × 200,000, host pool 45,568 | 600,000 tok | **4.20 GiB — byte-identical** | 93 % |
-  | 2 × 200,000, host pool 44,928 (live) | 400,000 tok | 2.76 GiB = 365,204 tok | 91 % (1.8 sessions) |
+  | 2 × 200,000, host pool 44,928 | 400,000 tok | 2.76 GiB | 91 % |
+  | 4 × 200,000, host pool 49,152 (live) | 800,000 tok | **5.64 GiB = 747,492 tok** | 93 % |

-  So `--max-num-seqs` is a **demand signal** for the resolver and an admission cap for the scheduler —
-  never a KV allocation — and no slot count fully funds the worst case. Growing host RAM by less than a
-  slab step (640 MiB) moved nothing: the expert plane's claim decides what is left for KV, so cache is
-  grown through slots × window or the dtype, and experts are grown by not leaving elastic pool idle.
-  `--host-pool-mib` is still load-bearing the other way — it is where the expert rows the cards cannot
-  hold get pinned, and under-sizing it has always been a boot failure (36,864 refused a boot in the fp8
-  era). The two-slot boot's own behavioural check: three concurrent conversations all answer `200` —
-  the third **queues** (`queue_cap: 16`), it is not refused; the 400 class is a single request over
-  `--max-model-len`, or three concurrent full-height conversations against a 1.8-session cache.
+  The 45,568 row is the ceiling case, not a mystery: at 3 slots the cache already held its whole
+  `ceiling`, so extra host room could not grow it and shows up as `unclaimed`. **The referee is the
+  ledger line `kv: the pool backs N % … the cache can grow to … N tokens`, never arithmetic.** To buy
+  window: raise `--host-pool-mib` (cost: pinned host RAM, and experts evicted to host cross PCIe —
+  `experts-from-host` was 5.99 GiB a rank at 44,928 and is 8.99 at 49,152) or cut the per-token cost
+  (`fp8` KV). Raising slots raises demand **and** costs 10 × 111.98 MiB of pinned checkpoint-store RAM
+  per slot. Under-sizing the pool the other way has always been a boot failure (36,864 refused a boot in
+  the fp8 era: the host cannot hold what the cards will not). Behavioural check on the live shape:
+  `/health` reports `queue_cap 32`; a request beyond the slot count queues rather than being refused,
+  and the 400 class is a single request over `--max-model-len`.
 - **Idle KV VRAM is the expert cache's spillover — the price is paid twice.** `mover: 2275
  slots can be lent out of the KV cache (2.69 GiB)`, `arena: lends 740 MiB while steps are <=
  256 tokens`; the lend grows with the pool (2.69 GiB from 3.46, 6.08 from 6.912, 4.22 in the pair era)
@@ -366,17 +389,18 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
  `--checkpoint-interval` does **not** buy the deep case: the coarsest ladder scale is the largest
  `interval × 2^k` under the depth, so the deepest gap stays ≈ half the prompt (129,024 tokens at
  interval 2048 against 130,816 at 256) while every cut multiplies the snapshot writes.
- **Slots cost KV, rows, and window — never speed.** `--max-num-seqs` carries no radiance constraint: it
-  adds ≈110 MiB of recurrent state per slot (the live `kv_full_backing` arithmetic prices them exactly),
-  declares the fused TP all-reduce over `seqs × (1 + spec)` rows, and `wht6` is bit-exact up to 25 rows
-  (single-sequence decode and any step of ≤ 25 rows is exact; only busier steps are lossy) — 2 × 3 = 6
-  today. Measured 2026-10-06: seqs 2 and seqs 3 are statistically tied (30.9-33.1 t/s, inside the ±3 %
-  floor). Measured 2026-10-19 at 1.8.1: the third slot's real price is **KV, not rows** — dropping it
-  moved the resolver's cache from 4.20 GiB to 2.76 GiB and the backing from 2.8 to 1.8 sessions, while
-  the ≈4 GiB of host RAM it freed came back unbudgeted, landing in elastic pool that turned into a bigger
-  mover lend instead of cache (resolver law above). The old "seqs 2 cannot boot" law belonged to the
-  retired tcclaviger fork's `torch.compile` shape specialization, and radiance booted seqs 2 on its first
-  fp8-native start.
+- **Slots cost KV, host RAM, rows, and pin — never speed.** `--max-num-seqs` carries no radiance
+  constraint: it adds ≈110 MiB of recurrent state per slot, 10 × 111.98 MiB of pinned checkpoint-store
+  RAM per slot (`--checkpoint-slots auto`), and declares the fused TP all-reduce over
+  `seqs × (1 + spec)` rows — `wht6` is bit-exact up to 25 rows, and the live shape is 4 × 3 = **12**, so
+  single-sequence decode stays exact and so does any step up to that many rows. Raising it also raises
+  the cache's *demand* (`ceiling = slots × window`), which the resolver funds from `elastic −
+  expert_floor` — so a slot count without matching host room is paid for by expert residency. Measured
+  2026-10-06: seqs 2 and 3 statistically tied (30.9-33.1 t/s, inside the ±3 % floor). Measured
+  2026-10-19 → 10-20 at 1.8.1: 2 → 4 slots cost **+4,224 MiB of pinned host pool** and **−3.01 GiB of
+  expert residency a card** and bought 2.76 → 5.64 GiB of cache (365,204 → 747,492 tokens). The old
+  "seqs 2 cannot boot" law belonged to the retired tcclaviger fork's `torch.compile` shape
+  specialization, and radiance booted seqs 2 on its first fp8-native start.
 - **A dequantize-to-compute path bills host RAM twice.** On the EMULATION backend an offload request
  of **118 GiB** landed as **~238 GiB** of real host RAM — the dequantized copy rides on top of the
  pinned weights, and the doubling is invisible in the weight count. Read the engine log for
@@ -570,13 +594,13 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
  — at 64 tokens you get empty `content` and the colour only inside `reasoning_content`.
 - **Capacity behaviour:** a prompt over `--max-model-len` is refused, never truncated; a request
  longer than the KV pool's token count is accepted then refused as too large to fit; a request
-  beyond `--max-num-seqs` is held by the HTTP port (`queue_cap`, default 8 × slots = 16 here) and
-  answered 503 only past that, never paged or squeezed. Because the cache funds **1.8** full-height
-  sessions, admitting a second near-full-height conversation can **preempt a running sequence from
-  the back** to free its blocks (`core/sched/scheduler.cpp:1291-1322`); the victim is recomputed, so
-  its session halts and then re-prefills. Meter: `vllm:num_preemptions_total` (264 in the current
-  boot, all in prefill-coincident bursts — 108 inside 105 s); `/stats preemptions` is the cumulative
-  count and `radiance:num_requests_preempted` only the instant depth.
+  beyond `--max-num-seqs` is held by the HTTP port (`queue_cap` = `max(8, 8 × slots)` — **32** here)
+  and answered 503 only past that, never paged or squeezed. The cache funds **3.7** full-height
+  sessions, so four conversations can be resident but a fourth near-full-height one can still
+  **preempt a running sequence from the back** to free its blocks
+  (`core/sched/scheduler.cpp:1291-1322`); the victim is recomputed, so its session halts and then
+  re-prefills. Meter: `vllm:num_preemptions_total` (cumulative for the boot, `/stats preemptions`);
+  `radiance:num_requests_preempted` is only the instant depth.
 - **Observability:** `/metrics` carries vLLM-compatible names plus `radiance:*`, scraped by the
  Dankclaw Prometheus job at `192.168.0.235:7770` (15 s interval) over plain LAN routing (the
  netavark published-port quirk is same-host only). **Latency lives under `vllm:`, cadence under
@@ -1,137 +0,0 @@
-{
-  "uid": "containers",
-  "title": "Containers — Dankclaw & Dankserver",
-  "description": "Per-container CPU and memory for the quadlet app stacks, from cadvisor (a rootless quadlet on each host). cadvisor keys each series by the cgroup path, and Prometheus relabels the quadlet unit name (app.slice/<name>.service) into a `container` label. Panels use only the metric families cadvisor actually exports for these rootless containers: CPU usage and memory working-set/limit/OOM. Network and filesystem are NOT exported for them (no netns/classid attribution under the rootless+host-PID layout), so no panels pretend otherwise. See serverAppConfigs-docs/Infrastructure/Observability.md.",
-  "tags": ["fleet", "cadvisor", "containers"],
-  "timezone": "browser",
-  "schemaVersion": 39,
-  "refresh": "30s",
-  "time": { "from": "now-6h", "to": "now" },
-  "templating": {
-    "list": [
-      {
-        "name": "host",
-        "label": "Host",
-        "type": "query",
-        "datasource": { "type": "prometheus", "uid": "prometheus" },
-        "query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\"}, host)", "refId": "host_var" },
-        "refresh": 2,
-        "includeAll": true,
-        "multi": true,
-        "current": { "text": "All", "value": "$__all" }
-      },
-      {
-        "name": "container",
-        "label": "Container",
-        "type": "query",
-        "datasource": { "type": "prometheus", "uid": "prometheus" },
-        "query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\"}, container)", "refId": "container_var" },
-        "refresh": 2,
-        "includeAll": true,
-        "multi": true,
-        "current": { "text": "All", "value": "$__all" }
-      }
-    ]
-  },
-  "panels": [
-    {
-      "id": 1,
-      "type": "row",
-      "title": "CPU",
-      "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
-      "panels": []
-    },
-    {
-      "id": 2,
-      "type": "timeseries",
-      "title": "CPU usage (cores)",
-      "description": "CPU-seconds/s per container = cores busy. A steady high line is a busy service; a spike is a burst (a render, an index, a backup). Only containers are shown, not the host or its daemons.",
-      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 1 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
-      "targets": [
-        {
-          "expr": "sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
-          "legendFormat": "{{host}} / {{container}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "id": 3,
-      "type": "bargauge",
-      "title": "Top containers by CPU (cores)",
-      "description": "Instant ranking over the selected window. Tells you which container to look at when the host is loaded.",
-      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": null, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 4 } ] } }, "overrides": [] },
-      "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true },
-      "targets": [
-        {
-          "expr": "topk(12, sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m])))",
-          "legendFormat": "{{host}} / {{container}}",
-          "format": "table",
-          "instant": true,
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "id": 4,
-      "type": "row",
-      "title": "Memory",
-      "collapsed": false,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 },
-      "panels": []
-    },
-    {
-      "id": 5,
-      "type": "timeseries",
-      "title": "Memory working set",
-      "description": "container_memory_working_set_bytes per container — the number the OOM killer acts on. Cache-inclusive and reclaimable; a flat high line is resident data, a sawtooth is churn.",
-      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 11 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": { "defaults": { "unit": "bytes", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
-      "targets": [
-        {
-          "expr": "container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}",
-          "legendFormat": "{{host}} / {{container}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "id": 6,
-      "type": "timeseries",
-      "title": "Memory vs limit (%)",
-      "description": "Working set over the cgroup memory limit (container_spec_memory_limit_bytes). Only containers that carry a limit are shown; approaching 1.0 means an OOM is near. Containers with no limit are omitted.",
-      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 11 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
-      "targets": [
-        {
-          "expr": "(container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} / on (job, host, container) group_left container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}) and (container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} > 0)",
-          "legendFormat": "{{host}} / {{container}}",
-          "refId": "A"
-        }
-      ]
-    },
-    {
-      "id": 7,
-      "type": "timeseries",
-      "title": "OOM kills",
-      "description": "container_oom_events_total rate. Any non-zero line means the kernel killed a process inside that container — the honest signal that its memory limit is too tight for its real working set.",
-      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 20, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] },
-      "targets": [
-        {
-          "expr": "sum by (host, container) (rate(container_oom_events_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
-          "legendFormat": "{{host}} / {{container}}",
-          "refId": "A"
-        }
-      ]
-    }
-  ]
-}
@@ -131,63 +131,3 @@ scrape_configs:
      - targets: ["192.168.0.235:7770"]
        labels:
          service: frank-archon
-
-  # Per-container metrics from cadvisor (a rootless quadlet on each host).
-  # cadvisor keys every series by the cgroup path; for a quadlet container that
-  # path is …/app.slice/<name>.service[/libpod-payload-<hex>], so the unit NAME
-  # lives in it. The relabel below pulls that name out into a `container` label
-  # and keeps only the per-unit aggregate — cadvisor's raw host root,
-  # system.slice daemons and machine.slice scopes carry no such name and are
-  # dropped, as is the per-CPU-core split (only cpu="total" survives), so the
-  # scrape stays small. cadvisor cannot bind per-interface, so it is reached at
-  # the host's PUBLISHED LAN port: unlike node-exporter's host-net bind (which
-  # is why node-dankclaw scrapes via host.containers.internal), a published port
-  # IS reachable from this bridge container at the host LAN address.
-  - job_name: "cadvisor-dankclaw"
-    static_configs:
-      - targets: ["192.168.0.143:8080"]
-        labels:
-          host: dankclaw
-    metric_relabel_configs:
-      - source_labels: [id]
-        regex: '.*/app\.slice/([^.]+)\.service(?:/.*)?$'
-        target_label: container
-        replacement: '$1'
-      # Keep only the unit-level cgroup; drop its payload/control children so a
-      # container is not counted twice.
-      - source_labels: [id]
-        regex: '.*/\.service/.*'
-        action: drop
-      # Drop everything that is not a named unit (no `container` was extracted).
-      - source_labels: [container]
-        regex: '.+'
-        action: keep
-      # cpu="total" for cpu series; metrics with no cpu label (memory, network,
-      # filesystem) are kept.
-      - source_labels: [cpu]
-        regex: '^$|total'
-        action: keep
-      - regex: 'id|name|image|container_label.*|machine_id|boot_id'
-        action: labeldrop
-
-  - job_name: "cadvisor-dankserver"
-    static_configs:
-      - targets: ["192.168.0.235:8080"]
-        labels:
-          host: dankserver
-    metric_relabel_configs:
-      - source_labels: [id]
-        regex: '.*/app\.slice/([^.]+)\.service(?:/.*)?$'
-        target_label: container
-        replacement: '$1'
-      - source_labels: [id]
-        regex: '.*/\.service/.*'
-        action: drop
-      - source_labels: [container]
-        regex: '.+'
-        action: keep
-      - source_labels: [cpu]
-        regex: '^$|total'
-        action: keep
-      - regex: 'id|name|image|container_label.*|machine_id|boot_id'
-        action: labeldrop
@@ -21,7 +21,7 @@ load helpers

 UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
 REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
-UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager)
+UNITS=(prometheus grafana loki alloy node-exporter alertmanager)

 # container_section <unit> — the [Container] key/value block only
 container_section() {
@@ -33,7 +33,7 @@ service_section() {
    awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
 }

-@test "all seven observability units exist under dankclaw/observability/quadlet/" {
+@test "all six observability units exist under dankclaw/observability/quadlet/" {
    local svc
    for svc in "${UNITS[@]}"; do
        [ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
@@ -277,7 +277,6 @@ loki http://127.0.0.1:3100/ready
 alloy http://127.0.0.1:12345/-/healthy
 node-exporter http://127.0.0.1:9100/metrics
 alertmanager http://127.0.0.1:9093/-/healthy
-cadvisor http://127.0.0.1:8080/healthz
 PAIRS
 }

@@ -569,72 +568,3 @@ assert any('ALERTS' in e for e in exprs), exprs
    [ "$status" -eq 0 ]
    [ "$output" = "alertmanager" ]
 }
-
-@test "cadvisor exports container metrics only because of --cgroupns=host, and binds exactly its two consumers" {
-    # Podman starts every container in a private cgroup namespace, so a cadvisor
-    # without --cgroupns=host sees only its own cgroup root and exports a single
-    # series (measured 2026-10-20: 1 series before the flag, 996 after). That
-    # flag and the host /sys mount are the entire mechanism — pin them, plus the
-    # publish set: loopback (host probe + health gate) and this host's LAN
-    # address (Prometheus reaches a host-published port there). Never a wildcard.
-    local unit lan
-    for pair in "dankclaw/observability/quadlet:192.168.0.143" "dankserver/observability/quadlet:192.168.0.235"; do
-        unit="$REPO_ROOT/${pair%%:*}/cadvisor.container"
-        lan="${pair##*:}"
-        [ -f "$unit" ] || { echo "missing $unit"; return 1; }
-        grep -q -- '--cgroupns=host' "$unit" || { echo "$unit lacks --cgroupns=host"; return 1; }
-        grep -q -- '--pid=host' "$unit" || { echo "$unit lacks --pid=host"; return 1; }
-        grep -q '^Volume=/sys:/sys:ro$' "$unit" || { echo "$unit lacks the host /sys mount"; return 1; }
-        grep -q '^WantedBy=default.target$' "$unit" || { echo "$unit is not boot-persistent"; return 1; }
-        grep -q '^Restart=always$' "$unit" || { echo "$unit is not self-healing"; return 1; }
-        grep -q '^HealthCmd=' "$unit" || { echo "$unit has no health check"; return 1; }
-        grep -q "^PublishPort=127.0.0.1:8080:8080\$" "$unit" || { echo "$unit must publish loopback :8080"; return 1; }
-        grep -q "^PublishPort=$lan:8080:8080\$" "$unit" || { echo "$unit must publish $lan:8080"; return 1; }
-        ! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$unit" || { echo "$unit publishes on a wildcard"; return 1; }
-    done
-}
-
-@test "Prometheus scrapes cadvisor where it is published and relabels the cgroup path to a container name" {
-    # Cross-file contract: cadvisor publishes the LAN port; a cadvisor scrape
-    # job pointing anywhere else reads nothing (unlike node-exporter's host-net
-    # bind, a published port IS reachable from this bridge container at the LAN
-    # address). And the whole dashboard depends on the id -> container relabel:
-    # cadvisor names nothing itself, the quadlet unit name is only in the cgroup
-    # path. If either half changes alone, the Containers dashboard is blank.
-    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
-    grep -q 'job_name: "cadvisor-dankclaw"' "$cfg" || { echo "no cadvisor-dankclaw job"; return 1; }
-    grep -q 'job_name: "cadvisor-dankserver"' "$cfg" || { echo "no cadvisor-dankserver job"; return 1; }
-    grep -q '"192.168.0.143:8080"' "$cfg" || { echo "cadvisor-dankclaw does not scrape the published LAN port"; return 1; }
-    grep -q '"192.168.0.235:8080"' "$cfg" || { echo "cadvisor-dankserver does not scrape the published LAN port"; return 1; }
-    # The relabel that turns …/app.slice/<name>.service… into a `container`.
-    grep -q 'app\\.slice/(\[\^\.\]+)\\.service' "$cfg" || { echo "no cgroup-path -> container relabel"; return 1; }
-    grep -q 'target_label: container' "$cfg" || { echo "relabel does not set the container label"; return 1; }
-}
-
-@test "the Containers dashboard is provisioned from git" {
-    local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/containers.json"
-    [ -f "$dash" ]
-    python3 -c "
-import json
-d = json.load(open('$dash'))
-assert d['uid'] == 'containers', d['uid']
-names = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
-# Only families cadvisor actually exports for rootless quadlet containers are
-# graphed; network/fs are absent for them, so a network/fs panel would be a
-# permanent empty lie.
-assert any('container_cpu_usage_seconds_total' in e for e in names), names
-assert any('container_memory_working_set_bytes' in e for e in names), names
-assert not any('container_network' in e or 'container_fs_' in e for e in names), names
-"
-}
-
-@test "the planner selects cadvisor for its unit on each target" {
-    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
-        --paths dankclaw/observability/quadlet/cadvisor.container
-    [ "$status" -eq 0 ]
-    [ "$output" = "cadvisor" ]
-    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
-        --paths dankserver/observability/quadlet/cadvisor.container
-    [ "$status" -eq 0 ]
-    [ "$output" = "cadvisor" ]
-}