fix(deploy): grafana copier provisions EVERY dashboard json, not a hand-listed subset

copy_grafana_provisioning named llm-fleet.json alone, so a new board in
shared/grafana/provisioning/dashboards/ silently never reached
/opt/grafana/provisioning. Measured 2026-10-21: the containers board was
merged, seam-tested, and documented as working since 2026-10-20, yet was
NEVER provisioned — its panel fixes were undeployable; the other four
boards existed on disk only because someone hand-copied them once (and
host-system.json there had gone stale against the repo). The copier now
globs the directory, and GRAFANA_PROV_ROOT lets the new bats case run
the copier for real into a sandbox (red against the old hand-listed
body, green against the glob). Vault: the copier law + the containers
incident on Observability Provisioning, the corrected dashboards claim
on Observability.

Note: a mktemp sandbox + trap RETURN races bats' trap machinery (the
dir vanishes mid-function); BATS_TEST_TMPDIR is the safe idiom.

AGENTS.md

@@ -419,16 +419,17 @@ disagrees with it is the bug.
   digest-pinned `ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84…`.
   Engine stdout (the boot ledger: weights, KV pool, clamp) goes to
   `/home/dank/logs/frank_dragoon.log`, NOT the journal.
-- Shape: `HALOGEN_CTX=200704` is the **per-request ceiling** (a longer prompt
-  is a 400, never a truncation); `HALOGEN_KV_SLOTS=2` over ONE
-  `HALOGEN_KV_POOL_POSITIONS=401408` pool — pool positions buy concurrency,
-  the per-request window never moves; `HALOGEN_SPEC_ADAPT=0` keeps the MTP
-  drafter always armed (the adaptive default disarms it and halves decode);
-  server sampler 1.0/0.95/20.
-- Harness contract: `models.yml` `contextWindow: 155648` = ceiling − 45,056.
-  The clamp is `min(45056, window − est − 4096)` and the chars/4 estimate
-  under-counts, so a harness window equal to the ceiling makes every long
-  request a guaranteed 400. Reasoning arrives in `reasoning_content`; the
+- Shape: `HALOGEN_CTX=262144` (the checkpoint's native window) is the
+  **per-request ceiling** (a longer prompt is a 400, never a truncation);
+  `HALOGEN_KV_SLOTS=2` over ONE `HALOGEN_KV_POOL_POSITIONS=401408` pool —
+  pool positions buy concurrency, the per-request window never moves;
+  `HALOGEN_SPEC_ADAPT=0` keeps the MTP drafter always armed (the adaptive
+  default disarms it and halves decode); server sampler 1.0/0.95/20.
+- Harness contract: `models.yml` `contextWindow: 200000` — ~45k+ under the
+  ceiling. The clamp is `min(45056, window − est − 4096)` and the chars/4
+  estimate under-counts (trigger is 95 %), so a harness window at the ceiling
+  makes every long request a guaranteed 400.
+  Reasoning arrives in `reasoning_content`; the
   template rejects `reasoning_effort` outside `xhigh|medium|low`; images are
   OpenAI `image_url` parts (probe ≥256 px); `response_format` unsupported;
   `/completion`, `/ping`, `/props`, `/tokenize` are 404.

scripts/deploy-remote.sh

@@ -321,14 +321,25 @@ copy_grafana_provisioning() {
     # datasource would be invisible to review, lost with the data volume, and
     # unknown to anyone reading the repository. The unit already mounts
     # /opt/grafana/provisioning read-only, so this only has to populate it.
-    run_privileged mkdir -p /opt/grafana/provisioning/datasources \
-        /opt/grafana/provisioning/dashboards
+    # GRAFANA_PROV_ROOT is the test seam: point it at a sandbox and the whole
+    # copy runs unprivileged (the bats case executes this function for real).
+    local root="${GRAFANA_PROV_ROOT:-/opt/grafana}"
+    run_privileged mkdir -p "$root/provisioning/datasources" \
+        "$root/provisioning/dashboards"
     run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \
-        /opt/grafana/provisioning/datasources/prometheus.yml
+        "$root/provisioning/datasources/prometheus.yml"
     run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \
-        /opt/grafana/provisioning/dashboards/provider.yml
-    run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \
-        /opt/grafana/provisioning/dashboards/llm-fleet.json
+        "$root/provisioning/dashboards/provider.yml"
+    # EVERY dashboard JSON rides, not a hand-listed subset: the fleet provider
+    # turns each file in the directory into a dashboard, so a copier that names
+    # files one by one silently drops new boards (measured 2026-10-21: the
+    # containers board existed in the repo since its commit and was NEVER
+    # provisioned — the panel-census browser check could not find it while the
+    # hand-placed older boards happened to be on disk). Same rule as the quadlet
+    # tier: the directory is the contract, the copy is generated state.
+    for dash in shared/grafana/provisioning/dashboards/*.json; do
+        run_privileged cp "$dash" "$root/provisioning/dashboards/$(basename "$dash")"
+    done
 }
 
 render_caddyfile() {
@@ -1401,9 +1412,10 @@ deploy_anubis() {
 # conversion (2026-10-11) of the abliterated Qwen3.8-Flash-Next —
 # qwen38-flash-next-v2-abliterated.hgn (66,687,678,432 B, ~62 GiB resident)
 # + the REQUIRED n-gram lookup table + the vision tower — MTP spec decode
-# (HALOGEN_SPEC_ADAPT=0), 2 slots over ONE 401,408-position KV pool (2 × ctx,
-# raised 2026-10-18 so BOTH slots can hold a maximum-length request) at the
-# ctx 200704 ceiling, image input — on the Strix Halo iGPU. It stays a
+# (HALOGEN_SPEC_ADAPT=0), 2 slots over ONE 401,408-position KV pool at the
+# native ctx 262144 ceiling (raised 2026-10-21 from 200704 so the harness can
+# run a 200,000 window with its ~45k fit gap; the pool still carries two
+# 200k conversations at once), image input — on the Strix Halo iGPU. It stays a
 # ROOT unit because the serve script drives `podman run` directly (the frank-archon pattern):
 # relink_unit points /etc/systemd/system/frank-dragoon.service at the repo
 # file, boot persistence is that unit's [Install], and LimitMEMLOCK=infinity

scripts/frank-dragoon.env

@@ -49,14 +49,21 @@ FRANK_DRAGOON_TOKENIZER_FILES="tokenizer.json chat_template.jinja tokenizer_conf
 # stays at the image default, which is what was qualified) ------------------
 # Served id. OpenWebUI api_configs model_ids for the :7770 backend MUST name it.
 HALOGEN_MODEL_ID=frank-dragoon
-# PER-REQUEST ceiling: a longer prompt is a 400 naming the limit, never a
-# truncation. The harness window must sit ~45k UNDER this (the 400-zone law on
-# the page) — a window equal to the ceiling makes every long request a 400.
-HALOGEN_CTX=200704
-# ONE pool behind both slots, sized so BOTH can carry a maximum-length request
-# at once. Pool positions buy concurrency; the per-request window never moves
-# with them. Legal band: ctx <= pool <= slots * ctx (the seam enforces it), and
-# RAM the pool takes is RAM the n-gram table's page cache loses.
+# PER-REQUEST ceiling, set to the checkpoint's NATIVE window (262,144 — a
+# longer prompt is a 400 naming the limit, never a truncation; stretching
+# past native would need HALOGEN_ROPE_YARN and is not on plan). The harness
+# `contextWindow` must sit ~45k under THIS number (the 400-zone law on the
+# page): with the window at 200,000 the gap is 62,144, which absorbs the
+# chars/4 under-count. A window at or near the ceiling makes every long
+# request a guaranteed 400.
+HALOGEN_CTX=262144
+# ONE pool behind both slots. The harness window is 200,000, so 401,408
+# still carries TWO maximum harness-length conversations at once
+# (2 x 200,000 + one 45,056 budget <= pool + cap headroom); two literal
+# ceiling-length requests would queue instead — buying those means raising
+# the pool, and RAM the pool takes is RAM the n-gram table's page cache
+# loses. Pool positions buy concurrency; the per-request window never moves
+# with them. Legal band: ctx <= pool <= slots * ctx (the seam enforces it).
 HALOGEN_KV_POOL_POSITIONS=401408
 # Two concurrent conversations: the MTP drafter is not slot-locked on this
 # engine, so the second slot is capacity, not a speculation tax.
@@ -107,10 +114,13 @@ HALOGEN_NPU_MODELS=qwen3-embedding-0.6b
 FRANK_DRAGOON_NPU_DIR=/home/dank/models/halogen-v2-abliterated/npu
 # Node paths and the XRT directory are variables so the bats seam can drive
 # these gates on a box with no NPU. Ubuntu's distro XRT (libxrt2 + libxrt-npu2)
-# is mounted FILE BY FILE and TWICE EACH — once where the image looks for XRT,
-# once at its own path, because those libraries find each other at their install
-# path; the vendor forbids mounting an /opt/xilinx/xrt of symlinks into the
-# system dir, which dangle inside the container.
+# is mounted FILE BY FILE and THREE TIMES EACH — where the image looks for XRT
+# (/opt/xilinx/xrt/lib), at its own path (those libraries find each other at
+# their install path), and at /opt/xilinx/lib/<multiarch>/ (the NPU child
+# searches that third prefix; without it the lane dies naming it and the
+# supervisor takes the container down — 2026-10-21 boot loop). The basename of
+# this directory IS the multiarch name. The vendor forbids mounting an
+# /opt/xilinx/xrt of symlinks into the system dir, which dangle in the container.
 FRANK_DRAGOON_DEV_ACCEL=/dev/accel/accel0
 FRANK_DRAGOON_XRT_DIR=/usr/lib/x86_64-linux-gnu
 # sysfs root the fabric-clock preflight reads (same knob the vendor's

scripts/frank-dragoon.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=halogen-flash-server container (Quat3rnion NATIVE v2 abliterated Qwen3.8-Flash-Next, MTP spec decode, 2 slots over ONE 401408-position pool at ctx 200704, vision ON), root systemd unit (dankclaw, frank-dragoon, Strix Halo iGPU)
+Description=halogen-flash-server container (Quat3rnion NATIVE v2 abliterated Qwen3.8-Flash-Next, MTP spec decode, 2 slots over ONE 401408-position pool at ctx 262144, vision ON), root systemd unit (dankclaw, frank-dragoon, Strix Halo iGPU)
 After=network-online.target
 Wants=network-online.target
 

scripts/gpu-exporter.py

@@ -32,6 +32,7 @@ import glob
 import os
 import subprocess
 import sys
+import threading
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 FAMILIES = [
@@ -198,6 +199,11 @@ def main(argv):
             return 2
         servers.append(ThreadingHTTPServer((b, args.port), Handler))
     sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds)))
+    # EVERY socket must be served: a listen()ed-but-never-accept()ed bind looks
+    # healthy in `ss` yet times out every scrape once its SYN backlog fills —
+    # and the loopback deploy gate cannot see it. One thread per extra bind.
+    for s in servers[1:]:
+        threading.Thread(target=s.serve_forever, daemon=True).start()
     servers[0].serve_forever()
     return 0
 

scripts/serve-frank-dragoon.sh

@@ -136,10 +136,17 @@ fi
 #      The engine re-checks size+sha256 against that catalog at boot; it cannot
 #      fetch anything here because /models is read-only and HALOGEN_DOWNLOAD
 #      stays unset — no outbound connection at boot),
-#   2. the distro XRT's three libraries, each mounted twice (where the image
-#      looks for XRT, and at its own path — the libraries find each other there;
-#      an /opt/xilinx/xrt of symlinks into the system dir dangles inside the
-#      container, which the vendor calls out explicitly),
+#   2. the distro XRT's three libraries, each mounted THREE times: where the
+#      image looks for XRT (/opt/xilinx/xrt/lib), at their own path (the
+#      libraries find each other there), and at /opt/xilinx/lib/<multiarch>
+#      — the NPU child searches that third prefix too, and since the host's
+#      libxrt2 was reconfigured 2026-10-20 it is the ONLY prefix the child
+#      names: omitting it crashes the lane with `No such library
+#      '/opt/xilinx/lib/x86_64-linux-gnu/libxrt_core.so.2'` and the engine
+#      supervisor takes the dead lane down on the whole container (measured
+#      2026-10-21: 88-restart boot loop). Real files only — an /opt/xilinx/xrt
+#      of symlinks into the system dir dangles inside the container, which the
+#      vendor calls out explicitly),
 #   3. the GPU fabric clock HELD: on Strix Halo the iGPU and the NPU co-running
 #      hang the machine while FCLK changes speed, and the engine refuses to
 #      start when it moves — it cannot read host sysfs from inside the
@@ -190,7 +197,9 @@ if [ -n "$NPU_MODELS" ]; then
             exit 1
         fi
         REAL="$(readlink -f "$XRT_DIR/$L")"
-        NPU_ARGS+=(-v "$REAL:/opt/xilinx/xrt/lib/$L:ro" -v "$REAL:$XRT_DIR/$L:ro")
+        MULTIARCH="${XRT_DIR##*/}"
+        NPU_ARGS+=(-v "$REAL:/opt/xilinx/xrt/lib/$L:ro" -v "$REAL:$XRT_DIR/$L:ro"
+                   -v "$REAL:/opt/xilinx/lib/$MULTIARCH/$L:ro")
     done
     HELD=0
     for D in "$SYSFS"/class/drm/card*/device; do

serverAppConfigs-docs/Agent/OMP Harness.md

@@ -52,28 +52,31 @@ Where the contract lives:
 The arithmetic (the coupled contract every endpoint's maxTokens comment
 encodes). The authoritative `contextWindow`/`maxTokens` pair per model is on the
 endpoint page ([[frank-archon]], [[frank-dragoon]], [[frank-zealot]]); what
-follows is a **worked example** recomputed from the live files on 2026-10-18 —
-a snapshot, not fleet state — with windows 200,000 / 155,648 / 200,704 and
+follows is a **worked example** recomputed from the live files on 2026-10-21 —
+a snapshot, not fleet state — with windows 200,000 / 200,000 / 200,704 and
 maxTokens 16,384 / 45,056 / 16,384 (archon / dragoon / zealot):
 
 1. **Source trigger** = `thresholdPercent × source contextWindow`; at the live
-   95 % that is 190,000 (archon), 147,866 (dragoon), 190,669 (zealot). The
+   95 % that is 190,000 (archon and dragoon), 190,669 (zealot). The
    bundle clamps the percentage to 1–99 and converts it to a token threshold.
 2. **Output budget** = capped at the target model's `maxTokens` — every role
    here compacts to frank-zealot, so the budget is its 16,384.
-3. **REQUIREMENT**: prompt (≤ trigger) + budget ≤ the target's per-slot
-   window. **At 95 % this no longer holds**: 190,000 + 16,384 = 206,384 >
-   200,704. The registry's own comments design around an 85 % trigger (archon:
-   170,000 est × 1.03 + 16,384 ≈ 191,500 ≤ 200,704, `models.yml:49-56`), which
-   does hold. While `config.yml` and those comments disagree, treat 85 % as the
-   design point and re-run this arithmetic after ANY window change — a 400 from
-   the compaction target mid-conversation is the symptom.
+3. **REQUIREMENT**: prompt (≤ trigger) + budget ≤ the fitting window. For a
+   dragoon session the fitting window is the ENGINE ceiling, not a slot: the
+   190,000 trigger is 62,144 under the 262,144 ceiling, so even ×1.03 est
+   error plus the 45,056 request budget lands inside — that headroom is the
+   reason the ceiling moved to native on 2026-10-21. For a session compacting
+   INTO frank-zealot the window is its 200,704-token slot, and **at 95 % this
+   does not hold for archon/zealot sources**: 190,000 + 16,384 = 206,384 >
+   200,704. Treat 85 % as the design point until config.yml and the registry
+   comments agree, and re-run this arithmetic after ANY window change — a 400
+   from the compaction target mid-conversation is the symptom.
    (Historic 2026-10-05 pairs, at `thresholdPercent: 70`: 140,492 + 36,864 =
    177,356 ≤ 200,704 archon; 140,490 + 57,344 ≤ 200,700 dragoon; 140,492 +
    32,768 ≤ 200,704 zealot.)
 
 Procedure (recompute from the LIVE files every time — windows rotate with
-every swap; the block above is the 2026-10-18 state):
+every swap; the block above is the 2026-10-21 state):
 
 1. Read the `config.yml` compaction block + `models.yml` entries for source
    and target. Every endpoint's `contextWindow` comment carries its own
@@ -94,9 +97,10 @@ Known reconciliation trap: comments in models.yml have described compaction
 as running "on the dankserver endpoint" while `remoteEndpoint` pointed at
 dankclaw — the comment/config disagreed. The comments now name the real
 target (`compaction.remoteEndpoint` = dankclaw :7771). **The config is
-truth; fix comments in the same change.** A fresh instance of the same trap is
-open 2026-10-18: every `maxTokens` comment still says "compaction fires at
-85 % of the window" while `config.yml:84` sets `thresholdPercent: 95`.
+truth; fix comments in the same change.** The 85 %-comments vs 95 %-config
+disagreement is not academic: it is why long dragoon sessions kept drawing
+400s at the old 200,704 ceiling — the trigger fires late AND the est
+under-counts, and a window parked near the ceiling has no room for either.
 `maxTokens` never changes for a reslice — it caps output; `contextWindow` caps
 input. Boot-shape OOM laddering is a different problem
 ([[Llama Cpp Serving Sizing]]).

serverAppConfigs-docs/Architecture/Stale Signal Audit.md

@@ -116,6 +116,20 @@ as working when they could never report up. **Clean cutover includes prose** —
 docs, unit docstrings, and the repo's rule file (the most expensive place to
 be stale).
 
+## Three rules that outlive the incident that taught them
+
+* **A target can read down beside a healthy service.** Read `lastError` from
+  `/api/v1/targets` before believing an outage — llama.cpp refuses `/metrics` at the
+  protocol layer without `--metrics` (501) while `/health` answers 200 for weeks.
+* **Alert state lives where the tool says it does.** Prometheus keeps rule state in
+  process memory — a restart re-stamps `activeAt`, so a "fresh" alert after a reboot is
+  not evidence of a fresh fault. Alertmanager **does** persist (`--storage.path` on a
+  host volume: silences + notification log, [[Alertmanager]]).
+* **Never quote a candidate arm's flags or VRAM as fleet state.** A stopped production
+  unit (`failed` after `stop-sigterm timed out. Killing` — the known shutdown shape,
+  [[frank-zealot]]) beside an answering port means the metric measured a scratch boot;
+  the process shape can change between two censuses ten minutes apart.
+
 ## Order and verification
 
 1. Enumerate the surfaces (status board, metrics targets, alert rules, test

serverAppConfigs-docs/Infrastructure/Observability Provisioning.md

@@ -3,7 +3,7 @@ type: reference
 title: Observability Provisioning
 description: "Monitoring config as code: the config triad, deploy copiers, planner mappings, and where each change type lands."
 tags: [observability, prometheus, grafana, provisioning, deployment]
-timestamp: 2026-10-05T00:00:00-07:00
+timestamp: 2026-10-21T00:00:00-07:00
 ---
 
 # Observability Provisioning
@@ -44,7 +44,7 @@ from `deploy_observability`:
 - `install_signal_relay` → the user unit + iptables + the health wait on `192.168.0.143:18093/health` (`:243-274`)
 - `copy_loki_config` → `/opt/loki/config/loki-config.yaml` (`:276-279`)
 - `copy_alloy_config` → `/opt/alloy/config/config.alloy` (`:281-284`)
-- `copy_grafana_provisioning` → `/opt/grafana/provisioning/` (`:286-300`)
+- `copy_grafana_provisioning` → `/opt/grafana/provisioning/` — datasources, provider, and **every** `dashboards/*.json` (a glob, not a hand-listed subset; `GRAFANA_PROV_ROOT` is the test seam the bats case uses to run the copier into a sandbox)
 
 `scripts/deploy-plan.py` maps `shared/prometheus/**` to `prometheus` (plus
 `alertmanager` when the changed file is `alertmanager.yml`, `:251-256`)
@@ -82,11 +82,14 @@ reports 4 alerting + 2 recording. Rules reload with the prometheus service.
 the backend (`http://prometheus:9090`), not loopback.
 
 **Add a dashboard** — drop the JSON into
-`shared/grafana/provisioning/dashboards/`. Provisioned dashboards live in
-unified storage, not (only) the SQLite table — grep the API, not the DB, when
-verifying (see the verify section of [[Fleet Audit and Posture]]). *(unverified
-2026-10-18: the storage-backend claim needs an admin password to confirm
-against `grafana.db` and the API side by side)*
+`shared/grafana/provisioning/dashboards/`; the copier globs the directory, so
+the next `--service grafana` deploy provisions it. This is a repaired law, not
+a given: until 2026-10-21 the copier named `llm-fleet.json` alone, and the
+`containers` board — merged, tested, documented as "working" — had **never
+been provisioned**; the other boards existed on disk only because someone had
+hand-copied them once. A repo JSON is not a live dashboard until the copier
+carries it, and `deploy.sh` cannot see the gap because it only reports the
+copy it performed.
 
 ## Verify through the APIs, not the filesystem
 
@@ -95,90 +98,32 @@ against `grafana.db` and the API side by side)*
 curl -s http://127.0.0.1:9090/api/v1/rules | python3 -m json.tool | head
 curl -s "http://127.0.0.1:9090/api/v1/targets?state=active" | \
   python3 -c "import json,sys; [print(t['labels'].get('job'), t['health'], (t['lastError'] or '')[:60]) for t in json.load(sys.stdin)['data']['activeTargets']]"
-# grafana: datasource + dashboard provisioned
-curl -s -u admin:<pw> http://127.0.0.1:3002/api/datasources | python3 -m json.tool | grep -E '"name"|"url"'
+# grafana 13: /api/health, /api/org, /login all 404 UNAUTHENTICATED, and there
+# is NO local admin password (root .env sets no GF_SECURITY_ADMIN_*; the only
+# account is the Authentik RP login). The proven surface is a real browser
+# session: warm the Anubis challenge on the vhost root, then hit
+# /login/generic_oauth with the e2e storage state (tests/e2e/.auth/qa-e2e-admin.json)
+# — exactly the path specs/20-grafana-sso.spec.ts drives — and read the boards.
 ```
 
 The two Prometheus queries were run on 2026-10-18 and are what every number on
 this page comes from: 4 alerting + 2 recording rules, and `up` for
 `prometheus`, `node-dankclaw`, `node-dankserver`, `grafana`, `loki`
 `frank-zealot`, `frank-dragoon`, `frank-archon` with the four structural downs
-listed in [[Prometheus]]. The Grafana query needs the local admin password and
-was not run here; the filesystem side (`ls /opt/grafana/provisioning/*`) is
-proof of installation only — see the next paragraph.
+listed in [[Prometheus]]. The Grafana proof is the browser session above, run
+2026-10-21: it found a provisioned-file gap the filesystem listing hid. The
+filesystem side (`ls /opt/grafana/provisioning/*`) is proof of installation
+only — see *Add a dashboard* for why a repo JSON is not a live board.
 
 Health answers from a stale process and provisioned resources can hide in
 unified storage — a file existing on disk is not proof the running server
 loaded it. See [[Stale Signal Audit]] for the never-green-target decision
 table (refused = outage, 404/HTML = config defect).
 
-## Current delivered state (delivery chain live 2026-09-17, re-verified 2026-10-18)
-
-Prometheus → Alertmanager (`dankclaw/observability/quadlet/alertmanager.container`
-loopback `:9093`) → host-side Signal relay (`scripts/signal-relay.py`, user
-unit, `192.168.0.143:18093` — the LAN IP, the one bind a container can reach
-via `host.containers.internal`) → signal-cli, addressed by number
-`+12062476336` (`scripts/signal-relay.py:36-43`). Every hop re-checked
-2026-10-18: `ss -ltnp` shows `python3` on `192.168.0.143:18093`
-`curl -s http://192.168.0.143:18093/health` → `ok`, and
-`journalctl --user -u signal-relay` ends with successful `sent to +12062476336:`
-lines for `LlmEndpointDown`, `LlmRequestsQueueing`, `LlmSpecDecodeOff` and
-`LlmPrefixCacheCollapse` (firing and resolved). Some targets read down for
-structural reasons (HTML-scraped "probes", Gitea metrics off); the remedy is
-the blackbox exporter. See [[Prometheus]] → Targets.
-
-**A llama.cpp endpoint is invisible unless it opts in.** Until 2026-10-18,
-`up{job="frank-zealot"}` was 0 for the entire retained history of the metric —
-the oldest sample, 2026-10-11, is already 0 — while the service itself answered
-`/health` 200 the whole time. Cause: llama.cpp only exports `/metrics` when it
-is started with the flag, so the scrape was refused at the protocol layer (the
-flag and its opt-out are on [[frank-zealot]]) and `LlmEndpointDown` (critical)
-stayed firing:
-the relay re-sent it on the 4 h `repeat_interval` for days (`journalctl --user
--u signal-relay` — firing sends at 01:37, 05:42, 09:47, 13:52, 17:57 and 22:02
-on 2026-10-12). It finally resolved at `2026-10-18T01:53:21-07:00`, after the
-metrics opt-in landed that day; the target
-reports up since (`curl -s http://127.0.0.1:9090/api/v1/targets` 2026-10-18).
-How many payload generations it spanned is not readable from any host
-artifact. *(unverified 2026-10-18)*
-
-Two laws fall out of it. A target can read down beside a healthy service, so
-read `lastError` from `/api/v1/targets` before believing an outage. And rule
-state lives in Prometheus' process, not on disk — a restart re-stamps
-`activeAt` on a long-standing alert, so a "fresh" alert after a reboot is not
-evidence of a fresh fault. (Alertmanager by contrast **does** persist:
-`--storage.path=/alertmanager` on a host volume keeps silences and the
-notification log — `dankclaw/observability/quadlet/alertmanager.container:39-42`
-`/opt/alertmanager/data`.)
-
-Remaining known downs are the structural ones above (HTML-scraped probes, Gitea
-metrics, ComfyUI), and `frank-dragoon` exports `/metrics` natively
-([[frank-dragoon]]), which is why
-`frank-dragoon` has always been green. The active-alert count is a
-moment-in-time value and should never be documented as a constant:
-`/api/v1/alerts` was empty at the 2026-10-18 morning census, held one firing
-`LlmPrefixCacheCollapse` (`activeAt` 11:10:48Z, value 2.6e-4) when sampled
-minutes later, and was empty again by 11:56Z; the relay logged the matching
-firing and resolved sends at `04:26:21-07:00` and `04:41:21-07:00`.
-
-**What that zealot scrape measured on 2026-10-18.** As of ~04:30 PDT the
-production `frank-zealot` root unit was stopped **on purpose** for a benchmark
-window on the 3090 — the repo Benchmarking Policy says stop the production
-service before benching — and `:7771` answered from a manually booted candidate
-arm (`systemctl is-active frank-zealot` → `failed` after
-`stop-sigterm timed out. Killing`, which is llama.cpp ignoring SIGTERM past
-`TimeoutStopSec`, the known shutdown shape for these engines). So
-`up{job="frank-zealot"} == 1` that day measured the trial, not the production
-shape in `scripts/frank-zealot.env` — that shape is [[frank-zealot]]'s to state,
-and it is the repo contract.
-The unit stayed `enabled`; `./scripts/deploy.sh dankclaw --service frank-zealot`
-reinstalls and starts it, restoring boot persistence and self-healing. Never
-quote a candidate arm's flags or VRAM as fleet state — the process and its shape
-changed between two censuses ten minutes apart.
-
 ## Related
 
 - [[Observability]] — stack topology and the alert delivery chain
-- [[Stale Signal Audit]] — classifying down targets before "fixing" them
+- [[Stale Signal Audit]] — classifying down targets before "fixing" them, and the
+  down-beside-healthy / alert-state-persistence / candidate-arm rules
 - [[Quadlet Lifecycle]] — the units that host these services
 - [[Deploy State Triage]] — how the planner selects these services

serverAppConfigs-docs/Infrastructure/Observability.md

@@ -67,10 +67,18 @@ Alertmanager has no vhost at all. node-exporter is the exception that must live
 in the host network namespace, and it binds exactly two addresses rather than
 the wildcard — see [[Node-Exporter]].
 
-The relay's bind is the one address a bridge container can reach on the host:
-`192.168.0.143:18093`, with iptables INPUT admitting loopback, the host itself
-and the podman bridge subnet `10.89.0.0/16` before the DROP
-(`scripts/deploy-remote.sh:255-262`).
+**Same-host reachability from a bridge container is measured, not inferred**
+(2026-10-21, from inside the Prometheus container): `host.containers.internal`
+(the podman gateway) reaches host services whether they bind host-net or are
+published ports — node `:9100`, cadvisor `:8080`, gpu `:9401` all answer there
+— while the host's own LAN IP is refused (`192.168.0.143:8080`, `:9401`), so
+every same-host Prometheus target uses the gateway and only cross-host targets
+(dankserver) use plain LAN. Whether LAN works at all is network-specific: the
+relay at `192.168.0.143:18093` IS reached from alertmanager's network, with
+iptables INPUT admitting loopback, the host itself and the podman bridge subnet
+`10.89.0.0/16` before the DROP (`scripts/deploy-remote.sh:255-262`). Never
+point a scrape target at a host LAN IP from its own host on the strength of
+one working example; dial the gateway and probe.
 
 ## Deployment
 
@@ -101,12 +109,15 @@ verify it through the APIs.
 * **Delivery: working.** Alertmanager's single `signal` receiver webhooks the
   relay, and `journalctl --user -u signal-relay` records successful
   `sent to +12062476336:` lines for every rule in the set.
-* **Dashboards: working.** Provisioned datasources — Prometheus (uid
-  `prometheus`) and Loki (uid `loki`) — and the `llm-fleet`, `host-system`,
-  `containers`, `observability` and `logs-overview` dashboards, all
-  installed from `shared/grafana/provisioning/`. The Loki datasource is how
-  you read logs: Loki has no web UI of its own, so Grafana Explore is the
-  human surface (the guarded `loki.franksdank.cloud` vhost serves only the API).
+* **Dashboards: working (after the 2026-10-21 copier fix).** Provisioned
+  datasources — Prometheus (uid `prometheus`) and Loki (uid `loki`) — and the
+  `llm-fleet`, `host-system`, `containers`, `observability` and
+  `logs-overview` boards. Until 2026-10-21 the copier carried only
+  `llm-fleet.json` by name and the `containers` board had never been
+  provisioned; the deployer now globs the directory. The Loki datasource is
+  how you read logs: Loki has no web UI of its own, so Grafana Explore is the
+  human surface (the guarded `loki.franksdank.cloud` vhost serves only the
+  API). Verify method and the copier law: [[Observability Provisioning]].
 * **Per-container metrics: the `containers` dashboard (added 2026-10-20; live
   after `deploy.sh <host> --service cadvisor`).** cadvisor is the only way to see
   per-container CPU/memory here, and on this rootless-Podman host it works on one

serverAppConfigs-docs/LLM/RDNA4 MoE Expert Offload VRAM Budget.md

@@ -70,12 +70,12 @@ back toward 52. Check `VLLM_CUDAGRAPH_*` actually present in
 
 `--kv-cache-memory-bytes` does NOT appear in the arena arithmetic — the plan
 reserves the registry price, not the actual pool. Raising the pool consumes
-real free VRAM without growing the arena; the registry over-prices the pool
-(9190 B/token planned vs 7,891.8 measured at 200704 fp8 — 7,701 at 262144;
-**never reuse a B/token ratio across geometries**, and subtract ~3.87 MB
-block-alignment loss when converting a token target to bytes). The ask lands
-block-aligned: 3,166,500,000 B → 401,408 tokens; verify with
-`GPU KV cache size: N tokens, Maximum concurrency … X.Xx`.
+real free VRAM without growing the arena; the registry OVER-PRICES the pool
+(era G measured ~14 % under the planned B/token — the gap is the lesson, not
+the number; **never reuse a B/token ratio across geometries**), and there is a
+~4 MB block-alignment loss to subtract when converting a token target to
+bytes. The ask lands block-aligned: verify against the engine's own
+`GPU KV cache size: N tokens, Maximum concurrency … X.Xx` line at boot.
 
 **Driver-reported free VRAM after hours of traffic is NOT allocatable
 arena.** It is the estimator gap: the registry over-prices KV and charges
@@ -117,11 +117,13 @@ done, and read post-boot free VRAM per rank (`rocm-smi --showmeminfo vram`)
 **KV accounting — pin the pool BYTES, floor-sized:** set
 `--kv-cache-memory-bytes` PER RANK to the contract floor
 (`seqs × MAXLEN × measured B/token`), not the native window — extra pool bytes
-come out of the arena/headroom and buy nothing. Measured B/token on this
-hybrid: **7,701 fp8 / 13,839 bf16** (decomposed: attention 6,138/12,276 +
-dtype-independent mamba/side 1,563). Measured rates are shape-dependent —
-7,891.8 at the 200704×3 shape. Capacity does NOT scale with `SEQS`: the
-planner's KV reserve is for ONE max-model-len request.
+come out of the arena/headroom and buy nothing. The B/token rate is a property
+of the GEOMETRY: era G (3 × 200,704 fp8, vLLM fork) measured **7,891.8**;
+the radiance-era hybrid measured ~7,700 fp8 / ~13,800 bf16 at its own shape.
+Read your live rate from the boot ledger every time; the decomposition (a
+dtype-independent mamba/side stream rides on top of the attention KV) is why
+the ratio moves. Capacity does NOT scale with `SEQS`: the planner's KV reserve
+is for ONE max-model-len request.
 
 **Parallelism ceilings you cannot flag past:**
 
@@ -179,18 +181,6 @@ Cutover order: ladder → winner passes and serves → repo env + seam + harness
 (`git stash -u`) if the tree is dirty → deploy (one restart) → verify health
 + engine flags in the log.
 
-**Measured ladder on this box (200704×3, fp8 KV, precise memory):**
-
-| shape | util | graphs | arena | result |
-|---|---|---|---|---|
-| E | 0.98 | 0.55 | 17.22 GiB / 7,522 slots | boot ok |
-| F | 0.982 | 0.32 | 17.52 GiB / 7,653 slots | boot ok |
-| 1.0 | 1.00 | 0.07 | — | UNBOOTABLE (worker-init ValueError, both ranks) |
-| G (deployed) | 0.982 | 0.07 | 17.77 GiB / 7,762 slots | KV 401,408 tokens / 2.00x; 0.89 GiB real free/card held through a live 4,096-token generation at ~38–39 tok/s |
-
-bf16 ceilings at util 0.98: 3 seqs → 241,664 passes / 245,760 and 262,144 OOM
-at graph capture; 6 seqs fp8 → 10.70 GiB ask with 9.18 GiB free (fails) —
-~207k per context max.
 
 ## 7. Arena audit procedure (when asked for "more experts")
 

serverAppConfigs-docs/LLM/frank-dragoon.md

@@ -10,14 +10,15 @@ tags:
   - gfx1151
   - halogen
   - endpoint
-timestamp: 2026-10-19
+timestamp: 2026-10-21
 ---
 
 # frank-dragoon — halogen-flash-server 0.16.2, native v2 abliterated (Dankclaw iGPU)
 
 Strix Halo (gfx1151) endpoint `http://192.168.0.143:7770`, served id `frank-dragoon`: a podman container
 under a root `User=dank` unit. Live shape since 2026-10-17, pool raised to 401,408 on 2026-10-18
-(`2b12716`). This page replaces every per-era dragoon runbook; the Eras table is the rollback record.
+(`2b12716`), ceiling raised to the native 262,144 on 2026-10-21 so the harness window is 200,000.
+This page replaces every per-era dragoon runbook; the Eras table is the rollback record.
 
 ## Current shape
 
@@ -37,10 +38,12 @@ Image `ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b
 unauthenticated token port 8730 stays on container loopback. Engine stdout →
 `/home/dank/logs/frank_dragoon.log`, **not** the journal.
 
-- Shape: `HALOGEN_CTX=200704` hard per-request ceiling · `HALOGEN_KV_SLOTS=2` ·
-  `HALOGEN_KV_POOL_POSITIONS=401408` = slots × ctx, the top of the band `tests/frank_dragoon_unit.bats`
-  enforces (`ctx ≤ pool ≤ slots × ctx`) · KV ~28 KiB/position → **11.0 GiB reserved** · no KV-dtype knob
-  exists here, the pool is the only KV lever · native window 262,144, `HALOGEN_ROPE_YARN` beyond it.
+- Shape: `HALOGEN_CTX=262144` hard per-request ceiling = the checkpoint's native window (raised
+  2026-10-21 from 200704; `HALOGEN_ROPE_YARN` beyond native is not on plan) · `HALOGEN_KV_SLOTS=2` ·
+  `HALOGEN_KV_POOL_POSITIONS=401408` inside the band `tests/frank_dragoon_unit.bats` enforces
+  (`ctx ≤ pool ≤ slots × ctx`) · KV ~28 KiB/position → **11.0 GiB reserved** · no KV-dtype knob
+  exists here, the pool is the only KV lever · the pool carries TWO maximum harness-length (200k)
+  conversations at once; two literal ceiling-length requests would queue — that is a pool raise.
 - Knobs: `HALOGEN_MAX_TOK=8192` sizes the prefill arena (5.7 GiB) and is only the default for requests
   omitting `max_tokens` · `HALOGEN_MAX_TOKENS_CAP=65536` + `HALOGEN_QUEUE_TIMEOUT=7200` are a coupled
   pair · MTP depth 2 with the head INSIDE the checkpoint (**`HALOGEN_MTP_HEAD` stays unset**) and
@@ -53,10 +56,11 @@ unauthenticated token port 8730 stays on container loopback. Engine stdout →
   (Pin::None)`; no clamp line. Host `MemAvailable` read 22 GiB right after the 2026-10-18 raise —
   ~20 GiB above the 20 GiB reserve, against 10–15 GiB of baseline co-tenancy (frank-zealot, ComfyUI,
   Caddy/observability).
-- Harness (`~/.omp/agent/models.yml`, provider `llamacpp (dankclaw)`): `contextWindow: 155648`
-  (= ceiling − 45k per the 400-zone law, never higher), `maxTokens: 45056`, `input: [text, image]`
-  `reasoningContentField: "reasoning_content"`, `high→xhigh` fold, `extraBody` 1.0/0.95. Compaction
-  rides [[frank-zealot]].
+- Harness (`~/.omp/agent/models.yml`, provider `llamacpp (dankclaw)`): `contextWindow: 200000`
+  — sits 62,144 under the ceiling, clearing the ~45k fit-law gap (400-zone law below);
+  `tests/frank_dragoon_unit.bats` fails if the page's declared window and `HALOGEN_CTX` drift back
+  together. `maxTokens: 45056`, `input: [text, image]`, `reasoningContentField: "reasoning_content"`,
+  `high→xhigh` fold, `extraBody` 1.0/0.95. Compaction rides [[frank-zealot]].
 - Deploy `./scripts/deploy.sh dankclaw --service frank-dragoon`; gate = boot-ledger greps (weights,
   pool, MTP, prompt cache, **no clamp**) + `/health` 200 + a tool call + the 512 px red-PNG probe.
   Kernel: `CONFIG_HSA_AMD_SVM=y` (CachyOS ships it) and `amdgpu.noretry=0` — the env says check it at
@@ -124,14 +128,18 @@ created it — earlier topologies in .
   prompt tokens. halogen 0.1.4's auto-sizing refused here and re-prefilled every 79–81k turn in
   85–180 s until `HALOGEN_CACHE_MB=16384` was pinned. What the pool takes is what the table loses, so
   re-measure the warm ratio after any pool change.
-- **400-zone law:** the harness window MUST sit ~45k under the engine ceiling (`contextWindow:
-  155648` = 200,704 − 45,056) because the harness clamps `max_tokens` to `contextWindow − est − 4096`
-  its chars/4 `est` under-counts code/agent traffic by ~3 %, and compaction only fires at 85 %. The
-  3 % is measured, not theory: 2026-10-07 at the old 200,192 ceiling, est 152,938 vs server 157,388 —
-  over by 354, a 400. At the live shape compaction fires at 132,301 est ≈ 136k measured (the 180k
-  target of the 262,144 era is unreachable — the window itself is 155,648), and the worst-case request
-  is 155,648 − 4,096 + 3 % ≈ 156.2k ≤ 200,704: ~9.5× the error of margin, robust to a ~25 %
-  under-count. Compaction rides [[frank-zealot]] at ≈ threshold − keepRecent(20k) ≈ 112k input.
+- **400-zone law:** the harness window MUST sit ~45k under the engine ceiling, because the harness
+  clamps `max_tokens` to `contextWindow − est − 4096`, its chars/4 `est` UNDER-counts code/agent
+  traffic by ~3 %, and the compaction trigger fires at 95 % of the window (`config.yml
+  thresholdPercent`, while the comments designed around 85 % — check the trigger BEFORE touching a
+  window; that disagreement is why long sessions kept drawing 400s). The 3 % is measured, not
+  theory: 2026-10-07 at the old 200,192 ceiling, est 152,938 vs server 157,388 — over by 354, a 400.
+  At the live pair (ceiling 262,144 / window 200,000) the worst-case ordinary request is
+  200,000 − 4,096 + 3 % ≈ 201.8k, and the trigger point is 190,000 est × 1.03 + 45,056 ≈ 240.7k —
+  both inside the ceiling with ~60k of margin. Compaction rides [[frank-zealot]] at
+  ≈ threshold − keepRecent(20k) ≈ 175k measured against its 200,704-token slot. The seam in
+  `tests/frank_dragoon_unit.bats` re-runs this arithmetic at the 95 % trigger and reads the window
+  from THIS page, so ceiling/window pairs that drift back together fail the suite, not a session.
 - **Admission reserves prompt + `max_tokens`:** an un-fittable request waits in arrival order, then
   503s `engine_busy` for the whole queue after `QUEUE_TIMEOUT` — and the cap/timeout pair moves
   together: 65536 tokens at ~10 t/s serial needs >6550 s, which is why 7200 is pinned (env, 2026-10-17).
@@ -248,9 +256,14 @@ lane is **armed and serving** since 2026-10-19 (`HALOGEN_NPU_MODELS=qwen3-embedd
    (`dankclaw/udev/98-halogen-npu-access.rules`, installed and re-triggered by the deployer, `0666` like the GPU
    compute nodes beside it); the serve script refuses any node that is not other-accessible, so the fault cannot
    return as a crash-loop.
-2. The host's XRT reachable inside: there is **no `/opt/xilinx/xrt`** here — distro packages put the three files the
-   container needs (`libxrt_coreutil.so.2`, `libxrt_core.so.2`, `libxrt_driver_xdna.so.2`) in
-   `/usr/lib/x86_64-linux-gnu`, where all three exist.
+2. The host's XRT reachable inside, at **THREE prefixes per library**: there is **no `/opt/xilinx/xrt`** here —
+   distro packages put the three files the container needs (`libxrt_coreutil.so.2`, `libxrt_core.so.2`,
+   `libxrt_driver_xdna.so.2`) in `/usr/lib/x86_64-linux-gnu`. The serve script mounts each REAL file (never a
+   symlink) at `/opt/xilinx/xrt/lib/`, at its own path, AND at `/opt/xilinx/lib/<multiarch>/` — the NPU child
+   searches that third prefix and a missing one is fatal: since the host's `libxrt2` was reconfigured
+   2026-10-20 the child died with `No such library '/opt/xilinx/lib/x86_64-linux-gnu/libxrt_core.so.2'` and the
+   engine supervisor took the WHOLE container down on it (88-restart loop, chat dead beside a healthy GPU line).
+   A dead lane is not survivable here — arm the lane only with all three prefixes mounted.
 3. **The GPU fabric clock held at its top level, or the engine refuses to start.** GPU and NPU work share the die:
    while `pp_dpm_fclk` is changing speed, concurrent GPU work can hang the machine and corrupt NPU results. The
    engine holds the clock itself only when it runs as root with `/sys` mounted; `frank-dragoon` is a **rootless**

shared/grafana/provisioning/dashboards/containers.json

@@ -104,14 +104,14 @@
     {
       "id": 6,
       "type": "timeseries",
-      "title": "Memory vs limit (%)",
-      "description": "Working set over the cgroup memory limit (container_spec_memory_limit_bytes). Only containers that carry a limit are shown; approaching 1.0 means an OOM is near. Containers with no limit are omitted.",
+      "title": "Memory vs host total (%)",
+      "description": "Working set over the host's total RAM. The fleet sets NO cgroup memory limits (every container_spec_memory_limit_bytes is 0 — a limit-share panel is structurally empty here), so the honest pressure view is share of the host. The join labels the node exporter's MemTotal onto the cadvisor series by host.",
       "gridPos": { "h": 9, "w": 12, "x": 12, "y": 11 },
       "datasource": { "type": "prometheus", "uid": "prometheus" },
       "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
       "targets": [
         {
-          "expr": "(container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} / on (job, host, container) group_left container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}) and (container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} > 0)",
+          "expr": "container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\", container!=\"\"} / on(host) group_left label_replace(node_memory_MemTotal_bytes{job=~\"node-.*\"}, \"host\", \"$1\", \"job\", \"node-(.*)\")",
           "legendFormat": "{{host}} / {{container}}",
           "refId": "A"
         }

shared/grafana/provisioning/dashboards/host-system.json

@@ -54,7 +54,7 @@
       "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
       "datasource": { "type": "prometheus", "uid": "prometheus" },
       "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 1 }] } }, "overrides": [] },
-      "targets": [{ "expr": "node_load5{job=~\"$node\"} / count by (job) (node_cpu_seconds_total{job=~\"$node\",mode=\"idle\"})", "legendFormat": "{{job}}", "refId": "A" }]
+      "targets": [{ "expr": "node_load5{job=~\"$node\"} / on(job) count by (job) (node_cpu_seconds_total{job=~\"$node\",mode=\"idle\"})", "legendFormat": "{{job}}", "refId": "A" }]
     },
     {
       "id": 4, "type": "stat", "title": "Open file handles",

shared/grafana/provisioning/dashboards/llm-fleet.json

@@ -194,7 +194,7 @@
       "id": 6,
       "type": "timeseries",
       "title": "Firing fleet alerts",
-      "description": "The LLM-fleet rules from shared/prometheus/rules/llm-fleet.yml. A firing alert is delivered to Signal: Prometheus -> Alertmanager (dankclaw quadlet) -> the host-side signal-relay user unit -> signal-cli, since 2026-09-17.",
+      "description": "The LLM-fleet rules from shared/prometheus/rules/llm-fleet.yml. EMPTY IS THE HEALTHY STATE: zero series means nothing is firing (ALERTS only carries active/pending alert series — do not read this panel as broken). A firing alert is delivered to Signal: Prometheus -> Alertmanager (dankclaw quadlet) -> the host-side signal-relay user unit -> signal-cli, since 2026-09-17.",
       "gridPos": {
         "h": 8,
         "w": 12,

shared/prometheus/prometheus.yml

@@ -140,12 +140,15 @@ scrape_configs:
   # system.slice daemons and machine.slice scopes carry no such name and are
   # dropped, as is the per-CPU-core split (only cpu="total" survives), so the
   # scrape stays small. cadvisor cannot bind per-interface, so it is reached at
-  # the host's PUBLISHED LAN port: unlike node-exporter's host-net bind (which
-  # is why node-dankclaw scrapes via host.containers.internal), a published port
-  # IS reachable from this bridge container at the host LAN address.
+  # the podman gateway like every same-host target: a BRIDGE CONTAINER CANNOT
+  # DIAL ITS OWN HOST'S LAN IP AT ALL — the netavark hairpin gap refuses the
+  # connection whether the port is a host-net bind or a published port (the
+  # original note here claimed published ports were exempt; measured 2026-10-21
+  # from inside this container: LAN refused, gateway answers). Cross-host
+  # targets (dankserver) have no hairpin problem and stay plain LAN.
   - job_name: "cadvisor-dankclaw"
     static_configs:
-      - targets: ["192.168.0.143:8080"]
+      - targets: ["host.containers.internal:8080"]
         labels:
           host: dankclaw
     metric_relabel_configs:

tests/frank_dragoon_unit.bats

@@ -3,7 +3,7 @@
 # of the abliterated Qwen3.8-Flash-Next (a .hgn flash-server payload) on the
 # Strix Halo iGPU through the halogen-flash-server CONTAINER (0.16.2,
 # digest-pinned), driven by the root unit's serve script — 2 slots over ONE
-# 200,704-position KV pool with the MTP head always on (HALOGEN_SPEC_ADAPT=0)
+# 401,408-position KV pool at the native 262,144 ceiling, MTP head always on (HALOGEN_SPEC_ADAPT=0)
 # and the vision tower mounted. 2026-10-17 cutover from the Swift1.5-finetune
 # shape, which itself cut over the halogen 27B-dense shape on 2026-10-06.
 #
@@ -85,7 +85,7 @@ good_env() {
         "FRANK_DRAGOON_VISION_BYTES=0"
         "FRANK_DRAGOON_TOKENIZER_FILES=tok.json"
         "HALOGEN_MODEL_ID=frank-dragoon"
-        "HALOGEN_CTX=200704"
+        "HALOGEN_CTX=262144"
         "HALOGEN_KV_POOL_POSITIONS=401408"
         "HALOGEN_KV_SLOTS=2"
         "HALOGEN_SPEC_ADAPT=0"
@@ -159,9 +159,11 @@ good_env() {
     reserve="$(sed -nE 's/^HALOGEN_HOST_RESERVE_GIB=([0-9]+)$/\1/p' "$ENV_FILE")"
     # The pool is ONE budget behind the slots (boot ledger form: "2 slots over
     # ONE <pool>-position KV pool, each request up to HALOGEN_CTX"), NOT
-    # a per-slot private pool. Since 2026-10-18 it is sized pool = slots * ctx
-    # (401,408 = 2 x 200,704), so two MAXIMUM-length conversations can be
-    # resident at once — frank-zealot parity on total in-flight context. What
+    # a per-slot private pool. Since 2026-10-21 the ceiling is the checkpoint's
+    # native 262,144 while the pool stays 401,408 (raised 2026-10-18): two
+    # MAXIMUM harness-length (200,000-token) conversations are resident at once;
+    # two literal ceiling-length requests would queue — widening that is a pool
+    # raise, not a window move.
     # still binds: the pool must back ONE full-length request (below that the
     # advertised per-request ceiling would be unreachable, hence pool >= ctx),
     # it must not pretend to be more than slots x ctx (pool <= slots * ctx), and
@@ -287,7 +289,7 @@ $(cat "$TEST_TMP/serve.log")"
     [[ "$output" == *"--name frank-dragoon-halogen"* ]]
     [[ "$output" == *"--publish 7770:8731"* ]]
     [[ "$output" == *"-e HALOGEN_MODEL_ID=frank-dragoon"* ]]
-    [[ "$output" == *"-e HALOGEN_CTX=200704"* ]]
+    [[ "$output" == *"-e HALOGEN_CTX=262144"* ]]
     [[ "$output" == *"-e HALOGEN_KV_POOL_POSITIONS=401408"* ]]
     [[ "$output" == *"-e HALOGEN_KV_SLOTS=2"* ]]
     [[ "$output" == *"-e HALOGEN_SPEC_ADAPT=0"* ]]
@@ -306,27 +308,32 @@ $(cat "$TEST_TMP/serve.log")"
 
 @test "the shipped env keeps the harness fit law inside the ceiling" {
     # HALOGEN_CTX is the hard per-request ceiling (a longer prompt is a 400
-    # naming the limit, never a silent truncation). The law is INVARIANT to
-    # the value: the harness window must sit ~45k UNDER it, because the
-    # harness clamps max_tokens = min(45056, window − est − 4096), compacts
-    # at 85 % of its declared window, and its chars/4 est UNDER-counts the
-    # rendered prompt by ~3 % on code/agent traffic (measured 4,450 tokens at
-    # est 152,938 on 2026-10-07 — with window == ceiling, every request over
-    # ~65 % of the window was a guaranteed 400 before compaction could fire).
-    # The shipped pair is ctx 200704 / window 155648 (lowered from
-    # 262144/217088 on 2026-10-17); the gap must absorb the error at BOTH
-    # monotone maxima of est + budget (est ≤ window − 4096):
-    #   full budget at the compaction threshold: 1.03*132300 + 45056
-    #   clamped budget at the clamp's zero point: 1.03*151552 + 0
-    # and the compaction request that rides on frank-zealot (its
-    # 200,704-token slot) must fit: input ≈ threshold − keepRecent(20k).
-    grep -qE '^HALOGEN_CTX=200704$' "$ENV_FILE"
-    ctx=200704
-    hw=155648; mt=45056
-    thr=$(( hw * 85 / 100 ))
+    # naming the limit, never a silent truncation). The law: the harness
+    # window must sit ~45k UNDER the ceiling, because the harness clamps
+    # max_tokens = min(45056, window − est − 4096), compacts at the trigger
+    # percent of its declared window, and its chars/4 est UNDER-counts the
+    # rendered prompt by ~3 % on code/agent traffic (measured 4,450 tokens
+    # at est 152,938 on 2026-10-07 — with window ≈ ceiling, every long
+    # request is a guaranteed 400 before compaction can fire). The 2026-10-21
+    # pair: ceiling 262144 (native) under a harness window of 200,000.
+    # The declared window is read from the endpoint page — the one owner —
+    # so env, models.yml and page that disagree FAIL HERE, not as a 400
+    # mid-session (the recurring failure this seam exists for).
+    grep -qE '^HALOGEN_CTX=262144$' "$ENV_FILE"
+    ctx="$(sed -nE 's/^HALOGEN_CTX=([0-9]+)$/\1/p' "$ENV_FILE")"
+    page="$ROOT/serverAppConfigs-docs/LLM/frank-dragoon.md"
+    grep -qE 'contextWindow: 200000' "$page"
+    hw="$(grep -oE 'contextWindow: [0-9]+' "$page" | head -1 | grep -oE '[0-9]+')"
+    mt=45056
+    # Invariant checked at the live 95 % trigger (config.yml
+    # thresholdPercent) — strictly stronger than the 85 % design point.
+    # Both monotone maxima of est + budget (est ≤ window − 4096):
+    thr=$(( hw * 95 / 100 ))
     [ $(( thr * 103 / 100 + mt )) -le "$ctx" ]
     zero=$(( hw - 4096 ))
     [ $(( zero * 103 / 100 )) -le "$ctx" ]
+    # The compaction request rides on frank-zealot's 200,704-token slot:
+    # input ≈ threshold − keepRecent(20k).
     [ $(( (thr - 20000) * 103 / 100 )) -le 200704 ]
 }
 
@@ -423,9 +430,12 @@ npu_fixture() {
     # the stock root:render 0660 mode, and the serve gate checks exactly that.
     touch "$TEST_TMP/accel0"
     chmod 666 "$TEST_TMP/accel0"
-    mkdir -p "$TEST_TMP/xrt"
+    # basename carries a multiarch shape: the third mount prefix is derived
+    # from the XRT dir's basename (the NPU child searches
+    # /opt/xilinx/lib/<multiarch>/, measured 2026-10-21).
+    mkdir -p "$TEST_TMP/x86_64-linux-gnu"
     for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
-        touch "$TEST_TMP/xrt/$l"
+        touch "$TEST_TMP/x86_64-linux-gnu/$l"
     done
     mkdir -p "$TEST_TMP/npu/qwen3-embedding-0.6b/devices"
     mkdir -p "$TEST_TMP/sys/class/drm/card1/device"
@@ -435,7 +445,7 @@ npu_fixture() {
         "HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"
         "FRANK_DRAGOON_NPU_DIR=$TEST_TMP/npu"
         "FRANK_DRAGOON_DEV_ACCEL=$TEST_TMP/accel0"
-        "FRANK_DRAGOON_XRT_DIR=$TEST_TMP/xrt"
+        "FRANK_DRAGOON_XRT_DIR=$TEST_TMP/x86_64-linux-gnu"
         "FRANK_DRAGOON_SYSFS=$TEST_TMP/sys"
     )
 }
@@ -465,11 +475,15 @@ $(cat "$TEST_TMP/serve.log")"
     ! grep -q 'accel0' "$TEST_TMP/serve.log"
 }
 
-@test "the lane arms the accel node, the staged model dir and each XRT library twice" {
-    # Each library mounts at the image's XRT path AND at its own path: the
-    # distro libraries find each other where they were installed, and an
-    # /opt/xilinx/xrt of symlinks into the system dir dangles inside the
-    # container (the vendor calls that out by name).
+@test "the lane arms the accel node, the staged model dir and each XRT library three times" {
+    # Each library mounts at the image's XRT path, at its own path (the distro
+    # libraries find each other where they were installed), and at
+    # /opt/xilinx/lib/<multiarch>/ — the prefix the NPU child searches and
+    # names when it is missing (`No such library
+    # '/opt/xilinx/lib/x86_64-linux-gnu/libxrt_core.so.2'`, measured
+    # 2026-10-21: the lane died and the supervisor took the whole container
+    # with it, 88-restart loop). An /opt/xilinx/xrt of symlinks into the
+    # system dir dangles inside the container (the vendor calls that out).
     podman_stub
     good_env
     npu_fixture
@@ -484,8 +498,9 @@ $(cat "$TEST_TMP/serve.log")"
     [[ "$output" != *":/models/npu"* ]]
     [[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]]
     for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
-        [[ "$output" == *"-v $TEST_TMP/xrt/$l:/opt/xilinx/xrt/lib/$l:ro"* ]]
-        [[ "$output" == *"-v $TEST_TMP/xrt/$l:$TEST_TMP/xrt/$l:ro"* ]]
+        [[ "$output" == *"-v $TEST_TMP/x86_64-linux-gnu/$l:/opt/xilinx/xrt/lib/$l:ro"* ]]
+        [[ "$output" == *"-v $TEST_TMP/x86_64-linux-gnu/$l:$TEST_TMP/x86_64-linux-gnu/$l:ro"* ]]
+        [[ "$output" == *"-v $TEST_TMP/x86_64-linux-gnu/$l:/opt/xilinx/lib/x86_64-linux-gnu/$l:ro"* ]]
     done
     # The Flash payload still rides the same container: the lane is additive.
     [[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]]
@@ -560,7 +575,7 @@ $(cat "$TEST_TMP/serve.log")"
     podman_stub
     good_env
     npu_fixture
-    rm "$TEST_TMP/xrt/libxrt_driver_xdna.so.2"
+    rm "$TEST_TMP/x86_64-linux-gnu/libxrt_driver_xdna.so.2"
     run_with_log
     [ "$status" -ne 0 ]
     [[ "$output" == *"libxrt_driver_xdna.so.2 missing"* ]]

tests/gpu_exporter.bats

@@ -195,3 +195,33 @@ for want in ('gpu_utilization_percent', 'gpu_memory_used_bytes',
 print('gpu dashboard panels ok')
 "
 }
+
+@test "--serve answers on EVERY declared bind, not just the first" {
+    # The deployed unit serves loopback + the host LAN address, and Prometheus
+    # reaches the host through the gateway — which dials the LAN-class bind.
+    # A server that only runs serve_forever() on the first socket leaves the
+    # second one listen()ing-but-never-accepting: the deploy health gate on
+    # loopback passes, every scrape on the other bind dies on the SYN backlog,
+    # and no gate but this one sees it. (This exact bug shipped 2026-10-21 and
+    # left both gpu-* targets down beside a green deploy.)
+    mk_amd 0 42 1073741824 34359738368 "" 71000
+    port=$((19400 + RANDOM % 500))
+    python3 "$EXPORTER" --serve --amdgpu-sysfs "$SYS" --nvidia-cmd false \
+        --bind 127.0.0.1 --bind 127.0.0.2 --port "$port" \
+        > "$BATS_TEST_TMPDIR/serve.log" 2>&1 &
+    local pid=$!
+    trap 'kill "$pid" 2>/dev/null' EXIT
+    local ok=0
+    for _ in $(seq 1 20); do
+        curl -fsS -m2 "http://127.0.0.1:$port/metrics" >/dev/null 2>&1 && { ok=1; break; }
+        sleep 0.25
+    done
+    [ "$ok" -eq 1 ] || { kill "$pid" 2>/dev/null; return 1; }
+    # Both binds must ANSWER within 2 s each — the second one is the bug.
+    for b in 127.0.0.1 127.0.0.2; do
+        curl -fsS -m2 "http://$b:$port/metrics" | grep -q '^gpu_utilization_percent{' \
+            || { echo "bind $b did not answer /metrics"; kill "$pid" 2>/dev/null; return 1; }
+    done
+    kill "$pid" 2>/dev/null
+    trap - EXIT
+}

tests/observability_quadlet.bats

@@ -595,16 +595,22 @@ assert any('ALERTS' in e for e in exprs), exprs
 }
 
 @test "Prometheus scrapes cadvisor where it is published and relabels the cgroup path to a container name" {
-    # Cross-file contract: cadvisor publishes the LAN port; a cadvisor scrape
-    # job pointing anywhere else reads nothing (unlike node-exporter's host-net
-    # bind, a published port IS reachable from this bridge container at the LAN
-    # address). And the whole dashboard depends on the id -> container relabel:
-    # cadvisor names nothing itself, the quadlet unit name is only in the cgroup
-    # path. If either half changes alone, the Containers dashboard is blank.
+    # Cross-file contract, measured 2026-10-21 from inside the Prometheus
+    # container: a bridge container cannot dial its OWN host's LAN IP — the
+    # hairpin refuses published ports exactly like host-net binds, so
+    # cadvisor-dankclaw must scrape via the podman gateway (the original pin
+    # here demanded the LAN address and shipped a permanently-down target).
+    # dankserver is a different machine: plain LAN there. And the whole
+    # dashboard depends on the id -> container relabel: cadvisor names nothing
+    # itself, the quadlet unit name is only in the cgroup path. If either half
+    # changes alone, the Containers dashboard is blank.
     local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
     grep -q 'job_name: "cadvisor-dankclaw"' "$cfg" || { echo "no cadvisor-dankclaw job"; return 1; }
     grep -q 'job_name: "cadvisor-dankserver"' "$cfg" || { echo "no cadvisor-dankserver job"; return 1; }
-    grep -q '"192.168.0.143:8080"' "$cfg" || { echo "cadvisor-dankclaw does not scrape the published LAN port"; return 1; }
+    grep -q '"host.containers.internal:8080"' "$cfg" \
+        || { echo "cadvisor-dankclaw does not scrape via the podman gateway"; return 1; }
+    ! grep -q '"192.168.0.143:8080"' "$cfg" \
+        || { echo "cadvisor-dankclaw dials its own host LAN IP — hairpin refuses it"; return 1; }
     grep -q '"192.168.0.235:8080"' "$cfg" || { echo "cadvisor-dankserver does not scrape the published LAN port"; return 1; }
     # The relabel that turns …/app.slice/<name>.service… into a `container`.
     grep -q 'app\\.slice/(\[\^\.\]+)\\.service' "$cfg" || { echo "no cgroup-path -> container relabel"; return 1; }
@@ -638,3 +644,27 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names),
     [ "$status" -eq 0 ]
     [ "$output" = "cadvisor" ]
 }
+
+@test "the grafana copier provisions EVERY dashboard json, not a hand-listed subset" {
+    # Behavioral, not a text grep: the copier is extracted and executed with a
+    # stubbed run_privileged against a sandbox. The fleet provider turns each
+    # file in its directory into a dashboard, so a file the copier skips is a
+    # dashboard that silently never exists (the containers board lived in the
+    # repo, never reached /opt, and its fixes were undeployable).
+    # BATS_TEST_TMPDIR: per-test, removed by bats itself; a mktemp dir with a
+    # RETURN trap races bats' own trap machinery and vanishes mid-function.
+    local sandbox="$BATS_TEST_TMPDIR/sandbox"
+    (
+        cd "$REPO_ROOT"
+        eval "$(sed -n '/^copy_grafana_provisioning()/,/^}/p' scripts/deploy-remote.sh)"
+        run_privileged() { "$@"; }
+        export GRAFANA_PROV_ROOT="$sandbox/opt/grafana"
+        copy_grafana_provisioning
+    )
+    local f
+    for f in "$REPO_ROOT"/shared/grafana/provisioning/dashboards/*.json; do
+        [ -f "$sandbox/opt/grafana/provisioning/dashboards/$(basename "$f")" ]
+    done
+    [ -f "$sandbox/opt/grafana/provisioning/dashboards/provider.yml" ]
+    [ -f "$sandbox/opt/grafana/provisioning/datasources/prometheus.yml" ]
+}