fix(media): a number too long to be an episode is not one

The Attack on Titan box set on the array names two files S03E2019 and
S04E47370355, and the provider rows the metadata server recorded for them carry
the same junk. EP_RANGE capped its first episode group at three digits without
guarding the run, so the pass-2 plan filed both under E201 and E473 — slots
that belong to real episodes — inside 進撃の巨人 (2013) [tvdbid-267440].

- guard the first episode group the way the second was already guarded, so a
  longer digit run states no marker at all; a glued resolution stays legal,
  because S01E01-1080p is still ONE episode
- cap a provider row's episode at EP_MAX: such a row is renumbered from tvdb's
  title when the series' id answers, and parks with a reason that names the
  junk when nothing answers
- split the operator-map exceptions into the two things they are: a folder that
  IS on the array whose files never became a series (bonus material with no
  owner: Fairy Tail, Kimetsu no Yaiba, Dorohedoro) is no longer reported as a
  folder the array no longer holds, and the missing-folder hint says the review
  pile counts as moved because the map reads top-level folders only
- the vault's evidence list carries the law; the four tests were read red
  against the shipping planner before any of it changed

Replanning the two real files now parks them under their own names with the
reasons "a provider row of season 3 episode 2019" and
"… season 4 episode 47370355".

dankclaw/Caddyfile

@@ -162,6 +162,25 @@ marreta.franksdank.cloud {
 	}
 }
 
+# --- Open Notebook (guarded: Anubis → Authentik; notebook-proxy provider, internal_host caddy:8017) ---
+# The podcast generator. ONE upstream carries the whole app: the image's own
+# Next.js layer rewrites /api/* to the FastAPI beside it (INTERNAL_API_URL,
+# default http://localhost:5055 inside the container), so a browser on this
+# vhost never needs the API port. That is why scripts/open-notebook.env sets
+# API_URL to exactly this origin — left unset, the app auto-detects
+# `<host-you-opened>:5055`, and 5055 on this host belongs to seerr — and why
+# the published API port sits on loopback. App-level login
+# (OPEN_NOTEBOOK_PASSWORD) stays on behind the chain: two owners, edge identity
+# and the app's own session.
+# Dialled by published LAN port, not container name: open-notebook is a
+# rootless quadlet and this Caddy lives in root's storage, so root's DNS does
+# not see the container (the same reason marreta and SearXNG are dialled by IP).
+notebook.franksdank.cloud {
+	reverse_proxy anubis:8923 {
+		header_up X-Real-Ip {remote_host}
+	}
+}
+
 # --- Pi-hole (quadlet on slirp4netns; published 8083) ---
 pihole.franksdank.cloud {
 	basicauth {
@@ -420,6 +439,16 @@ http://:8017 {
 	handle @marreta {
 		reverse_proxy 192.168.0.143:8082
 	}
+
+	@notebook host notebook.franksdank.cloud
+	handle @notebook {
+		# Bare proxy, like ai.franksdank.cloud: the app's SSE answers
+		# (/api/search/ask, chat sessions) stream for minutes through this
+		# same listener with no flush or timeout tuning, so nothing here
+		# needs it — a read/write timeout on the route would be the thing
+		# that cuts them.
+		reverse_proxy 192.168.0.143:8502
+	}
 }
 
 # --- Optional: Jackett ---

dankclaw/authentik/quadlet/authentik-outpost-ldap.container

@@ -23,7 +23,7 @@ Network=dankclaw_default
 # Image=ghcr.io/goauthentik/ldap:2026.8.3 (amd64 digest; podman 5.7.0 refuses
 # the tag's nested manifest list, so pin the single-platform digest).
 EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
-Image=ghcr.io/goauthentik/ldap@sha256:2e7b259eb8c37faac916946ddf48ec681ee9d3a9f724fe8521bf161973f13cf9
+Image=ghcr.io/goauthentik/ldap:2026.8.5@sha256:4c75bd27918418d2d6fbbfe4581bbd5a199fe38942d24559c4483edd64a1ea87
 Environment=AUTHENTIK_HOST=https://auth.franksdank.cloud
 Environment=AUTHENTIK_INSECURE=false
 Environment=AUTHENTIK_TOKEN=${AUTHENTIK_LDAP_OUTPOST_TOKEN}

dankclaw/authentik/quadlet/authentik-outpost.container

@@ -20,7 +20,7 @@ Wants=authentik-server.service
 
 [Container]
 ContainerName=authentik-outpost
-Image=ghcr.io/goauthentik/server:2026.8.3
+Image=ghcr.io/goauthentik/server:2026.8.5@sha256:602227a86455d6772e3d16b7d852713c4f864f40df5df894abd1285bb53979a5
 Exec=worker
 Network=dankclaw_default
 EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env

dankclaw/authentik/quadlet/authentik-postgres.container

@@ -14,7 +14,7 @@ After=network.target
 
 [Container]
 ContainerName=authentik-postgres
-Image=docker.io/library/postgres:16-alpine
+Image=docker.io/library/postgres:16-alpine@sha256:0cefaabd882e42bcb2a2f3dc56d3ce4770560ba62fd7f197f1a2e008fd5731ab
 Network=dankclaw_default
 EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
 Environment=POSTGRES_DB=authentik

dankclaw/authentik/quadlet/authentik-server.container

@@ -19,7 +19,7 @@ Wants=authentik-postgres.service
 
 [Container]
 ContainerName=authentik-server
-Image=ghcr.io/goauthentik/server:2026.8.3
+Image=ghcr.io/goauthentik/server:2026.8.5@sha256:602227a86455d6772e3d16b7d852713c4f864f40df5df894abd1285bb53979a5
 Exec=server
 Network=dankclaw_default
 EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env

dankclaw/authentik/quadlet/authentik-worker.container

@@ -16,7 +16,7 @@ Wants=authentik-server.service
 
 [Container]
 ContainerName=authentik-worker
-Image=ghcr.io/goauthentik/server:2026.8.3
+Image=ghcr.io/goauthentik/server:2026.8.5@sha256:602227a86455d6772e3d16b7d852713c4f864f40df5df894abd1285bb53979a5
 Exec=worker
 Network=dankclaw_default
 EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env

dankclaw/gitea/quadlet/gitea-postgres.container

@@ -15,7 +15,7 @@ After=network.target
 [Container]
 ContainerName=gitea-postgres
 Network=gitea.network
-Image=docker.io/library/postgres:16-alpine
+Image=docker.io/library/postgres:16-alpine@sha256:0cefaabd882e42bcb2a2f3dc56d3ce4770560ba62fd7f197f1a2e008fd5731ab
 EnvironmentFile=/home/dank/Containers/gitea/env
 Environment=POSTGRES_USER=gitea
 Environment=POSTGRES_PASSWORD=${GITEA_DB_PASSWORD}

dankclaw/gitea/quadlet/gitea-server.container

@@ -19,7 +19,7 @@ After=gitea-postgres.service network.target
 [Container]
 ContainerName=gitea-server
 Network=gitea.network
-Image=docker.io/gitea/gitea:latest
+Image=docker.io/gitea/gitea:latest@sha256:30777c91280d973fa02be48b1ade735bd6859e8f6f7af3b4444d519437852504
 EnvironmentFile=/home/dank/Containers/gitea/env
 Environment=DATABASE_TYPE=postgres
 Environment=DATABASE_HOST=gitea-postgres
@@ -29,6 +29,13 @@ Environment=DATABASE_PASSWD=${GITEA_DB_PASSWORD}
 Environment=DATABASE_PORT=5432
 Environment=USER_UID=1000
 Environment=USER_GID=1000
+# Prometheus scrapes /metrics over the published LAN port (job "gitea" in
+# shared/prometheus/prometheus.yml). Off, /metrics is a 404 and the target is
+# permanently red (the audit's "disabled metrics" class). 0.0.0.0 binds inside
+# the container; the only route in is this host's published 3000, and the
+# exposed surface is counters only — same posture as the LLM /metrics.
+Environment=GITEA__metrics__ENABLED=true
+Environment=GITEA__metrics__ENABLED_HOST=0.0.0.0
 PublishPort=0.0.0.0:3000:3000
 PublishPort=2222:22
 Volume=/home/dank/Containers/gitea/data:/data

dankclaw/hermes/quadlet/chrome-devtools-protocol.container

@@ -25,7 +25,7 @@ Description=Chrome DevTools Protocol daemon (headless-shell) for hermes
 [Container]
 ContainerName=chrome-devtools-protocol
 Network=host
-Image=docker.io/chromedp/headless-shell@sha256:2d349b544a1ea6b5b5fd7c0fe99215ff662339c57407ee2e8c0a11af93516b04
+Image=docker.io/chromedp/headless-shell@sha256:7bee201478b2f6266aae01243740ee85c16281e32f42d3aed9fdf04f4eace7a1
 Entrypoint=/headless-shell/headless-shell
 Exec=--no-sandbox --disable-gpu --use-gl=angle --use-angle=swiftshader --remote-debugging-address=127.0.0.1 --remote-debugging-port=9222
 ShmSize=1073741824

dankclaw/host/halogen-fabric-clock

@@ -0,0 +1,65 @@
+#!/bin/sh
+# halogen-fabric-clock hold | release | status: the AMD GPU's fabric clock (FCLK), held at its top level.
+#
+# On Strix Halo the GPU and the NPU working at the same time can hang the machine, and corrupt the NPU's results,
+# while the GPU's fabric clock changes speed; held at its top level, the two run together cleanly. halogen-flash-server
+# with HALOGEN_NPU_MODELS refuses to start until the clock is held. This holds it: the performance level `manual` and
+# only the top pp_dpm_fclk level, on every amdgpu device that has the control (the GPU and SoC clocks stay automatic).
+# Cost: about 1 W at idle. Run as root; halogen-fabric-clock.service runs `hold` at every boot and after a resume.
+#
+#   hold     hold it (waits up to 60 s for the GPU driver to come up)
+#   release  give it back to the driver (the performance level `auto`)
+#   status   print each device's state; exit 0 when every device is held
+set -u
+SYS="${HALOGEN_SYSFS:-/sys}"
+
+devices() {
+  for d in "$SYS"/class/drm/card*/device; do
+    [ -w "$d/pp_dpm_fclk" ] || [ -r "$d/pp_dpm_fclk" ] || continue
+    [ -e "$d/power_dpm_force_performance_level" ] || continue
+    echo "$d"
+  done | sort -u
+}
+
+held() {   # held DEVICE_DIR: the performance level high, or manual with only the top fclk level selected
+  perf=$(cat "$1/power_dpm_force_performance_level" 2>/dev/null)
+  [ "$perf" = high ] && return 0
+  [ "$perf" = manual ] || return 1
+  top=$(grep . "$1/pp_dpm_fclk" | tail -n 1)
+  [ "$(grep -c '\*' "$1/pp_dpm_fclk")" = 1 ] && [ "${top%\*}" != "$top" ]
+}
+
+cmd="${1:-status}"
+case "$cmd" in
+  hold)
+    n=0
+    while [ -z "$(devices)" ] && [ "$n" -lt 60 ]; do sleep 1; n=$((n + 1)); done
+    [ -n "$(devices)" ] || { echo "halogen-fabric-clock: no amdgpu device with a fabric clock control (pp_dpm_fclk) under $SYS/class/drm" >&2; exit 1; }
+    rc=0
+    for d in $(devices); do
+      lvl=$(grep . "$d/pp_dpm_fclk" | tail -n 1 | cut -d: -f1)
+      # the driver applies the level several seconds after the write, so the read-back waits up to 30 s
+      ok=0
+      if echo manual > "$d/power_dpm_force_performance_level" && echo "$lvl" > "$d/pp_dpm_fclk"; then
+        t=0; until held "$d" || [ "$t" -ge 300 ]; do sleep 0.1; t=$((t + 1)); done
+        held "$d" && ok=1
+      fi
+      if [ "$ok" = 1 ]; then
+        echo "halogen-fabric-clock: $d: held at level $lvl ($(grep . "$d/pp_dpm_fclk" | tail -n 1 | awk '{print $2}'))"
+      else
+        echo "halogen-fabric-clock: $d: could not hold it (level $lvl)" >&2; rc=1
+      fi
+    done
+    exit "$rc" ;;
+  release)
+    for d in $(devices); do echo auto > "$d/power_dpm_force_performance_level" && echo "halogen-fabric-clock: $d: released (auto)"; done ;;
+  status)
+    [ -n "$(devices)" ] || { echo "halogen-fabric-clock: no amdgpu device with a fabric clock control"; exit 1; }
+    rc=0
+    for d in $(devices); do
+      if held "$d"; then echo "$d: held ($(cat "$d/power_dpm_force_performance_level"))"
+      else echo "$d: NOT held ($(cat "$d/power_dpm_force_performance_level"))"; rc=1; fi
+    done
+    exit "$rc" ;;
+  *) echo "usage: halogen-fabric-clock hold | release | status" >&2; exit 2 ;;
+esac

dankclaw/host/halogen-fabric-clock.service

@@ -0,0 +1,19 @@
+# Holds the AMD GPU's fabric clock at its top level at boot and after every resume, for halogen-flash-server with
+# the NPU (HALOGEN_NPU_MODELS): on Strix Halo the GPU and the NPU at once are safe only with that clock held.
+# Install (as root), from this directory:
+#   install -m 755 halogen-fabric-clock /usr/local/sbin/
+#   install -m 644 halogen-fabric-clock.service /etc/systemd/system/
+#   systemctl daemon-reload && systemctl enable --now halogen-fabric-clock.service
+# Check: halogen-fabric-clock status. Undo: systemctl disable halogen-fabric-clock.service, then
+#   halogen-fabric-clock release (or reboot). The unit runs once at boot and once after each resume (a resume can hand
+#   the clock back to the driver); it holds nothing open, so it shows inactive between runs.
+[Unit]
+Description=Hold the AMD GPU fabric clock at its top level (GPU and NPU together)
+After=systemd-modules-load.service suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/halogen-fabric-clock hold
+
+[Install]
+WantedBy=multi-user.target suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target

dankclaw/marreta/quadlet/marreta-browser.container

@@ -23,7 +23,7 @@ After=gluetun.service network.target
 [Container]
 ContainerName=marreta_browser
 Network=gluetun.container
-Image=docker.io/lightpanda/browser:nightly
+Image=docker.io/lightpanda/browser:nightly@sha256:e72730e364d1fbf352c1149f29ffa4b278f3b1a55ff0dec3262298ebe624ec7a
 
 [Service]
 TimeoutStartSec=300

dankclaw/marreta/quadlet/marreta.container

@@ -32,7 +32,7 @@ After=gluetun.service marreta-browser.service network.target
 [Container]
 ContainerName=marreta
 Network=gluetun.container
-Image=ghcr.io/manualdousuario/marreta:latest
+Image=ghcr.io/manualdousuario/marreta:latest@sha256:30f2a34567b79b66da794b4b9423751bbbcd07ceff600d9f6a2d219e7e439362
 # Relocate off SearXNG's 8080 inside the shared netns.
 Environment=NGINX_HTTP_PORT=8085
 Environment=APP_NAME=Marreta

dankclaw/observability/quadlet/alertmanager.container

@@ -33,7 +33,7 @@ Wants=network-online.target
 
 [Container]
 ContainerName=alertmanager
-Image=docker.io/prom/alertmanager:latest
+Image=docker.io/prom/alertmanager:latest@sha256:39d8bb3966c694a124428a3a88fbcffd40bd069129ecd47c059f8a3f17c55f3b
 Network=dankclaw_default
 User=0:0
 Exec=--config.file=/etc/alertmanager/alertmanager.yml --storage.path=/alertmanager

dankclaw/observability/quadlet/alloy.container

@@ -1,55 +1,51 @@
-# Quadlet unit: alloy (rootless, boot-persistent).
+# Quadlet unit: alloy (ROOT system service, boot-persistent).
 #
-# Install path on dankclaw: ~/.config/containers/systemd/alloy.container
-# Unit name: alloy.service (container name: alloy)
+# Install path on dankclaw: /etc/containers/systemd/alloy.container
+# Unit name: alloy.service (container name: alloy) — SYSTEM manager, root.
 #
-# Log shipper. This replaces the compose service of the same name in
-# dankclaw/compose.yaml ("Observability" block).
+# WHY ROOT (not rootless, unlike its observability siblings). Alloy reads the
+# host journal. The journal files are root:systemd-journal 0640 (other::---);
+# a rootless container carries no supplementary groups and cannot open them —
+# a rootless read of system.journal returns DENIED (see the Alloy page in the
+# vault). Running as root, one loki.source.journal reads system.journal AND
+# user-1000.journal, so it carries both host logs and the rootless Quadlet
+# containers' logs. Validated 2026-10-20: a root probe read 31,825 journal
+# lines and pushed 26,966 into Loki in 20 s, zero drops.
 #
-# IMAGE. franksdank/alloy is built locally by deploy-remote.sh
-# (build_compose_image) from dankclaw/alloy/Containerfile: upstream
-# grafana/alloy plus a static busybox for the healthcheck. Pull=never keeps a
-# boot from reaching for that local tag at a registry.
+# NETWORK=host. Loki publishes 127.0.0.1:3100 (loopback only) on the host, and
+# no bridge container — rootless or root — can reach the host's loopback, so
+# this unit runs on the host network and pushes to 127.0.0.1:3100. The side
+# effect: the UI must bind loopback too, so Exec sets the listen-addr to
+# 127.0.0.1 (the old rootless unit used 0.0.0.0 only because container loopback
+# was not host loopback; under Network=host they are the same address, so the
+# unauthenticated UI stays loopback-only — nothing off-host reaches it).
 #
-# NETWORK. It joins dankclaw_default so its config's loki.write endpoint
-# resolves the container name loki:3100.
+# THE /etc/machine-id MOUNT IS LOAD-BEARING. sd_journal locates the journal
+# directory by the local machine id; without it the container opens nothing and
+# the journal source reads exactly 0 lines — silently, no error, health green.
+# This mount is the entire reason the pipeline carries data. Never remove it.
 #
-# THE UI PORT EXISTS ON PURPOSE. Alloy's UI listens on the container's own
-# 127.0.0.1 by default and cannot be moved from the config file (alloy 1.x
-# refuses the ui block), so the compose service had no reachable health
-# endpoint at all: the homepage status card for alloy reported "down"
-# continuously and there was nothing for a deploy to wait on. The command
-# below therefore moves the UI to the container's wildcard address and the
-# port is published on host loopback only, which puts alloy in the same class
-# as the other three: reachable from the host, by Caddy over the private
-# network, and by nothing off-host. Alloy has no authentication, which is
-# exactly why it is not published wider.
-#
-# SOURCES. The host journald socket and the host log/container-storage trees
-# are bind-mounted read-only; Alloy tails them and pushes to Loki. The config
-# lives at /opt/alloy/config/config.alloy, installed from
-# shared/alloy/config.alloy on every deploy.
-#
-# ORDERING. Wants/After loki.service: without a reachable Loki, Alloy's write
-# client fails and its log pipeline backs up.
+# IMAGE. Built rootless by build_compose_image alloy, then mirrored into the
+# root image store by the deployer (a root build runs in the build container's
+# own resolver and fails on DNS here). Pull=never: boot must not reach for the
+# local tag at a registry.
 
 [Unit]
 Description=Alloy log shipper (Dankclaw)
-After=network-online.target loki.service
-Wants=network-online.target loki.service
+After=network-online.target
 
 [Container]
 ContainerName=alloy
 Image=localhost/franksdank/alloy:latest
 Pull=never
-Network=dankclaw_default
-Exec=run --server.http.listen-addr=0.0.0.0:12345 /etc/alloy/config.alloy
-PublishPort=127.0.0.1:12345:12345
-Volume=/opt/alloy/config:/etc/alloy
+Network=host
+Exec=run --server.http.listen-addr=127.0.0.1:12345 /etc/alloy/config.alloy
+Volume=/opt/alloy/config:/etc/alloy:ro
+Volume=/etc/machine-id:/etc/machine-id:ro
 Volume=/run/systemd/journal/socket:/run/systemd/journal/socket:ro
-Volume=/var/log:/var/log:ro
-Volume=/var/lib/containers:/var/lib/containers:ro
-HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:12345/-/healthy
+Volume=/var/log/journal:/var/log/journal:ro
+Volume=/run/log/journal:/run/log/journal:ro
+HealthCmd=/usr/local/bin/busybox wget --spider -q http://127.0.0.1:12345/-/healthy
 HealthInterval=30s
 HealthTimeout=10s
 HealthRetries=3
@@ -60,4 +56,4 @@ Restart=always
 RestartSec=10
 
 [Install]
-WantedBy=default.target
+WantedBy=multi-user.target

dankclaw/observability/quadlet/blackbox-exporter.container

@@ -0,0 +1,56 @@
+# Quadlet unit: blackbox exporter (rootless, host network, boot-persistent).
+#
+# Install path on dankclaw:
+#   ~/.config/containers/systemd/blackbox-exporter.container
+#
+# WHY Network=host. The probes it runs are (a) the host's own LAN IP
+# (192.168.0.143:8081 SearXNG, :3005 OpenWebUI — host-process/portmap binds a
+# bridge container cannot reach; netavark DNATs published ports only, see
+# Container Host Reachability in the vault) and (b) ComfyUI's published port
+# via 127.0.0.1 (rootless --network=host shares the real host netns, so the
+# 0.0.0.0:8188 publish answers on loopback). A bridge-networked prober could
+# reach neither class reliably.
+#
+# WHY the listener binds 127.0.0.1 + LAN. A bridge container CANNOT dial
+# its own host's LAN IP (netavark hairpin gap, measured 2026-10-21), but a
+# bridge container dialing the podman gateway 169.254.1.2 lands on the host
+# LOOPBACK — that is the scrape path, and it needs this exporter to be
+# LISTENING ON LOOPBACK (proved: 200 from the prometheus container to
+# http://169.254.1.2:9115/-/healthy with a loopback bind). The gateway
+# address itself is NOT bindable from the host netns — it exists only
+# inside each rootless netns; binding it dies "bind: cannot assign
+# requested address" (live 2026-10-21). The LAN bind stays for diagnostics.
+# No public DNS name, no guard chain: internal net only, per the exposure
+# policy. The listen flag is --web.listen-address and it is REPEATABLE, one
+# flag per bind; plain --listen-address does not exist and the exporter
+# exits in 100 ms with "unknown long flag" (live 2026-10-21).
+
+[Unit]
+Description=Blackbox exporter (HTTP probe prober)
+After=network-online.target
+
+[Container]
+Image=docker.io/prom/blackbox-exporter@sha256:ea1c7aa3ef3f307cd03e85f34abb604478ccb90b567fc690da0cd9776da76b13
+ContainerName=blackbox-exporter
+Network=host
+# The image's ENTRYPOINT is /bin/blackbox_exporter itself (proved live
+# 2026-10-21: `podman run IMAGE sh` dies "unexpected sh"), so Exec= carries
+# FLAGS ONLY — repeating the binary makes it a positional arg and kingpin
+# exits "unexpected /bin/blackbox_exporter".
+Exec=--config.file=/etc/blackbox_exporter/config.yml --web.listen-address=127.0.0.1:9115 --web.listen-address=192.168.0.143:9115
+Volume=/opt/blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+# Plain exec form like every sibling unit. Shell form (`/bin/sh -c "…"`)
+# is passed as one quoted token and quadlet then folds the following
+# --health-interval/--health-retries (and the image name) into the command
+# string — unit died twice that way, live 2026-10-21. wget only; this
+# busybox variant has no curl, and host-network wget answers on loopback.
+HealthCmd=wget -q -O /dev/null http://127.0.0.1:9115/-/healthy
+HealthInterval=30s
+HealthRetries=3
+
+[Service]
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=default.target

dankclaw/observability/quadlet/cadvisor.container

@@ -0,0 +1,56 @@
+# Quadlet unit: cadvisor (rootless, boot-persistent) — DANKCLAW.
+#
+# Install path on dankclaw: ~/.config/containers/systemd/cadvisor.container
+# Unit name: cadvisor.service (container name: cadvisor)
+#
+# WHY THIS EXISTS. The app stack is ~34 rootless Podman containers and until
+# now nothing reported per-container CPU/memory — Grafana could only see the
+# host (node_exporter) and the LLM engines. cadvisor reads the host cgroup
+# tree and exports container_cpu_* / container_memory_* per container.
+#
+# THE LOAD-BEARING FLAG IS --cgroupns=host. Podman starts every container in a
+# PRIVATE cgroup namespace, so an exporter container sees only its OWN cgroup
+# subtree: cadvisor exported a single root series until this flag was added,
+# then it walked the whole tree (996 per-container series measured 2026-10-20).
+# The quadlet unit name is carried in the cgroup path — app.slice/<name>.service/
+# libpod-payload-<hex> — so Prometheus relabels that path into a `container`
+# label (shared/prometheus/prometheus.yml, cadvisor-dankclaw job) and no
+# libpod/podman socket is needed to name containers.
+#
+# BINDING. cadvisor has no per-interface listen flag (unlike node_exporter's
+# --web.listen-address), so it binds 0.0.0.0 INSIDE its own network namespace.
+# The bridge PublishPort is what keeps the unauthenticated endpoint off the
+# host's other interfaces: it publishes exactly loopback (host-local probes and
+# the deployer's health gate) and this host's LAN address (Prometheus, a bridge
+# container, reaches a host-published port at its LAN address — the same netavark
+# rule node-exporter relies on, only here the port really is published).
+#
+# It is rootless on purpose: dank owns the app containers' cgroups, so a
+# rootless cadvisor reads every one of them; root would buy nothing here.
+
+[Unit]
+Description=cAdvisor container metrics (Dankclaw)
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+ContainerName=cadvisor
+Image=gcr.io/cadvisor/cadvisor:v0.49.1@sha256:3cde6faf0791ebf7b41d6f8ae7145466fed712ea6f252c935294d2608b1af388
+PodmanArgs=--pid=host --cgroupns=host
+PublishPort=127.0.0.1:8080:8080
+PublishPort=192.168.0.143:8080:8080
+Volume=/sys:/sys:ro
+Volume=/:/rootfs:ro
+Exec=--housekeeping_interval=30s
+HealthCmd=wget --spider -q http://localhost:8080/healthz
+HealthInterval=30s
+HealthTimeout=10s
+HealthRetries=3
+HealthStartPeriod=40s
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target

dankclaw/observability/quadlet/grafana.container

@@ -34,7 +34,7 @@ Wants=network-online.target prometheus.service
 
 [Container]
 ContainerName=grafana
-Image=docker.io/grafana/grafana:latest
+Image=docker.io/grafana/grafana:latest@sha256:488984b716c0862639f91a698adbe2862677a5a07bad42da5f03f3030883b205
 Network=dankclaw_default
 User=0:0
 PublishPort=127.0.0.1:3002:3000
@@ -47,9 +47,24 @@ Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${GRAFANA_OAUTH_CLIENT_SECRET}
 # client_id query parameter — /application/o/<slug>/authorize/ does not exist.
 Environment=GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://auth.franksdank.cloud/application/o/authorize/
 Environment=GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://auth.franksdank.cloud/application/o/token/
-# ?api=1 = introspection-style userinfo: returns custom claims (groups) too.
-Environment=GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.franksdank.cloud/application/o/userinfo/?api=1
-Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=false
+# Standard OIDC userinfo. Grafana v13 fetches <api_url>/emails for the address
+# ONLY when the token carries no `email` claim (authentik has no /emails path and
+# 404s it, failing the login) — so the claim must actually be there: the fix is
+# the grafana-email-scope mapping (see scripts/authentik-grafana-sso-fix.sh),
+# not a Grafana flag. Do NOT append ?api=1 to this URL: Grafana then GETs
+# <...>/userinfo/?api=1/emails, a 403.
+Environment=GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.franksdank.cloud/application/o/userinfo/
+# authentik's scopes are openid/email/profile. `user:email` (a GitLab-ism) is
+# not a known scope here; requesting it makes the authorize view crash.
+# Quoted: quadlet word-splits Environment=, so an unquoted spaced list
+# truncates SCOPES to its first token.
+Environment="GF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile"
+Environment=GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email
+Environment=GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=sub
+Environment=GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=preferred_username
+# First SSO login auto-provisions the user. Safe: unauthenticated traffic never
+# reaches this callback — the Anubis -> Authentik guard fronts every request.
+Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true
 Environment=GF_AUTH_GENERIC_OAUTH_AUTO_UPSERT=true
 Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_ORG=true
 Environment=GF_AUTH_GENERIC_OAUTH_ORG_ID=1

dankclaw/observability/quadlet/node-exporter.container

@@ -33,7 +33,7 @@ Wants=network-online.target
 
 [Container]
 ContainerName=node-exporter
-Image=docker.io/prom/node-exporter:latest
+Image=docker.io/prom/node-exporter:latest@sha256:1b4e4438faca4dd7e001dd445d161a4a2091b0fededa84093b3a8dfeae1f1be0
 Network=host
 PodmanArgs=--pid=host
 Exec=--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/rootfs --web.listen-address=127.0.0.1:9100 --web.listen-address=192.168.0.143:9100

dankclaw/observability/quadlet/prometheus.container

@@ -43,8 +43,16 @@ Wants=network-online.target
 
 [Container]
 ContainerName=prometheus
-Image=docker.io/prom/prometheus:latest
+Image=docker.io/prom/prometheus:latest@sha256:fa3302279710be2f21a8301ec17a8b4f2301fd829bd113839b3b6b02b6f191a2
 Network=dankclaw_default
+# Same-host scrape targets are dialed at host.containers.internal (the podman
+# gateway): a bridge container cannot dial its own host's LAN IP (netavark
+# hairpin gap, measured 2026-10-21), and traffic to the gateway lands on host
+# loopback / published ports (cadvisor/gpu jobs and the blackbox scrape all
+# ride it). The name resolves natively — NO HostAddr key: quadlet 5.7.0 calls
+# it unsupported and the WHOLE user quadlet batch then fails to generate,
+# silently orphaning every other unit (live 2026-10-21: prometheus.service
+# vanished while its container kept running as an orphan).
 User=0:0
 Exec=--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=30d --web.enable-lifecycle
 PublishPort=127.0.0.1:9090:9090

dankclaw/open-notebook/quadlet/open-notebook-surrealdb.container

@@ -48,13 +48,22 @@
 # without `podman unshare` — accepted for a datastore nobody edits by hand;
 # episode AUDIO is a host directory for the opposite reason (see the app unit).
 #
-# HEALTH. The base image is distroless: no shell, no curl, no wget, so no
-# CMD-SHELL probe and no wget/curl probe is possible here (same class of unit
-# as loki, whose probe had to move into the image). SurrealDB ships its own
-# probe instead — `surreal is-ready` connects to ws://localhost:8000 by default
-# and needs no credentials — verified against this digest: it prints `OK` and
-# exits 0 while the server is up. Argument-only HealthCmd, like every unit on
-# this host, because the quadlet generator splits it on spaces.
+# HEALTH. The base image is distroless — measured against this digest, not even
+# `ls` exists (`exec: "ls": executable file not found in $PATH`), so there is no
+# `/bin/sh`, no curl and no wget. SurrealDB ships its own probe instead:
+# `surreal is-ready` connects to ws://localhost:8000 by default, needs no
+# credentials, prints `OK` and exits 0 while the server is up.
+#
+# It must be handed over as a QUADLET-SIDE JSON ARRAY, not a bare command.
+# podman wraps a bare `HealthCmd` string as `CMD-SHELL` (measured on this host's
+# podman 5.7: `--health-cmd "/surreal is-ready"` stores
+# `{"Test":["CMD-SHELL","/surreal is-ready"]}`), and with no `/bin/sh` in the
+# image that probe fails in ~30 ms with empty output on every interval while the
+# database answers perfectly — which is exactly how this service's first deploy
+# died on 2026-10-19 (`did not become healthy in 90s`). The quoted array survives
+# the generator (`--health-cmd "[\"/surreal\",\"is-ready\"]"`) and stores
+# `["CMD","/surreal","is-ready"]`, which execs directly and goes healthy. Do not
+# "simplify" it back to a bare command.
 
 [Unit]
 Description=SurrealDB (open-notebook datastore)
@@ -70,7 +79,7 @@ EnvironmentFile=/home/dank/serverAppConfigs/scripts/open-notebook.env
 EnvironmentFile=/home/dank/Containers/open-notebook/env
 Exec=start --log info rocksdb:/mydata/mydatabase.db
 Volume=open-notebook-db:/mydata
-HealthCmd=/surreal is-ready
+HealthCmd=["/surreal","is-ready"]
 HealthInterval=30s
 HealthTimeout=10s
 HealthRetries=3

dankclaw/open-notebook/quadlet/open-notebook.container

@@ -33,17 +33,21 @@
 # and 5055 is exactly the port seerr already holds on this host
 # (`PublishPort=127.0.0.1:5055:5055`; `ss -ltn` on Dankclaw 2026-10-18 shows
 # 127.0.0.1:5055 taken). So the API leaves the container on host port 5056.
-# That forces one setting: the frontend auto-detects the API as
-# `<host-you-opened>:5055`, so API_URL in scripts/open-notebook.env must name
-# the remapped port explicitly, or the UI loads and every call fails. Both
-# ports are published on all interfaces because the consumer is a browser on
-# the LAN — the fleet's browser-facing services (open-webui, comfyui, gitea)
-# bind the same way, while loopback-only is reserved for services whose user
-# path is the guard chain. This service has NO vhost and no Caddy route: it is
-# internal-only until it is guarded, and its own password
-# (OPEN_NOTEBOOK_PASSWORD) is what stands in front of it. That password is not
-# decoration: with it unset the API answers everyone on the LAN, including the
-# unauthenticated episode-audio download.
+# Only 8502 is published on all interfaces, because it is the single upstream
+# the guard chain dials: notebook.franksdank.cloud → anubis → authentik →
+# http://:8017 → 192.168.0.143:8502 (dankclaw/Caddyfile), and this Caddy lives
+# in root's storage, so a published port is the only way it can reach a rootless
+# quadlet. The API needs no outward face at all — the image's own Next.js layer
+# rewrites /api/* to the FastAPI beside it, so one origin serves UI and API —
+# and the fleet's rule is that loopback-only is for services whose user path is
+# the guard chain, which this became on 2026-10-19; 5056 stays published on
+# loopback because the deployer's health gate dials it there. That is also why
+# API_URL in scripts/open-notebook.env names the vhost rather than a port: left
+# to itself the frontend auto-detects `<host-you-opened>:5055`, and the UI loads
+# while every call fails. OPEN_NOTEBOOK_PASSWORD is not made redundant by the
+# chain: it is what keeps the API honest for everything else on this host, and
+# with it unset the API answers anyone, including the unauthenticated
+# episode-audio download.
 #
 # VOLUMES. `/app/data` is a named volume: uploads, the LangGraph SQLite
 # checkpoints and the extraction caches the image keeps under it
@@ -81,7 +85,7 @@ Pull=never
 Network=dankclaw_default
 Network=open-notebook.network
 PublishPort=0.0.0.0:8502:8502
-PublishPort=0.0.0.0:5056:5055
+PublishPort=127.0.0.1:5056:5055
 EnvironmentFile=/home/dank/serverAppConfigs/scripts/open-notebook.env
 EnvironmentFile=/home/dank/Containers/open-notebook/env
 Volume=open-notebook-data:/app/data

dankclaw/open-webui/quadlet/open-webui.container

@@ -27,7 +27,7 @@ After=network.target
 [Container]
 ContainerName=open-webui
 Network=open-webui.network
-Image=ghcr.io/open-webui/open-webui:main
+Image=ghcr.io/open-webui/open-webui:main@sha256:0b399a8ba34558e181d316750455f2a95376a38cd60a61b9c942e138898944bd
 PublishPort=3005:8080
 Environment=OPENAI_API_BASE_URL=http://192.168.0.143:7770
 Environment=ANONYMIZED_TELEMETRY=False

dankclaw/pihole/addn-hosts

@@ -45,3 +45,4 @@
 192.168.0.143 pihole.franksdank.cloud
 192.168.0.143 seerr.franksdank.cloud
 192.168.0.143 marreta.franksdank.cloud
+192.168.0.143 notebook.franksdank.cloud

dankclaw/pihole/quadlet/pihole.container

@@ -24,7 +24,7 @@ Wants=network-online.target
 
 [Container]
 ContainerName=pihole
-Image=docker.io/pihole/pihole:latest
+Image=docker.io/pihole/pihole:latest@sha256:d34134716fc32038b052a8674c7c6879e0ff21f0e6d8af345166fffa2c27c113
 Network=slirp4netns
 PublishPort=0.0.0.0:53:53/udp
 PublishPort=0.0.0.0:53:53/tcp

dankclaw/searxng/quadlet/gluetun.container

@@ -16,7 +16,7 @@ After=searxng-network.service network.target
 [Container]
 ContainerName=gluetun
 Network=searxng.network
-Image=docker.io/qmcgaw/gluetun:latest
+Image=docker.io/qmcgaw/gluetun:latest@sha256:4af0c804003481e100180e1f8d4d5e9c48163fedc0f88d061881b615aef9d183
 AddCapability=NET_ADMIN
 AddDevice=/dev/net/tun:/dev/net/tun
 Environment=VPN_SERVICE_PROVIDER=${VPN_SERVICE_PROVIDER}

dankclaw/searxng/quadlet/searxng-valkey.container

@@ -16,7 +16,7 @@ After=searxng-network.service network.target
 ContainerName=searxng-valkey
 Network=searxng.network
 NetworkAlias=valkey
-Image=docker.io/valkey/valkey:9-alpine
+Image=docker.io/valkey/valkey:9-alpine@sha256:bf09d58e96d6b4524a15396b65b242bbe18beebbcc15a04d8ee75ddac62fae0b
 Volume=/opt/searxng/valkey:/data
 
 [Service]

dankclaw/searxng/quadlet/searxng.container

@@ -16,7 +16,7 @@ After=gluetun.service searxng-valkey.service network.target
 [Container]
 ContainerName=searxng
 Network=gluetun.container
-Image=ghcr.io/searxng/searxng:latest
+Image=ghcr.io/searxng/searxng:latest@sha256:f4aa660c54d062dedb268be84e821ea24e39f48c7a496af8dd02cc9eedb93847
 Environment=SEARXNG_SECRET=${SEARXNG_SECRET}
 Environment=SEARXNG_BASE_URL=${SEARXNG_BASE_URL}
 Volume=/opt/searxng/config:/etc/searxng

dankclaw/seerr/quadlet/seerr.container

@@ -29,7 +29,7 @@ Wants=network-online.target
 
 [Container]
 ContainerName=seerr
-Image=ghcr.io/seerr-team/seerr:v3.4.1
+Image=ghcr.io/seerr-team/seerr:v3.5.1@sha256:d8fab675b55ad293ded98fd71efcfd46054cbad9746534ae1cc4794a88c0b3be
 Network=dankclaw_default
 PublishPort=127.0.0.1:5055:5055
 Volume=/home/dank/Containers/seerr/data:/app/config

dankclaw/udev/98-halogen-npu-access.rules

@@ -0,0 +1,26 @@
+# /etc/udev/rules.d/98-halogen-npu-access.rules
+#
+# Why the NPU node is world-writable: `frank-dragoon` runs rootless podman under a
+# unit with `User=dank`, and the container's user namespace maps exactly one uid and
+# one gid (dank). `/dev/accel/accel0` is `root:render 0660`, so inside the container
+# the node reads `nobody:nogroup` and the engine dies with its own diagnosis:
+#
+#   halogen npu: /dev/accel/accel0 is here, but this process cannot open it
+#     (crw-rw---- nobody:nogroup).
+#
+# `--group-add keep-groups` is the vendor's advice and it does NOT work in this
+# shape: measured 2026-10-19 from the host as a `dank` shell that was already in
+# `render`, with and without the flag, the node still read `nobody:nogroup` and the
+# open was refused — an unmapped gid cannot be granted to a process inside a
+# namespace that does not map it. Same conclusion `99-comfyui-gpu-access.rules`
+# reached for `/dev/kfd` from the systemd-user-spawn direction: the mode is the only
+# spawn-context-independent door short of running the stack rootful.
+#
+# Blast radius: any process in this host's single uid domain (every rootless
+# container here shares it) can submit to the XDNA2 NPU. The NPU holds no user data
+# and no privileged context — the same trade already accepted for the GPU compute
+# nodes beside it. Installed by `deploy-remote.sh` when the NPU lane is armed and left
+# in place afterwards — the mode costs nothing while the lane is idle. `serve-frank-dragoon.sh`
+# refuses to arm the lane on a node that is not other-accessible, so a host that lost this
+# file fails loudly at boot instead of crash-looping.
+KERNEL=="accel[0-9]*", SUBSYSTEM=="accel", MODE="0666"

dankclaw/voiceservice/Containerfile.engine

@@ -19,11 +19,14 @@
 # is recorded in the parent tree, so cloning the parent commit recursively is
 # reproducible without a second pin.
 #
-# THE BUILD LINE IS THE ONE PROVEN ON DANKCLAW, verbatim:
+# THE BUILD LINE IS THE ONE PROVEN ON DANKCLAW, verbatim — with the Vulkan door
+# open since 2026-10-19. That configure was run in this exact base image and
+# answers `Found Vulkan: /usr/lib/x86_64-linux-gnu/libvulkan.so (found version
+# "1.3.275") found components: glslc` then `Including Vulkan backend`:
 #
 #   cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_CCACHE=OFF \
 #       -DGGML_BACKEND_DL=OFF -DGGML_CPU_ALL_VARIANTS=OFF -DGGML_OPENMP=ON \
-#       -DGGML_BLAS=OFF -DGGML_CUDA=OFF -DGGML_HIP=OFF -DGGML_VULKAN=OFF \
+#       -DGGML_BLAS=OFF -DGGML_CUDA=OFF -DGGML_HIP=OFF -DGGML_VULKAN=ON \
 #       -DGGML_METAL=OFF -DGGML_SYCL=OFF -DGGML_RPC=OFF \
 #   && cmake --build build -j 8
 #
@@ -42,8 +45,12 @@
 #     defaults it ON outside wasm/cross-compiles, i.e. -march=native for THIS
 #     host — correct while build host == run host, wrong if anyone ever builds
 #     this image elsewhere, which is why the fleet builds it on Dankclaw.
-#   CUDA/HIP/VULKAN/METAL/SYCL/RPC all OFF: this is a CPU endpoint (the trial
-#     ran on CPU; the box's GPUs belong to the text endpoints).
+#   CUDA/HIP/METAL/SYCL/RPC stay OFF, VULKAN is ON: the endpoint computes on the
+#     gfx1151 iGPU (decision and co-tenancy gates:
+#     report/tts-npu-spike-verdict-2026-10-19.md, shape in [[Frank TTS Engine]]).
+#     Nothing else is available to this image: no ROCm userland, no NVIDIA
+#     userland, no macOS, no oneAPI, and RPC would add a second process to
+#     supervise on a box that already has one engine per card.
 #
 # BUILD SHARED LIBS, RUNTIME SMALL. BUILD_SHARED_LIBS defaults ON on Linux, so
 # tts-server links libggml*.so rather than swallowing them; the runtime stage
@@ -67,9 +74,28 @@ ARG BASE_IMAGE=docker.io/library/ubuntu:24.04@sha256:68ee22004a9109b44973313b02f
 # ------------------------------------------------------------------ build
 FROM ${BASE_IMAGE} AS build
 
+# The three Vulkan build deps are not optional and not interchangeable:
+#   libvulkan-dev    vulkan/vulkan.h + the VulkanHeaders/VulkanLoader CMake
+#                    configs, i.e. what find_package(Vulkan) resolves to.
+#   glslc            ggml compiles its shaders with glslc, not glslangValidator.
+#                    Ubuntu's glslang-tools ships glslangValidator and NOT glslc,
+#                    so without this package `find_package(Vulkan COMPONENTS
+#                    glslc REQUIRED)` fails and there is no Vulkan backend.
+#                    The archive's glslc is old (2023.8): it compiles
+#                    GL_KHR_cooperative_matrix — the path ggml's quantized
+#                    matmul needs — but reports cooperative_matrix2,
+#                    integer_dot_product, bfloat16 and the micro-float
+#                    extensions as unsupported, so those shader families are
+#                    compiled out. If a Vulkan prefill ever reads as
+#                    inexplicably slow, suspect this compiler before the device;
+#                    the fix is a pinned glslang build passed as
+#                    -DVulkan_GLSLC_EXECUTABLE=<path>, not a different runtime.
+#   spirv-headers    find_package(SPIRV-Headers CONFIG REQUIRED); the distro
+#                    package ships /usr/share/cmake/SPIRV-Headers/.
 RUN apt-get update -qq \
     && apt-get install -y -qq --no-install-recommends \
         ca-certificates cmake g++ git make \
+        libvulkan-dev glslc spirv-headers \
     && rm -rf /var/lib/apt/lists/*
 
 # Checked out detached at a full SHA, then the recorded submodule. No branch
@@ -90,7 +116,7 @@ RUN cmake -B build -DCMAKE_BUILD_TYPE=Release \
         -DGGML_BLAS=OFF \
         -DGGML_CUDA=OFF \
         -DGGML_HIP=OFF \
-        -DGGML_VULKAN=OFF \
+        -DGGML_VULKAN=ON \
         -DGGML_METAL=OFF \
         -DGGML_SYCL=OFF \
         -DGGML_RPC=OFF \
@@ -102,9 +128,21 @@ FROM ${BASE_IMAGE}
 # libgomp1: the OpenMP runtime the build above links against. curl: the unit's
 # HealthCmd. ca-certificates: so that curl can speak https if anyone ever
 # points a probe at a URL with a scheme this host terminates elsewhere.
+#
+# libvulkan1 + mesa-vulkan-drivers are what make `--device Vulkan0` mean
+# anything at runtime: libvulkan1 is the loader libggml-vulkan.so dlopens, and
+# mesa-vulkan-drivers is the RADV ICD that claims the gfx1151. Loader without an
+# ICD enumerates ZERO devices, and tts-server then dies in device lookup AFTER
+# both GGUFs have been read — the journal reads as a model fault, not a missing
+# driver. Enumerated from this base on 2026-10-19 with `--device /dev/dri
+# --group-add keep-groups` plus the host's ComfyUI udev rule: "Radeon 8060S
+# Graphics (RADV GFX1151)", API 1.4.318 (the heap figure is owned by
+# [[Frank TTS Engine]]; it is the GTT-backed aperture, not the ~2 GiB DRM
+# carve-out). No NVIDIA ICD is installed on purpose: one Vulkan device in the
+# enumeration is what makes the NAME Vulkan0 safe to write down.
 RUN apt-get update -qq \
     && apt-get install -y -qq --no-install-recommends \
-        ca-certificates curl libgomp1 \
+        ca-certificates curl libgomp1 libvulkan1 mesa-vulkan-drivers \
     && rm -rf /var/lib/apt/lists/*
 
 # Same unprivileged uid as the frank-tts façade (10000): the voice packs and the

dankclaw/voiceservice/quadlet/frank-tts-engine.container

@@ -40,12 +40,28 @@
 # speaks to the server from inside its own namespace, which is where the address
 # is valid.
 #
-# MEMORY / NO GPU. Peak RSS measured 5.7 GiB with both GGUFs resident
-# (2026-10-18 trial); no memory limit is set because the box has the room and a
-# limit would convert a busy moment into an OOM kill mid-utterance. No
-# /dev/kfd, no /dev/dri, no keep-groups: every GPU backend is compiled out of
-# the image, the endpoint is CPU-only by decision, and the GPUs on this host
-# belong to the text endpoints.
+# MEMORY / THE RENDER NODE. Host RSS is ~1.7 GiB, not the size of the GGUFs:
+# the Vulkan backend copies both tensors into device memory at load, so they
+# stop counting in the process's RSS (measured after the 2026-10-19 cutover —
+# `podman stats` / `ps -o rss`). No memory limit is set: the box has the room
+# and a limit would convert a busy moment into an OOM kill mid-utterance.
+#
+# `AddDevice=/dev/dri` is what makes the iGPU visible; without it the Vulkan
+# ICD enumerates nothing and the server dies on device lookup after it has
+# already read both GGUFs. `GroupAdd=keep-groups` is belt-and-braces, not the
+# thing that works here: the units podman spawns carry no supplementary groups
+# at all, and this host grants render access a different way — the udev rule
+# that makes the render nodes world-writable (same rule ComfyUI rides, see
+# dankclaw/comfyui). Measured: `id` inside this container lists only the
+# container's own group, and `tts-server` still holds renderD129.
+# The node is NOT named: this host is hybrid, renderD128 is the NVIDIA card and
+# renderD129 the amdgpu iGPU, so a node number in the unit is a guess that a
+# re-probe can invalidate, while the directory hands the container every render
+# node and the driver picks by name.
+#
+# /dev/kfd stays absent on purpose: that is the ROCm compute interface, owned by
+# the engines that use HIP. A Vulkan endpoint has no use for it, and handing a
+# container a second way onto the GPU is surface for nothing.
 #
 # THE VOICE MOUNT IS READ-ONLY AND IS FOR THE WARMER, NOT THE ENGINE. A voice
 # becomes known to the engine only through POST /v1/audio/voices, and the
@@ -98,6 +114,9 @@ Network=frank-tts-engine.network
 EnvironmentFile=/home/dank/serverAppConfigs/scripts/frank-tts-engine.env
 Volume=/home/dank/models/Qwen3-TTS-GGUF:/models:ro
 Volume=/home/dank/voices:/voices:ro
+# Render access for the iGPU (see MEMORY / THE RENDER NODE above).
+AddDevice=/dev/dri
+GroupAdd=keep-groups
 HealthCmd=curl -fsS http://127.0.0.1:8880/health
 HealthInterval=30s
 HealthTimeout=10s

dankclaw/voiceservice/tts-engine-start.sh

@@ -7,7 +7,7 @@
 # supervision, no retries, no logging of its own. PID 1 stays the server, so
 # `podman stop` and the unit's Restart=always drive it directly.
 #
-# THE FLAGS THAT ARE NOT HERE, and must stay absent:
+# THE FLAG THAT MUST STAY ABSENT:
 #
 #   --threads / -t   does not exist in this build. src/backend.h computes the
 #                    CPU thread count as std::thread::hardware_concurrency()/2
@@ -18,13 +18,19 @@
 #                    placement — pinning is deliberately not done anywhere in
 #                    this stack. See the env file header for the same warning
 #                    aimed at the operator.
-#   --device         left at its default. Every GPU backend is compiled out of
-#                    the image, so the default device pick can only land on
-#                    CPU; naming a device would add a way to be wrong.
 #
-# Everything defaults to the shape the 2026-10-18 trial measured, so an empty
-# environment still boots the decided configuration: 1.7 B talker Q4_K_M with
-# the Q8_0 tokenizer, one request at a time, on the private bridge.
+# --device IS here, but conditionally: it reaches the binary only when
+# QWENTTS_DEVICE is non-empty, and empty is the CPU rollback — the binary then
+# takes its own default device, which is CPU because the CPU backend is always
+# built. One line in scripts/frank-tts-engine.env moves the endpoint between the
+# two shapes, with no rebuild. What must never reach argv is a device NAME the
+# loader does not enumerate: that is not refused by the argument parser, it dies
+# during device lookup with the loader's own error, so the journal reads as a
+# load that failed rather than a configuration refusal.
+#
+# Everything defaults to the shape this endpoint runs: 1.7 B talker Q4_K_M with
+# the Q8_0 tokenizer, one request at a time, on the private bridge, on the
+# gfx1151 iGPU. An empty environment still boots that shape.
 set -euo pipefail
 
 # The binary path. Overridable for one reason: the bats seam runs a stub
@@ -60,13 +66,33 @@ set -euo pipefail
 : "${QWENTTS_LANG:=auto}"
 
 # One request at a time. The server-side queue is a serialised FIFO at this
-# setting, which is what keeps the measured per-voice timings true; batching
-# on a CPU build splits the same cores between two syntheses and the warm RTF
-# margin (0.55-0.67 against real time 1.0) disappears. The engine's own
-# default is also 1; it is written here because it is a decision, not an
-# accident.
+# setting, which is what keeps the per-voice timings predictable. A second
+# concurrent synthesis shares the device with the first — and on this box that
+# device is also carrying frank-dragoon — so it buys no throughput and it
+# doubles the wait for the caller behind it. The engine's own default is also 1;
+# written here because it is a decision, not an accident.
 : "${QWENTTS_MAX_BATCH:=1}"
 
+# The compute device, NAMED and not numbered: `Vulkan0` is how ggml enumerates
+# the gfx1151 iGPU on this host. An index would follow whatever order the
+# loader's device list returns; a name says which device this endpoint means.
+# The container sees exactly one Vulkan device because the image carries the
+# Mesa ICD and no NVIDIA one, so the 3090 is not in the enumeration at all.
+#
+# EMPTY IS THE CPU ROLLBACK: a value that is set but EMPTY passes no `--device`,
+# the binary picks its own default, and the CPU backend is always built — which
+# is why moving this endpoint between the iGPU and the CPU is one line in
+# scripts/frank-tts-engine.env and not a rebuild.
+#
+# That is also why the default below is NOT `${QWENTTS_DEVICE:=Vulkan0}`: `:=`
+# fires on a set-but-empty variable as well as an unset one, so the rollback line
+# `QWENTTS_DEVICE=` would be rewritten back to Vulkan0 and the endpoint would sit
+# on the shared GPU with no way out short of a rebuild. Only UNSET takes the
+# default.
+if [[ ! -v QWENTTS_DEVICE ]]; then
+    QWENTTS_DEVICE=Vulkan0
+fi
+
 # Fail loudly BEFORE binding, so the journal says which artifact is missing
 # instead of the unit reporting a container that starts and dies. This is the
 # self-healing path: Restart=always re-runs it every RestartSec, and the same
@@ -79,11 +105,20 @@ for artifact in "$QWENTTS_MODEL" "$QWENTTS_CODEC"; do
     fi
 done
 
-exec "$QWENTTS_BIN" \
-    --model "$QWENTTS_MODEL" \
-    --codec "$QWENTTS_CODEC" \
-    --alias "$QWENTTS_ALIAS" \
-    --host "$QWENTTS_HOST" \
-    --port "$QWENTTS_PORT" \
-    --lang "$QWENTTS_LANG" \
+ARGS=(
+    --model "$QWENTTS_MODEL"
+    --codec "$QWENTTS_CODEC"
+    --alias "$QWENTTS_ALIAS"
+    --host "$QWENTTS_HOST"
+    --port "$QWENTTS_PORT"
+    --lang "$QWENTTS_LANG"
     --max-batch "$QWENTTS_MAX_BATCH"
+)
+
+# Only when a device is named. An empty QWENTTS_DEVICE has to produce NO flag,
+# because "the rollback shape" is argv identical to the pre-iGPU endpoint.
+if [[ -n "$QWENTTS_DEVICE" ]]; then
+    ARGS+=(--device "$QWENTTS_DEVICE")
+fi
+
+exec "$QWENTTS_BIN" "${ARGS[@]}"

dankserver/calibre-server/quadlet/calibre-server.container

@@ -43,7 +43,7 @@ After=network.target
 
 [Container]
 ContainerName=calibre-server
-Image=lscr.io/linuxserver/calibre:9.15.0
+Image=lscr.io/linuxserver/calibre:9.17.0@sha256:b198fc03a6abc6be7e251a9958e50e74634b00e447c7b25e92c191c8077ba378
 Network=pasta
 EnvironmentFile=/home/dank/Containers/calibre-server/env
 Environment=PUID=1000

dankserver/calibre-web/quadlet/calibre-web.container

@@ -16,7 +16,7 @@ After=network.target
 
 [Container]
 ContainerName=calibre-web
-Image=lscr.io/linuxserver/calibre-web:0.6.27-ls402
+Image=lscr.io/linuxserver/calibre-web:0.6.27-ls407@sha256:470b4ed9a7d6ea1475fef17dbcf957cb60880910296d226bd0dca652c05f68ca
 Network=pasta
 Environment=DOCKER_MODS=linuxserver/mods:universal-calibre
 Environment=LANG=en_US.UTF-8

dankserver/compose.yaml

@@ -94,34 +94,12 @@ services:
       timeout: 10s
       retries: 3
 
-  # ═══════════════════════════════════════════════
-  # Jellyfin — media server (movies, TV, music)
-  # ═══════════════════════════════════════════════
-  jellyfin:
-    image: jellyfin/jellyfin:latest
-    container_name: jellyfin
-    restart: unless-stopped
-    ports:
-      - "8096:8096/tcp"
-      - "8920:8920/tcp"
-      - "7359:7359/udp"
-      - "1900:1900/udp"
-    environment:
-      - JELLYFIN_PublishedServerUrl=http://192.168.0.235:8096
-      - DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true
-    volumes:
-      - /opt/jellyfin/config:/config
-      - /opt/jellyfin/cache:/cache
-      - /mnt/entropy:/data:ro
-    devices:
-      - /dev/dri:/dev/dri
-    group_add:
-      - "992"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8096/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
+  # Jellyfin left this file when it became a Quadlet: the container is owned by
+  # dankserver/jellyfin/quadlet/jellyfin.container, its data roots are under
+  # /home/dank/Containers/jellyfin, and its library arrives through the SSD
+  # overlay at /var/lib/media-meta/view (scripts/media-view.sh). The block that
+  # was here named /opt/jellyfin/* and a read-only array bind — a second owner for
+  # the container name `jellyfin`, wrong on every path. One owner per container.
 
   # ═══════════════════════════════════════════════
   # Calibre-Web — e-book web frontend

dankserver/copyparty/quadlet/copyparty.container

@@ -21,7 +21,13 @@ After=network.target
 
 [Container]
 ContainerName=copyparty
-Image=ghcr.io/9001/copyparty-ac:latest
+# Digest-pinned: upstream stopped publishing version tags after 1.19.15, so
+# the pin rides :latest@digest (the tag in the reference is cosmetic; the
+# digest is the pull). Pinned 2026-10-21 at the build whose label is
+# 1.20.26 - closes GHSA-3c88-p299-7hvr (1.20.25) and GHSA-hg99-jqfp-rf76
+# (1.20.26); the previously running build was the rolling :latest at
+# 1.20.24 (issue #73 on command-center).
+Image=ghcr.io/9001/copyparty-ac:latest@sha256:7f77459160a4bf34fdca79f2965dff4f0c1d7069a3d9cdfb3632acb041dd2f6a
 Network=dankserver_appnet
 EnvironmentFile=/home/dank/Containers/copyparty/env
 Exec=-a ${COPAY_ACCOUNT}:${COPAY_PASSWORD} -v /w::r:ra,dank

dankserver/host/crypttab

@@ -0,0 +1 @@
+entropy	UUID=cd86c728-459e-4f3a-bcc2-41ecf84685ec	/etc/entropy	nofail,luks

dankserver/host/fstab

@@ -0,0 +1,18 @@
+# /etc/fstab: static file system information.
+#
+# Use 'blkid' to print the universally unique identifier for a
+# device; this may be used with UUID= as a more robust way to name devices
+# that works even if disks are added and removed. See fstab(5).
+#
+# <file system> <mount point>   <type>  <options>       <dump>  <pass>
+# / was on /dev/nvme0n1p2 during curtin installation
+/dev/disk/by-uuid/4f3731d2-fb7b-48e7-a79f-0e33cb8a4162 / ext4 defaults 0 1
+# /boot/efi was on /dev/nvme0n1p1 during curtin installation
+/dev/disk/by-uuid/183C-442D /boot/efi vfat defaults 0 1
+#/dev/disk/by-label/archive-one /mnt/archive-one auto nosuid,nodev,nofail,x-gvfs-show 0 0
+#/dev/disk/by-label/archive-two /mnt/archive-two auto nosuid,nodev,nofail,x-gvfs-show 0 0
+# /dev/md0 is a LUKS container, not a filesystem; its ext4 is mounted via
+# /dev/mapper/entropy on the next line. Mounting /dev/md0 directly fails.
+#/dev/md0 /mnt/md0 ext4 auto,nofail,nosuid,nodev 0 0 
+/dev/mapper/entropy /mnt/entropy ext4 auto,nosuid,nofail,nodev,x-gvfs-show 0 0
+/swapfile none swap sw 0 0

dankserver/host/systemd/media-meta-view.mount

@@ -0,0 +1,30 @@
+# /etc/systemd/system/var-lib-media\x2dmeta-view.mount
+# (repo copy: dankserver/host/systemd/media-meta-view.mount — systemd names a
+# mount unit after its ESCAPED path, so the host file carries the \x2d spelling)
+#
+# Installed by `scripts/media-view.sh install` — the repo copy is the source of
+# truth (fleet law: never edit host state by hand).
+#
+# An OverlayFS merge that shows the media library as one tree while splitting
+# where its bytes live: lowerdir is the RAID6 array (the source of truth, never
+# written), upperdir is a directory on the NVMe that receives every write. The
+# two media servers bind the result at /var/lib/media-meta/view, so Jellyfin's
+# sidecars and artwork are SSD bytes — which is what makes them fast to browse
+# and what keeps Plex's NFO agent off the spinners — while the media itself stays
+# on the array untouched.
+#
+# RequiresMountsFor is load-bearing: without the array mounted, lowerdir is an
+# empty directory and the view would present an empty library as truth.
+
+[Unit]
+Description=SSD-backed overlay view of the media library (array + NVMe metadata layer)
+RequiresMountsFor=/mnt/entropy
+
+[Mount]
+What=overlay
+Where=/var/lib/media-meta/view
+Type=overlay
+Options=lowerdir=/mnt/entropy,upperdir=/var/lib/media-meta/upper,workdir=/var/lib/media-meta/work
+
+[Install]
+WantedBy=local-fs.target

dankserver/jellyfin/quadlet/jellyfin.container

@@ -4,8 +4,9 @@
 # Unit name: jellyfin.service (podman 5.x drops the container- prefix)
 #
 # Data roots migrated from /opt/jellyfin to /home/dank/Containers/jellyfin at
-# cutover. All of /mnt/entropy is mounted read-only at /data: nothing automated
-# anywhere writes to the entropy volume. Every
+# cutover. The library is seen through /var/lib/media-meta/view, the SSD-backed
+# overlay view (scripts/media-view.sh): the array stays read-only and untouched,
+# while the sidecars this server writes land on the NVMe. Every
 # creation-time environment variable of the compose-era container is kept as
 # an Environment= line (zero drift); none of them are secrets, so this unit
 # needs no EnvironmentFile.
@@ -16,7 +17,7 @@ After=network.target
 
 [Container]
 ContainerName=jellyfin
-Image=ghcr.io/jellyfin/jellyfin:latest
+Image=ghcr.io/jellyfin/jellyfin:latest@sha256:fea863d0cd9ec445985d9c75fc6ed64e38a1d2766cf0c250a33d553f4fc1b771
 Network=dankserver_appnet
 Environment=DEBIAN_FRONTEND=noninteractive
 Environment=DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true
@@ -41,7 +42,12 @@ PublishPort=7359:7359/udp
 PublishPort=1900:1900/udp
 Volume=/home/dank/Containers/jellyfin/config:/config
 Volume=/home/dank/Containers/jellyfin/cache:/cache
-Volume=/mnt/entropy:/data:ro
+# The view, writable: Jellyfin is the metadata writer (movie.nfo / tvshow.nfo /
+# episode.nfo + artwork) and those bytes belong on the SSD. The condition is the
+# fail-closed guard — starting against a plain, unmounted view directory would let
+# a scan conclude the library is empty.
+Volume=/var/lib/media-meta/view:/data
+UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view
 AddDevice=/dev/dri
 GroupAdd=keep-groups
 HealthCmd=curl -f http://localhost:8096/health

dankserver/observability/quadlet/cadvisor.container

@@ -0,0 +1,39 @@
+# Quadlet unit: cadvisor (rootless, boot-persistent) — DANKSERVER.
+#
+# Install path on dankserver: ~/.config/containers/systemd/cadvisor.container
+# Unit name: cadvisor.service (container name: cadvisor)
+#
+# The Dankserver twin of dankclaw/observability/quadlet/cadvisor.container.
+# Same rootless shape, same --cgroupns=host requirement (see that file for the
+# full story), only the LAN address differs: this host publishes on
+# 192.168.0.235:8080 so Prometheus on Dankclaw can scrape it over the real
+# network (cross-host, so the host.containers.internal trick does not apply).
+# It reports the media/library stack (romm, plex, jellyfin, copyparty, calibre,
+# tautulli) by their quadlet unit names via the cgroup path.
+
+[Unit]
+Description=cAdvisor container metrics (Dankserver)
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+ContainerName=cadvisor
+Image=gcr.io/cadvisor/cadvisor:v0.49.1@sha256:3cde6faf0791ebf7b41d6f8ae7145466fed712ea6f252c935294d2608b1af388
+PodmanArgs=--pid=host --cgroupns=host
+PublishPort=127.0.0.1:8080:8080
+PublishPort=192.168.0.235:8080:8080
+Volume=/sys:/sys:ro
+Volume=/:/rootfs:ro
+Exec=--housekeeping_interval=30s
+HealthCmd=wget --spider -q http://localhost:8080/healthz
+HealthInterval=30s
+HealthTimeout=10s
+HealthRetries=3
+HealthStartPeriod=40s
+
+[Service]
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=default.target

dankserver/observability/quadlet/node-exporter.container

@@ -37,7 +37,7 @@ Wants=network-online.target
 
 [Container]
 ContainerName=node-exporter
-Image=docker.io/prom/node-exporter:latest
+Image=docker.io/prom/node-exporter:latest@sha256:1b4e4438faca4dd7e001dd445d161a4a2091b0fededa84093b3a8dfeae1f1be0
 Network=host
 PodmanArgs=--pid=host
 Exec=--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/rootfs --web.listen-address=127.0.0.1:9100 --web.listen-address=192.168.0.235:9100

dankserver/plex/quadlet/plex.container

@@ -4,14 +4,19 @@ After=network-online.target
 Wants=network-online.target
 
 [Container]
-Image=lscr.io/linuxserver/plex:1.43.4.10903-e5521bd8c-ls325
+Image=lscr.io/linuxserver/plex:1.43.5.11079-b3b13e506-ls330@sha256:01a53d9c7adcbb3701fbb0a4230c0a37c85453759b6b05a67356c3c7a09de08d
 ContainerName=plex
 Network=dankserver_appnet:alias=plex
 Environment=PUID=0
 Environment=PGID=0
 Environment=TZ=America/Los_Angeles
 Volume=/var/lib/plexmediaserver:/config
-Volume=/mnt/entropy:/mnt/entropy:ro
+# The container path stays /mnt/entropy on purpose: every path Plex recorded in
+# its own database is written against it, so swapping the HOST source of this bind
+# to the overlay view re-points Plex at the sidecars without touching its DB.
+# Read-only: Plex consumes the NFO files Jellyfin writes and writes no metadata.
+Volume=/var/lib/media-meta/view:/mnt/entropy:ro
+UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view
 PublishPort=32400:32400
 
 [Service]

dankserver/romm/quadlet/romm-mariadb.container

@@ -14,7 +14,7 @@ After=network.target
 
 [Container]
 ContainerName=romm-mariadb
-Image=docker.io/library/mariadb:11
+Image=docker.io/library/mariadb:11@sha256:c4124d1afa875e47f2892edf26b482fa54f796e2f342d3717a04ca091a22a9bd
 Network=dankserver_appnet:alias=mariadb
 EnvironmentFile=/home/dank/Containers/romm/env
 Environment=MARIADB_DATABASE=romm

dankserver/romm/quadlet/romm-redis.container

@@ -12,7 +12,7 @@ After=network.target
 
 [Container]
 ContainerName=romm-redis
-Image=docker.io/library/redis:7-alpine
+Image=docker.io/library/redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f
 Network=dankserver_appnet:alias=redis
 Volume=/home/dank/Containers/romm/redis:/data
 HealthCmd=redis-cli ping

dankserver/romm/quadlet/romm.container

@@ -18,7 +18,7 @@ After=romm-mariadb.service romm-redis.service network.target
 
 [Container]
 ContainerName=romm
-Image=ghcr.io/rommapp/romm:latest
+Image=ghcr.io/rommapp/romm:latest@sha256:8677fb7b81b1d3db20b849e989a1dadc2a11159e51eb34a2acdcb01102274edf
 Network=dankserver_appnet
 EnvironmentFile=/home/dank/Containers/romm/env
 Environment=DB_HOST=mariadb

dankserver/tautulli/quadlet/tautulli.container

@@ -4,7 +4,7 @@ After=network-online.target
 Wants=network-online.target
 
 [Container]
-Image=lscr.io/linuxserver/tautulli:v2.18.1-ls245
+Image=lscr.io/linuxserver/tautulli:v2.18.2-ls250@sha256:529c7eabcc40554c1dc48f33f1f3cc3b1438462d73e0ec2dac8ac0b66aeeb8d4
 ContainerName=tautulli
 Network=dankserver_appnet:alias=tautulli
 Environment=PUID=0

frankbook-pro/gluetun/quadlet/gluetun.container

@@ -19,7 +19,7 @@ Description=Gluetun VPN tunnel (FrankBook-Pro podman machine)
 
 [Container]
 ContainerName=gluetun
-Image=docker.io/qmcgaw/gluetun:latest
+Image=docker.io/qmcgaw/gluetun:latest@sha256:c2ee4191762fca6b6a3c3359a731198a95d9c9af4db32d466e223bf3a84e779b
 AddCapability=NET_ADMIN
 AddDevice=/dev/net/tun:/dev/net/tun
 Environment=VPN_SERVICE_PROVIDER=${VPN_SERVICE_PROVIDER}

frankbook-pro/lidarr/quadlet/lidarr.container

@@ -15,7 +15,7 @@ After=gluetun.service
 
 [Container]
 ContainerName=lidarr
-Image=lscr.io/linuxserver/lidarr:latest
+Image=lscr.io/linuxserver/lidarr:latest@sha256:d8549ff9ea29b4015c32257360652c1000c31ea471a2b3f0835ae77292faf35b
 Environment=PUID=0
 Environment=PGID=0
 Environment=TZ=America/Los_Angeles

frankbook-pro/prowlarr/quadlet/prowlarr.container

@@ -24,7 +24,7 @@ After=gluetun.service
 
 [Container]
 ContainerName=prowlarr
-Image=lscr.io/linuxserver/prowlarr:latest
+Image=lscr.io/linuxserver/prowlarr:latest@sha256:b3b00047ed20f6ce2b16780145cb6a0acb51c9e28ea6aa6f5dcd33c7034f1797
 Environment=PUID=0
 Environment=PGID=0
 Environment=TZ=America/Los_Angeles

frankbook-pro/qbittorrent/quadlet/qbittorrent.container

@@ -29,7 +29,7 @@ Network=gluetun.container
 # Pinned to the last 4.x line: Readarr's retired qB client requires the
 # qB 4.x login response (HTTP 200, body "Ok."); qB 5.x answers 204 with an
 # empty body, which fails Readarr's auth check. All other clients accept 4.x.
-Image=lscr.io/linuxserver/qbittorrent:4.6.7
+Image=lscr.io/linuxserver/qbittorrent:4.6.7@sha256:55f15d44396315551f87294a176efae733b16e283f38980308e46073950257c6
 Environment=PUID=0
 Environment=PGID=0
 Environment=TZ=America/Los_Angeles

frankbook-pro/radarr/quadlet/radarr.container

@@ -15,7 +15,7 @@ After=gluetun.service
 
 [Container]
 ContainerName=radarr
-Image=lscr.io/linuxserver/radarr:latest
+Image=lscr.io/linuxserver/radarr:latest@sha256:fc0ae3c50f8128b97e84381903ec8d6845c3a569641cb7117c124376cccb57c1
 Environment=PUID=0
 Environment=PGID=0
 Environment=TZ=America/Los_Angeles

frankbook-pro/readarr/quadlet/readarr.container

@@ -19,7 +19,7 @@ After=gluetun.service
 
 [Container]
 ContainerName=readarr
-Image=lscr.io/linuxserver/readarr:amd64-1.0.980
+Image=lscr.io/linuxserver/readarr:amd64-1.0.980@sha256:a95e82dbb0eb2bb43de4c150f1dab9dd821306c67ea0b418983a957e731f2b61
 Environment=PUID=0
 Environment=PGID=0
 Environment=TZ=America/Los_Angeles

frankbook-pro/sonarr/quadlet/sonarr.container

@@ -16,7 +16,7 @@ After=gluetun.service
 
 [Container]
 ContainerName=sonarr
-Image=lscr.io/linuxserver/sonarr:latest
+Image=lscr.io/linuxserver/sonarr:latest@sha256:889dcc58c6aea859033985894b519b2e6b1e55d92527ed123f9f52e42ae7c97b
 Environment=PUID=0
 Environment=PGID=0
 Environment=TZ=America/Los_Angeles

report/npu-tts-runtime-landscape-2026-10-19.md

@@ -0,0 +1,319 @@
+# XDNA2 NPU inference runtimes on Linux (Strix Halo) — can any of them serve TTS?
+
+Research slice, 2026-10-19. Worker: `NpuRuntimeLandscape`. Research only, no code.
+Question: every way to execute model inference on an AMD XDNA2 NPU on Linux as of
+2026-10, and whether any of them can serve a speech-synthesis model.
+
+> **Status, same night — the spike that followed this slice settled it.** NO-GO on the
+> NPU (the loader declines the graph, not the silicon); the actionable door is the gfx1151
+> iGPU behind `-DGGML_VULKAN=ON`. Measured contract, exoneration chain and host gates:
+> `tts-npu-spike-verdict-2026-10-19.md`. That page supersedes §8–§9 below.
+
+Convention: vendor claims carry URL + the version/date they hold for.
+**[host]** = Dankclaw state verified on the live machine 2026-10-19 (Main), not a
+vendor claim. `[INFERENCE]` = my deduction from the cited artifacts.
+
+## 0. Answers to the three questions Main ranked
+
+**(1) A runtime that already runs TTS on XDNA2 today: it exists at AMD, not on our
+host.** Ryzen AI Software **1.8.0** (docs `latest` == 1.8.0; released 2026-07-23)
+lists `neutts-air` and `neutts-nano` as model type **TTS**, variant
+*full-optimized* (NPU-only) — <https://ryzenai.docs.amd.com/en/latest/llm_list.html>;
+release note "New TTS model support: neutts-air, neutts-nano" —
+<https://ryzenai.docs.amd.com/en/latest/relnotes.html>. Pre-compiled artifacts are
+published: `amd/neutts-air_rai_1.8.0_npu_4K` (arch `qwen2`, lastModified
+2026-07-23, Apache-2.0, "AWQ / Group 128 / Asymmetric / BFP16 activations / UINT4
+Weights") and `amd/neutts-nano_rai_1.8.0_npu_4K` (arch `llama`).
+**Neither artifact is a TTS engine.** Both file lists are LM-only (`model.onnx`,
+`genai_config.json`, BPE tokenizer, AIE-ML compiler metastate: `dd_metastate_*`
+`.ctrlpkt/.super/.state`, `cache/txn_bins.zip`) with **no codec, no vocoder, no
+reference-audio encoder**. [INFERENCE from HF file lists]
+
+**(2) FastFlowLM: no TTS, and we cannot add one** — see §2. Halogen: no TTS entry
+**[host]**. So "something installed already serves TTS on the NPU" is FALSE.
+
+**(3) Our own ONNX graph compiled for the NPU on Linux:** possible for the plain
+ONNX flows (they compile on Linux), **impossible for a new LLM/OGA model** — model
+generation is Windows-only in 1.8.0 — and gated by an operator table where BF16
+lacks Softmax/LayerNorm/ReLU/Gelu (see §4).
+
+## 1. Landscape: every way to reach the XDNA2 array on Linux
+
+| # | Path | License | Strix Halo + Linux, 2026-10 | Speech synthesis |
+|---|------|---------|------------------------------|------------------|
+| 1 | Ryzen AI SW → ONNX Runtime + Vitis AI / Ryzen AI EP; compile INT8 (`X2`) or BF16 (`VAIML`) ONNX at session creation | AMD proprietary (tarball) | supported: our device ID/rev is in AMD's STX table; compile+run works on Linux (§3, §4) | user-authored ONNX only; no TTS recipe/demo |
+| 2 | Same EP driven by **OGA** (`onnxruntime-genai`), "full fusion" NPU-only LLM flow; runner = AMD `model_benchmark` / OGA C++ API | AMD + MS OGA | Linux: **vendor-precompiled models only**, no hybrid flow | **neutts-air / neutts-nano = TTS** (LM half) |
+| 3 | **FastFlowLM** `flm` | MIT + free proprietary kernels | Strix Halo "Supported"; Ubuntu 24.04/25.10/**26.04**, Arch | none (ASR input only) |
+| 4 | **IRON / mlir-aie** + Peano → hand-written AIE-ML design → XRT/`amdxdna` | Apache-2.0 + LLVM exception | explicitly documented "AMD Ryzen™ AI on Linux"; same driver/XRT stack | anything, hand-written; nothing ships TTS |
+| 5 | Vendor-curated AIE-ML graph sets (halogen `HALOGEN_NPU_MODELS`, FLM packs) | vendor | present **[host]** | none **[host]** |
+| 6 | Windows-only: DirectML, Ryzen AI Windows EP (`ryzenai-llm`), Studio Effects, `whispercpp` NPU | — | not usable | (per Lemonade matrix: Windows only) |
+| 7 | ggml/llama.cpp/**qwentts.cpp**, whisper.cpp, sherpa-onnx, Piper | — | no XDNA backend in ggml/ORT-mainline | CPU / iGPU only |
+| 8 | IREE, TVM, ONNX Runtime mainline EPs (OpenVINO = Intel NPU, MIGraphX/ROCm = AMD GPU) | — | no XDNA2 target; AMD's NPU EP is only "Xilinx Vitis-AI (preview)" in the ORT EP list and ships inside AMD's own build | n/a |
+
+Sources: §2–§6 below.
+
+## 2. FastFlowLM (`/usr/bin/flm` 1.0.1 **[host]**) — catalog + HTTP surface
+
+* Repo moved into AMD's ROCm org **2026-08-11** (`github.com/ROCm/FastFlowLM`,
+  v1.0.0). "Supports all Ryzen™ AI Series chips with XDNA2 NPUs (Strix, Strix Halo,
+  Kraken, and Gorgon Point)."
+  <https://github.com/ROCm/FastFlowLM> (README fetched 2026-10-19).
+* Linux support landed 2026-03; distros: Ubuntu 24.04, 25.10, **26.04**, Arch,
+  generic. Needs NPU firmware ≥ 1.1.0.0, kernel **7.0+** with `amdxdna` (or
+  `amdxdna-dkms`), XRT stack (`ppa:lemonade-team/stable` → `libxrt-npu2`,
+  `amdxdna-dkms`); `flm validate` reads the DRM path (`/dev/accel/accel0`), `flm
+  run` goes through **XRT** and fails with `No such device with index '0'` unless
+  `xrt-smi examine` lists the NPU; `memlock` must be raised.
+  <https://fastflowlm.com/docs/install_lin/> (page dated 2026-03-05, fetched
+  2026-10-19). Static build bundles **XRT v2.21.75** — the same XRT series as the
+  distro package on the host **[host]**.
+* HTTP surface: `flm serve <model>` → OpenAI-compatible REST, **port 52625**, no
+  auth. Maintained endpoints: `v1/models`, `v1/chat/completions`,
+  `v1/audio/transcriptions`, `v1/embeddings`. **There is no
+  `/v1/audio/speech`.**
+  <https://fastflowlm.com/docs/instructions/server/openapi/>
+* Catalog 2026-10-19 (<https://fastflowlm.com/docs/models/>): LLaMA, DeepSeek,
+  Qwen, Gemma, MedGemma, TranslateGemma, gpt-oss, LiquidAI/LFM, Phi, Nanbeige,
+  **Whisper — card states "Type: Speech-to-Text (ASR)"**
+  (<https://fastflowlm.com/docs/models/whisper/>), EmbeddingGemma, SmolVLA,
+  Hy-MT2, MiniCPM. No speech-synthesis entry.
+* Engines are **prebuilt, closed, downloaded from Hugging Face per architecture**
+  ("Internet access to HuggingFace is required to download the optimized model
+  kernels"). The only in-tree engine build is Gemma-4E
+  (`cmake --preset linux-default -DFLM_BUILD_GEMMA4E=ON`).
+  → **A new TTS architecture cannot be added to `flm` by us.**
+
+## 3. AMD Ryzen AI Software 1.8.0 on Linux — install source, device support
+
+* **Install channel (important correction to the pypi.amd.com evidence).** The pip
+  index is Windows-only for the EP runtime (`ryzenai_dynamic_dispatch-1.7.0-…
+  win_amd64.whl`, `onnxruntime_vitisai-1.23.2-…win_amd64.whl`; `flexmlrt` has a
+  single **linux_aarch64** wheel) — but AMD's Linux stack is not on pip. It ships
+  as a tarball that is, as of 2026-10-19, **publicly reachable without an AMD
+  account**: `https://download.amd.com/opendownload/RyzenAI/1.8.0/ryzen_ai-1.8.0.tgz`
+  → HTTP 206 on a range probe, `Content-Range: bytes 0-8/**7819506855**`
+  (**≈7.8 GiB**, magic `1f 8b 08` = gzip). (The docs link it behind the
+  EULA form `account.amd.com/...ryzenai-eula-public-xef.html`; the CDN path
+  serves it directly. Verified by me with curl 2026-10-19. I did not finish
+  downloading 7.8 GiB, so the tarball's *contents* list is unverified —
+  `[INFERENCE]` only where noted.)
+* Driver/XRT package (public, 90,016,137 bytes, `application/zip`):
+  `https://download.amd.com/opendownload/RyzenAI/Driver/RAI_1.8_Linux_NPU_XRT.zip`
+  contains `xrt_202620.2.25.37_24.04-amd64-{base,base-dev,npu}.deb` +
+  `xrt_plugin.2.25.260102.56.release_24.04-amd64-amdxdna.deb` (verified by
+  download + unpack). **The plugin deb ships firmware for
+  `/usr/lib/firmware/amdnpu/17f0_11/npu.dev.sbin`** — exactly the directory our
+  NPU loads **[host]** — plus `1502_00`, `17f0_10`, `17f1_10`, `17f1_13`,
+  `libvxdna.so`, `libxrt_driver_xdna.so.2.25.260102.56`, and the DKMS driver
+  source tarball. **No xclbin/PDI and no compiler binary in this package**: it is
+  driver + XRT shim + firmware only.
+* **Our device is officially STX.** AMD's APU table
+  (<https://ryzenai.docs.amd.com/en/latest/app_development.md#apu-types>):
+  `0x1022:0x17F0 rev 0x11 → STX` (0x00/0x10 also STX, 0x20 = KRK, `0x1502` rev 0
+  = PHX/HPT). Release notes: "Strix and **Strix Halo** as STX", and the STX/KRK
+  column enables CNN INT8, CNN BF16, NLP BF16, LLM (OGA).
+* Vendor driver source (same deb, `share/amdxdna/amdxdna.tar.gz`, source dated
+  2026-06-29) maps `0x17f0 rev 0x11 → dev_npu5_info`, `.fw_path =
+  "amdnpu/17f0_11/"`, `default_vbnv = "RyzenAI-npu5"`, and the revision→name table
+  has `{ AIE2_DEV_REVISION_HALO, "NPU Strix Halo" }`; `xrt-smi examine` therefore
+  reports the device as **NPU Strix Halo**.
+* Linux install recipe (1.8.0 docs
+  <https://ryzenai.docs.amd.com/en/latest/linux.html>): Ubuntu 24.04 packages,
+  **Python 3.12** (`apt install python3.12 python3.12-venv`),
+  `libboost-filesystem1.74.0`, dkms; then `install_ryzen_ai.sh -a yes -p <venv>`;
+  `LD_LIBRARY_PATH=…:${RYZEN_AI_INSTALLATION_PATH}/onnxruntime/lib/`;
+  `quicktest.py` "picks up a simple CNN model, **compiles it** and runs on AMD's
+  Neural Processing Unit" — i.e. **compilation runs on Linux** for the non-LLM
+  flows. "Linux currently supports NPU only flow" (no iGPU EP).
+  Host friction: our box is Ubuntu 26.04 + **Python 3.14** **[host]**, so the venv
+  must be built with a 3.12 interpreter and the packages are 24.04-built
+  `[INFERENCE]`.
+* LLM flow on Linux
+  (<https://ryzenai.docs.amd.com/en/latest/llm_linux.html>): prequantized
+  ready-to-run models from HF, `deployment/lib`,
+  `RYZENAI_EP_PATH=$PWD/deployment/lib/libonnxruntime_providers_ryzenai.so`,
+  run via the `model_benchmark` binary; **"Linux does not support Hybrid flow"**;
+  **"Model generation is not supported in current release"** (repeated in
+  relnotes 1.8 Known Issues: "Models generated on Windows are compatible with
+  Linux"); needs `memlock unlimited`.
+  Sample throughput in that doc: **49.1 tokens/s decode**, 753 tokens/s prefill
+  (Phi-3.5-mini, 4K model).
+* Third-party confirmation of the OS split: Lemonade's backend matrix
+  (<https://raw.githubusercontent.com/lemonade-sdk/lemonade/main/README.md>,
+  fetched 2026-10-19) — `flm`/npu/XDNA2 = **Windows, Linux**;
+  `ryzenai-llm`/npu = **Windows only**; `whispercpp`/npu (STT) = **Windows only**;
+  text-to-speech engines `kokoro` and `acestep` = metal / cuda / vulkan / rocm,
+  **no npu device at all**. Lemonade is the "Server Interface" layer of AMD's own
+  Ryzen AI stack (docs index), so this matrix is the vendor-sanctioned view.
+
+## 4. Can we compile our own graph on Linux? (the no-recipe path)
+
+Mechanics from <https://ryzenai.docs.amd.com/en/latest/modelrun.md> (1.8.0):
+ONNX only, opset 17 recommended; the EP compiles on first `InferenceSession` and
+caches (VitisAI EP cache, or ORT **EP context cache** — `ep.context_enable`, and
+"precompile the EP context model using Python and deploy it using a C++ program"
+is explicitly supported); EP auto-partitions unsupported nodes to CPU;
+BF16 = `target VAIML` (`preferred_data_storage: unvectorized` for transformers),
+BF16 **STX/KRK-or-newer only**; INT8 = `target=X2` with XINT8/A8W8/A16W8 and
+"general asymmetric quantization … to enable third-party quantized models";
+**deployment EP cannot compile BF16 on the fly** — ship a pre-compiled model
+(<https://ryzenai.docs.amd.com/en/latest/app_development.md>);
+cache dirs are not portable across EP or NPU-driver versions.
+
+Operator coverage is the hard gate
+(<https://ryzenai.docs.amd.com/en/latest/ops_support.md>, "operators supported in
+Ryzen AI 1.8.0"; "Y" = broad coverage, some configurations may still fail):
+
+* Good for TTS front-ends/vocoders: `ConvTranspose` **Y in BF16/A16W8/A8W8**
+  (absent only from XINT8), `Conv`, `Resize`, `Upsample`, `Pad`, `LSTM` (BF16),
+  `CumSum` (BF16/XINT8), `STFT` (XINT8 only).
+* **Blockers for a decoder-style transformer in the plain flows:** in **BF16** the
+  table has **no Softmax, no LayerNormalization, no Relu/Gelu**; LayerNorm is
+  XINT8-only and Softmax is A16W8/A8W8/XINT8-only. So attention either lives in
+  the **LLM/OGA flow** (whose op set is exactly
+  `SimplifiedLayerNormalization`, `SkipSimplifiedLayerNormalization`, `MatMulNBits`
+  (W4ABF16/W4ABFP16), `RotaryEmbedding`, **`GroupQueryAttention`**, `Sigmoid`,
+  `Mul`, `Add`) or must be decomposed into a quantized INT8 flow.
+* Missing entirely: `GRU`, `Trilu`, `OneHot`, `EyeLike`, `Multinomial`,
+  `NonMaxSuppression`, `CastLike`, `MelWeightMatrix`/`MelFilterGroup`,
+  `RMSNormalization` → graphs that emit them must be rewritten before they
+  partition onto the NPU. [INFERENCE from absence in the table.]
+* Quantization is a separate tool, not in the driver package: **AMD Quark**
+  (`quark.torch` / `quarkonnx`), which documents `ConvTranspose2d`, `conv1d`,
+  softmax/LayerNorm/BFP quantization (relnotes 1.3/1.2/1.1/0.9) — Quark is *not*
+  on pypi.org under the obvious names **[host probe]** and ships from AMD's
+  gated/pip-private channel.
+* **New LLM-shaped model (our Qwen-Talker analogue): Windows-only today.**
+  OGA model generation is not available on Linux in 1.8.0 (§3). So a *new*
+  speech-LM cannot be self-served onto the NPU on Linux without AMD (or a
+  Windows box) in the loop.
+* Practical shape cost: the neutts artifacts carry **one compiled variant per
+  padded sequence length** (128/256/512/768/1024/1536/2048/2304/3072/4096 for
+  prefill; 256…4096 for token-phase attention mask) — i.e. per-shape compile
+  artifacts are the norm for the LLM flow. [INFERENCE from the HF file list.]
+
+## 5. Does the AMD TTS artifact actually produce speech? (cloned-voice survival)
+
+NeuTTS is **LM + separate neural codec** ("Simple LM + codec architecture"),
+clone from ~3 s of reference audio, and the codec is **NeuCodec (50 Hz, single
+codebook)**: `neuphonic/neucodec` (PyTorch, encode+decode),
+`neucodec-onnx-decoder{,-int8}` (**decode only**), "Decoder-only (ONNX) codecs
+require **pre-encoded references** — these ship with the repo for all bundled
+voices." Phoneme input; 2048-token context ≈ 30 s of audio; NeuTTS-Air ~360 M
+active / 552 M with embeddings; upstream benchmarks are **CPU-only llama.cpp**
+(119 tok/s prefill/decode mix on a Ryzen 9 HX 370) and explicitly exclude the
+codec. <https://raw.githubusercontent.com/neuphonic/neutts/main/README.md>
+(fetched 2026-10-19); `neuphonic/neutts-air` HF metadata (lastModified
+2026-08-26, `qwen2`, GGUF + safetensors).
+
+Consequences for the fleet spike:
+
+1. AMD's NPU artifact = LM only; **waveform decode and reference encoding are CPU
+   work either way**. Our `reference.wav` voices survive only if we run the
+   NeuCodec **encoder** (PyTorch, per the codec table there is no ONNX encoder)
+   once per voice and then feed pre-encoded codes. That is a one-time offline
+   step per voice; after that the compiled graph takes codes/prompt prefix as
+   ordinary inputs, so nothing about the compiled graph forbids a caller-supplied
+   speaker prefix. `[INFERENCE]`
+2. Real-time math: 50 Hz codec means **≥50 tokens/s decode** for 1× RT. AMD's own
+   Linux sample output for a 4K Phi-3.5-mini is **49.1 tok/s** — so budget for
+   roughly RT-1× on one 4K "full fusion" graph, before the codec's CPU cost.
+   `[INFERENCE — different model; must be measured.]`
+3. Host RAM: the same doc reports **peak working set 3.54 GB** for that 4K model.
+   NeuTTS-Air is ~552 M params at W4A-BFP16, so expect a smaller footprint than
+   3.5 GB `[INFERENCE]`; still non-trivial against 14 GiB available **[host]**.
+4. AMD ships **no TTS demo, runner, or server** on any OS: the examples page has
+   only CNN/BF16/DistilBERT/super-resolution/OGA-LLM/VLM/RAG
+   (<https://ryzenai.docs.amd.com/en/latest/examples.md>); the RyzenAI-SW repo
+   tree (669 paths, `main`, fetched 2026-10-19) contains speech demos **only under
+   `Demos/ASR/{Whisper,Parakeet-TDT}` and `Transformer-examples/ASR/Whisper-AI`**
+   — no TTS directory. Serving TTS therefore means writing our own OGA host
+   (or driving `model_benchmark`-style C++ code) plus phonemizer + NeuCodec glue.
+
+## 6. Two graphs on one NPU, and the iGPU hazard
+
+* **Concurrency is a first-class driver feature.** The vendor `amdxdna` source
+  (in `xrt_plugin.2.25.260102.56`, 2026-06-29) sets `.hwctx_limit = 16` for
+  npu4/npu5 — i.e. **16 hardware contexts** on our device — and allocates each
+  context a set of AIE columns with QoS `opc = col_opc (4096) × num_col`
+  (`aie2_ctx.c`, `aie2_solver.h`), with `frame_boundary_preempt` /
+  `fill_preempt_slot` in `aie2_pci.h` (firmware preemption at frame boundaries).
+  So halogen's armed models and a TTS graph can hold separate contexts
+  simultaneously; they contend for columns and OPC share, they are not mutually
+  exclusive.
+* AMD's user-visible history agrees: 1.4 "Support for up to **16 hardware
+  contexts** on Ryzen AI 300 series"; 1.5 "Application concurrency: improves the
+  resource distribution across applications"; 1.3 "Idle detection and suspension
+  of contexts", "Rebalance feature for AIE hardware resource optimization";
+  1.0 known issue "Support for multiple applications is limited to up to eight"
+  (all in <https://ryzenai.docs.amd.com/en/latest/relnotes.html>).
+* **Package conflict, the real risk** [INFERENCE]: the vendor stack wants AMD's
+  XRT 2.25.37 + `xrt_plugin…amdxdna` deb **[host: distro `libxrt-npu2
+  1:2.21.75+dfsg-4` installed]**, and firmware must match the loaded `amdxdna.ko`
+  — the vendor README warns that a stale/mismatched `npu.sbin` causes
+  `ERT_CMD_ABORT`/mailbox timeouts, and that a distro **in-tree** `amdxdna` older
+  than the XRT SHIM makes `GET_ARRAY`/`QUERY_TELEMETRY` return
+  `EOPNOTSUPP`/`EINVAL` (<https://github.com/amd/xdna-driver> README,
+  fetched 2026-10-19). Installing AMD's debs over the distro XRT changes the ABI
+  under FastFlowLM/halogen, so the spike must decide the XRT version **once**, for
+  the whole host.
+* **iGPU / fabric clock:** no vendor document supports a fabric-clock pin, and no
+  vendor document addresses NPU+iGPU concurrency hangs. AMD's Strix Halo
+  optimization doc (ROCm docs, dated 2026-02-20, and its successor "AMD RDNA3.5
+  system optimization", ROCm 10.1.0) covers GPUVM/GART/GTT, the TTM `pages_limit`
+  knob, `amd-ttm`, BIOS VRAM sizing, and minimum kernel versions
+  (<https://rocm.docs.amd.com/en/docs-7.2.0/how-to/system-optimization/strixhalo.html>,
+  <https://rocm.docs.amd.com/en/latest/reference/system-optimization/rdna3-5.html>).
+  The `pp_dpm_fclk` level-7 pin and the GPU+NPU hang observation stay
+  fleet-observed **[host]**, i.e. `[INFERENCE]` territory — no citation exists.
+
+## 7. Paths I checked and ruled out (so nobody re-walks them)
+
+* **Windows-only doors**: DirectML; Ryzen AI Windows EP; Lemonade's
+  `ryzenai-llm` and `whispercpp`-NPU engines (matrix above); AMD's OGA model
+  generation. Not usable on Dankclaw.
+* **Mainline ONNX Runtime** has no AMD-NPU EP: its EP summary lists AMD ROCm and
+  AMD MIGraphX (GPU) and "Xilinx Vitis-AI (*preview*)" only
+  (<https://onnxruntime.ai/docs/execution-providers/>, 2026-10-19); the working
+  EP is AMD's own build (`libonnxruntime_providers_ryzenai.so` in the tarball's
+  deployment dir, per llm_linux).
+* **ggml / llama.cpp / qwentts.cpp**: no XDNA backend → the current engine
+  (`tts-server` on GGUF, CPU-only build **[host]**) cannot be "turned on" for the
+  NPU; migration means a different runtime, not a flag.
+* **`flm`**: catalog/engines are closed per-architecture (§2) — the cheapest door
+  by install cost, and it is not a door for TTS.
+* **IRON / mlir-aie** is the only open toolchain that documents running on the
+  Ryzen AI NPU from Linux ("A close-to-metal Python API for programming AMD Ryzen™
+  AI NPUs, built on an open-source MLIR-based compiler toolchain"; "Getting
+  Started for AMD Ryzen™ AI on Linux", Ubuntu 24.04/24.10, XRT from
+  `ppa:amd-team/xrt`: `libxrt2 libxrt-npu2 … amdxdna-dkms`, `render` group
+  required) — <https://raw.githubusercontent.com/Xilinx/mlir-aie/main/README.md>,
+  fetched 2026-10-19. This is how FLM's kernels were made ("Low-level kernels
+  optimized using IRON + AIE-MLIR", FLM README) — i.e. real, but it is
+  hand-written per-op tile programming, not a model compiler: writing a TTS model
+  here is a multi-month engineering project, not a migration.
+* **Embedded** Ryzen AI P100 support in the same Linux doc is irrelevant to STX.
+
+## 8. Recommendation implied by the above (for Main's spike)
+
+Spike the AMD path as decided, with these constraints planned in: Python 3.12 venv
++ 24.04-built wheels on Ubuntu 26.04; one host-wide XRT decision (AMD 2.25.37 vs
+distro 2.21.75) with firmware/driver pairing checked via `xrt-smi examine` →
+"NPU Strix Halo"; 7.8 GiB tarball staged off the public CDN URL (§3); pre-encode
+each `reference.wav` with the **PyTorch** NeuCodec encoder because the ONNX codec
+is decoder-only; keep phonemizer + codec on CPU; measure decode tokens/s against
+the 50 Hz budget and peak RSS against the ~14 GiB headroom; expect to write the
+serving shell yourself (AMD ships `model_benchmark`, no TTS demo).
+
+## 9. Loose ends (not verified; I stopped on Main's stand-down)
+
+* Contents of `ryzen_ai-1.8.0.tgz` (7.8 GiB) — whether the Linux tarball contains
+  the AIE-ML compiler (`vaiml`, `aiecompiler_client`) for the *BF16/INT8* flows
+  and which Python wheels it carries. My statement that non-LLM compilation works
+  on Linux rests on AMD's own Linux doc ("quicktest.py … compiles it and runs on
+  … NPU"), not on the package contents. Download was killed mid-flight.
+* Whether AMD's `neutts` artifacts run correctly under the Linux `model_benchmark`
+  runner (the TTS rows are listed without OS qualifiers; no Linux TTS run is
+  documented anywhere in 1.8.0 docs).
+* Exact AIE column count/total `ncols` exposed for Strix Halo (driver reads it
+  from AIE metadata: `ndev->total_col = min(XRS_MAX_COL, ndev->aie.metadata.cols)`).

report/qwen3-tts-payload-portability-2026-10-19.md

@@ -0,0 +1,438 @@
+# Qwen3-TTS payload + qwentts.cpp runtime: architecture and portability
+
+Research deliverable (no code). Slice: *what the fleet's TTS payload actually is, and how
+portable it is off the CPU ggml path.* Companion slice: `NpuRuntimeLandscape` (what NPU
+runtimes exist on Dankclaw).
+
+Evidence date: 2026-10-19. Everything below is grounded in:
+
+- `github.com/ServeurpersoCom/qwentts.cpp` @ `5916bf7c79dbb4e66fc059cdc507a79a23a28bb4`
+  (2026-10-16, upstream HEAD on 2026-10-19) and its ggml submodule
+  `ServeurpersoCom/ggml` @ `cc5df49a6b878db2b68b9a7b45976e279594627d` (2026-10-16).
+- The two GGUF files as they sit on Dankclaw (`/home/dank/models/Qwen3-TTS-GGUF/`), read
+  byte-for-byte from the live host.
+- Upstream Qwen3-TTS model cards / release material.
+
+Status: COMPLETE for the assigned slice. ANSWER C was confirmed on the live host the same
+night — the running `frank-tts-engine` container carries `libggml-base.so.0.27.0` +
+`libggml-cpu.so.0.27.0` and nothing else, and our own
+`dankclaw/voiceservice/Containerfile.engine` writes `-DGGML_VULKAN=OFF` out explicitly, so
+the iGPU change is a flag in a file the fleet owns. Verdict, measured contract and the NPU
+exoneration chain: `tts-npu-spike-verdict-2026-10-19.md`.
+
+---
+
+## 1. What the payload is
+
+Two files, both read byte-for-byte off the live host (`/home/dank/models/Qwen3-TTS-GGUF/`,
+owned `dank:dank`, mtimes 2026-10-18 18:31), both from
+`https://huggingface.co/Serveurperso/Qwen3-TTS-GGUF` (community converter, not Qwen):
+
+| file | bytes | GGUF | tensors | arch string | file_type |
+|---|---|---|---|---|---|
+| `qwen-talker-1.7b-base-Q4_K_M.gguf` | 1,219,245,248 | v3, kv=64 | 480 | `qwen3-tts` | `Q4_K_M` |
+| `qwen-tokenizer-12hz-Q8_0.gguf` | 291,150,624 | v3, kv=51 | 398 | `qwen3-tts-tokenizer` | `Q8_0` |
+
+`general.name` of the talker file is `Qwen3-TTS-12Hz-1.7B-base`; the header sizes add up
+to the file sizes (sum of tensor bytes + header + alignment padding), so nothing in these
+files is unaccounted for.
+
+### 1.1 Talker file (480 tensors)
+
+Three models in one container, per `docs/ARCHITECTURE.md` §GGUF layout
+(qwentts.cpp @ pinned commit) and confirmed by the tensor census:
+
+| prefix | tensors | bytes | what it is |
+|---|---|---|---|
+| `talker.*` | 316 | 1,061.9 MB | 28-layer Qwen3 decoder (hidden 2048, FFN 6144, 16 Q heads / 8 KV heads, head_dim 128, RoPE θ=1e6, RMS ε=1e-6, ctx 32768), text stream + codec stream summed, `codec_head` → 3072 codebook-0 logits |
+| `code_pred.*` | 88 | 114.8 MB | 5-layer Qwen3 MTP head (hidden 1024, FFN 3072, 15 acoustic codebooks × 2048), own KV cache |
+| `spk_enc.*` | 76 | 36.6 MB | ECAPA-TDNN speaker encoder (x-vector, base checkpoints only; mel → conv0 → 3 SE-Res2Net → MFA → ASP → 2048-d vector) |
+
+Dtype split (exact tensor byte sums): `q4_K` 250 tensors / 864.1 MB, `q6_K` 17 / 312.0 MB,
+`f32` 176 / 25.8 MB, `f16` 37 / 11.4 MB.
+
+Which tensors carry which quant is the load-bearing portability fact:
+
+- `q6_K` (17): `talker.text_embd.weight [2048, 151936]` (255.3 MB), `talker.codec_embd.weight
+  [2048, 3072]`, and the 15 `code_pred.codec_embd.<g>.weight [2048, 2048]` embedding tables.
+  These are the llama.cpp `Q4_K_M` "embedding/output rows go to Q6_K" rule, applied to
+  embedding tables rather than to an `lm_head`.
+- `q4_K` (250): every attention/FFN projection of the talker and of the code predictor.
+  Distinct matmul shapes: `[2048,2048]`×73, `[2048,1024]`×62, `[2048,6144]`×56,
+  `[6144,2048]`×28, `[1024,2048]`×20, `[1024,3072]`×10, `[1024,1024]`×10, `[3072,1024]`×5,
+  `[2048,3072]`×2, `[2048,151936]`×1. Every one is 2-D and ≥ 1024 on both axes — good
+  GEMM shapes, nothing exotic.
+- `f32` (176) / `f16` (37): all of `spk_enc` (conv weights F16 because K=1/3/5 rows are not
+  block-alignable, per the converter's `should_quantize` policy in `tools/quantize.cpp` +
+  `quantize.sh`; every bias/norm/LayerScale vector F32), plus the talker/predictor norms and
+  the `text_proj` biases.
+
+Metadata the runtime actually reads (talker file): `qwen3-tts.num_code_groups=16`,
+`talker.vocab_size=3072`, `text_vocab_size=151936`, `text_hidden_size=2048`,
+`position_id_per_seconds=13`, `rope.mrope_section=[24,20,20]`, `mrope_interleaved=1`,
+the 7 codec special ids (pad 2148 / bos 2149 / eos 2150 / think 2154 / nothink 2155 /
+think_bos 2156 / think_eos 2157), `language_names` (10: chinese, english, german, italian,
+portuguese, spanish, japanese, korean, french, russian) with their codec ids, the text-side
+special ids (im_start 151644, im_end 151645, tts_pad 151671, tts_bos 151672, tts_eos 151673),
+the whole `generation_config.json` (do_sample, top_k 50, top_p 1.0, temperature 0.9,
+repetition_penalty 1.05, `subtalker_*` mirrors, max_new_tokens 8192), and an embedded
+**Qwen2 BPE tokenizer** (`tokenizer.ggml.model=gpt2`, 151,676 tokens, 151,291 merges,
+eos 151643) — that BPE block is 5.95 MB of the header, i.e. 49 % of the header is vocabulary.
+
+Two live-file details that contradict the upstream doc text (worth knowing when reading
+`docs/ARCHITECTURE.md`: the doc example block is stale against the shipped file):
+`mrope_interleaved` is **1** (doc shows `false`), and `spk_enc.sample_rate` is **24000**
+(doc says 16000). Doc also says "404 tensors on the 1.7B voice_design build" — the base
+build shipped here has 480 because it carries `spk_enc.*`.
+
+### 1.2 Tokenizer/codec file (398 tensors)
+
+One neural audio codec, 12.5 frames/s over 24 kHz mono, hop 1920, 16 codebooks × 2048
+entries (1 semantic + 15 acoustic). Dtype split: `f16` 50 / 143.6 MB, `q8_0` 110 / 76.9 MB,
+`f32` 238 / 70.7 MB. `tok_dec.*` 253 tensors / 201.3 MB, `tok_enc.*` 145 / 89.9 MB.
+
+Composition (metadata + tensor census, both files agree):
+
+- **Encoder** (used only to make voice references, not on the synthesis hot path): SEANet
+  conv stack (init k=7, 4 stages with ratios 8/6/5/4 → cumulative 960×, 64→512 ch, final
+  k=3), an 8-layer transformer (hidden 512, 8 heads, head_dim 64, FFN 2048, RoPE 1e4,
+  LayerNorm+bias, GELU, LayerScale), a downsample conv k=4 s=2 (25 → 12.5 Hz), then the RVQ
+  argmin loop. Metadata records `encoder.num_quantizers=32` with
+  `encoder_valid_num_quantizers=16` — 16 codebooks are used, 16 are dead weight carried from
+  the upstream config.
+- **Decoder** (the hot path: every generated frame): 16 codebook lookups + per-split
+  `output_proj` (256→512), `conv_pre`, an 8-layer transformer (hidden 512, 16/16 heads,
+  head_dim 64, FFN 1024, **sliding window 72**, RMSNorm, SwiGLU, LayerScale, input_proj
+  1024→512 / output_proj 512→1024), a ConvNeXt upampler (`upsampling_ratios=[2,2]`, causal
+  transposed convs + depthwise k=7), then a DAC-style decoder `dec.0..3`
+  (`upsample_rates=[8,5,4,3]`, channels 1536→768→384→192→96, `decoder_dim=1536`) whose blocks
+  are SnakeBeta → causal transposed conv (k = 2·stride) → 3 residual units (dilations 1/3/9)
+  → snake_post → conv_post k=7 → 96→1.
+- All 32 codebook tensors are **F32, 67.1 MB total** (16 encode + 16 decode, `[256, 2048]`),
+  deliberately unquantized: the converter's `should_quantize` keeps RVQ codebooks and their
+  wrapping projections at F32 because nearest-neighbour lookup is sensitive to per-row
+  quantisation noise (`docs/ARCHITECTURE.md` "Quantisation policy"; mirrored in
+  `quantize.sh`). 49 of the codec's 55 3-D conv tensors are **F16** with non-alignable K
+  (7, 3, 2, 1, 16, 10, …) — they cannot be block-quantized at all.
+- **No flow matching, no diffusion, no vocoder-with-ODE.** It is a pure RVQ codec
+  (Mimi/SoundStream/Mimi-style: SEANet + transformer + DAC decoder). That matters for the
+  NPU question: every component is convolution, attention, or embedding lookup.
+
+### 1.3 Where the payload came from, and what the official weights look like
+
+Official upstream (`huggingface.co/api/models`, retrieved 2026-10-19):
+
+| repo | last modified | license | weights |
+|---|---|---|---|
+| `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | 2026-01-23 | apache-2.0 | `model.safetensors` BF16, 1,928,677,440 params, plus an embedded `speech_tokenizer/model.safetensors` |
+| `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | 2026-01-29 | apache-2.0 | same layout, named-speaker tables |
+| `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | 2026-01-29 | apache-2.0 | 1.7B only |
+| `Qwen/Qwen3-TTS-12Hz-0.6B-Base` / `-0.6B-CustomVoice` | 2026-01-29 | apache-2.0 | narrow variant (hidden 1024) |
+| `Qwen/Qwen3-TTS-Tokenizer-12Hz` | 2026-01-29 | apache-2.0 | codec alone, F32, 170,557,441 params |
+
+So the GGUF is **not** the only public form: the safetensors are the canonical form
+(Apache-2.0, no separate community license risk), and the GGUF is a lossy re-quantization of
+them produced by the same author who wrote the runtime (`Serveurperso`), via
+`checkpoints.sh` → `convert.py` (F32 GGUF) → `quantize.sh` (BF16/Q8_0/Q4_K_M).
+Sizes to remember for any re-export: the talker container is 1.93 B params in BF16
+(that count includes the embedded codec), the codec alone is 170.6 M params in F32.
+
+**Portability consequence #1 — the GGUF is a dead end off-ggml.** Q4_K/Q6_K are ggml
+block formats (256-element super-blocks, F16 sub-block scales, 144 B / 210 B per block). No
+ONNX graph, no Vitis-AI/AIE-ML compiler, and no vendor NPU runtime consumes them; AMD's own
+quantizer works from ONNX (`ryzenai-quantize`, QDQ/QOP power-of-two — Ryzen AI Software
+1.8.0 release notes, https://ryzenai.docs.amd.com/en/latest/relnotes.html). Moving the
+payload to a non-ggml stack therefore means starting from the safetensors again, not
+converting the GGUF.
+
+**Portability consequence #2 — the quantization split is semantic, not cosmetic.** The
+converter keeps RVQ codebooks, `input_proj`/`output_proj` around them, and every 1-D and
+non-alignable-conv tensor out of the k-quants precisely because quantizing them breaks voice
+fidelity. Any re-quantization path (GPTQ/AWQ/QDQ-INT8/INT4 for an NPU compiler) inherits that
+constraint: the talker backbone is requantizable, the codec's codebook lookups and SnakeBeta
+convs are not, at least not without re-measuring voice quality.
+
+## 2. What the runtime is
+
+`qwentts.cpp` @ `5916bf7c` (2026-10-16, MIT; upstream model Apache-2.0) is a C++17 ggml
+application — 43 files under `src/` + `tools/`, three binaries (`qwen-tts`, `qwen-codec`,
+`tts-server`) over one public plain-C99 ABI (`src/qwen.h`, `qt_*`, `QT_ABI_VERSION`
+range-checked; `tests/abi-c.c` is compiled `-std=c99 -Wall -Werror -pedantic` on every
+build). It is not a llama.cpp derivative: it links ggml directly and builds its own graphs.
+
+Five graph families, and their shapes are the whole portability story:
+
+| graph | batch/time axis | notes |
+|---|---|---|
+| talker prefill | T = prompt length (hundreds–thousands) | summed text+codec embedding streams, 28-layer GQA decoder, KV cache |
+| talker decode | **T = 1 per audio frame** | 12.5 frames/s → 80 ms of audio per step; `codec_head` → 3072 logits |
+| code predictor (MTP) | **T = 1, 16 steps per frame** (2 prefill + 14 decode, cache reset every frame) | 5-layer decoder, 15 heads × 2048 |
+| speaker encoder | one pass per voice registration | ECAPA-TDNN with mel fused into the graph |
+| codec decode | offline: T = 300-frame chunks; streaming: **T = 1 frame** with persistent conv/KV-ring state | the streaming graph is built once with fixed topology so the CUDA graph cache stays in pure replay |
+
+Per generated frame the host therefore runs ~1 + 16 + 1 tiny graph computes for 80 ms of
+audio, and `tts-server` serializes synthesis FIFO (`--max-batch` defaults to 1, and the
+fleet runs it at 1). This is the archetypal latency-bound, bandwidth-bound,
+batch-1 workload; the only big-M matmuls in the whole system are the prefill and the
+chunked (buffered) codec decode.
+
+Op contract actually issued to ggml (`src/`, counted): `mul_mat` 77, `cont` 70, `add` 57,
+`mul` 40, `rms_norm` 22, `clamp` 15, `rope_ext` 12, `get_rows` 12, `set_rows` 8,
+`scale_bias` 6, `swiglu_split` 5, `sqr` 5, `mean` 5, `elu` 4, `col2im_1d` 4, `soft_max` 3,
+`conv_1d` 3, `im2col` 2, `cumsum` 2, `sin`, `sigmoid`, `tanh`, `div`, `argsort`, `transpose`,
+`permute`, `reshape/view`, plus the CPU-side Philox multinomial sampler.
+
+**No custom ggml op is required today.** The repo doc still says the fork "provides the two
+custom ops the codec needs: `GGML_OP_SNAKE` and `GGML_OP_COL2IM_1D`"; that is stale at the
+pinned commit. Verified 2026-10-19:
+
+- the submodule `ServeurpersoCom/ggml` @ `cc5df49` is upstream `ggml-org/ggml` master
+  (`a05c992`, 2026-10-15) **+2 commits, 0 behind** (GitHub compare API), and
+  `include/ggml.h` is byte-identical to upstream master (`diff` empty);
+- the 2 commits are `193e339` (2026-06-19) "cpu: add fused snake activation
+  (mul -> sin -> sqr -> mul -> add)" — a **CPU subgraph fusion**, not an op (the runtime
+  builds snake with `ggml_sin`/`ggml_sqr`/`ggml_mul`/`ggml_add` in `src/dac-decoder-v2.h:222`),
+  and `cc5df49` "sycl: fix get_rows on reordered quantized weights";
+- `ggml_col2im_1d` is an **upstream ggml API** (same declaration, line 2097, in both trees);
+  the Vulkan backend fuses the snake chain through ggml's generic subgraph fuser
+  (`ggml-vulkan.cpp:14548 ggml_vk_can_fuse_snake`, `vulkan-shaders/snake.comp`).
+
+Consequence: any stock ggml build (upstream, or llama.cpp's vendored ggml) can execute this
+payload; only the CPU snake fusion is fork-specific, and it is a performance nicety.
+
+Backend/device support: every tool takes `--device <name>` (`CUDA0`, `Vulkan0`, `MTL0`,
+`CPU…`, default best available), resolved through the ggml backend registry; the deployed
+container turns CUDA/HIP/Vulkan/Metal/SYCL/RPC/BLAS all OFF and compiles the native-CPU
+backend with OpenMP, with CPU threads = `hardware_concurrency()/2` (`src/backend.h`).
+The registry in this ggml knows CPU, CUDA, Metal, SYCL, Vulkan, WebGPU, ZDNN, OpenCL,
+**Hexagon**, BLAS, RPC, VirtGPU, **CANN**, ZenDNN, **OpenVINO**, ET — and nothing AMD-NPU
+(no `xdna`/`aie`/`vitis`/`ryzen` string anywhere in the ggml tree).
+
+Voice handling (what the fleet depends on): the registry is **process RAM only**; a restart
+empties it, which is why the unit has a boot warmer. `POST /v1/audio/voices` accepts either
+`{name, ref_text, wav_b64}` (server extracts) or `{name, ref_text, spk_b64, rvq_b64}`
+(pre-extracted latents verbatim); `ref_text` present ⇒ ICL clone mode, absent ⇒ x-vector-only
+mode. `.spk` = the ECAPA x-vector; `.rvq` = 16 codes/frame packed 11 bits LSB-first
+(`src/rvq-file.h`). `wav` and `pcm` `response_format` are different audio at the same seed
+(buffered chunked decode vs stateful streaming decode) — the fleet already encodes that rule.
+
+Parity expectations for any port: the upstream harness matches the PyTorch reference per
+stage (embed cos 0.999988, all 28 hidden taps ≥ 0.999893, logits 0.999824) but explicitly
+reports free-run divergence (`CodesFull exact 4.96 %`, `Audio cos 0.095`) and explains why —
+one argmax tie at FP epsilon diverts the AR trajectory. So a migrated endpoint must be
+accepted by listening + per-stage cosine, never by waveform diff against today's audio.
+
+---
+
+## 3. The three answers Main asked for
+
+### ANSWER A — Qwen3-TTS GGUF → ONNX: no, and it does not matter
+
+There is **no GGUF→ONNX path** and there will not be a useful one: GGUF is a weight
+container for ggml, its Q4_K/Q6_K block layouts have no ONNX consumer, and nothing in the
+GGUF carries the *graph* (the talker/predictor/codec structure lives in C++ code, only the
+op-level structure exists in ggml at runtime). But the GGUF is not the payload's only form —
+official Apache-2.0 safetensors are upstream, and **four independent ports of the full
+pipeline already exist**:
+
+| port | form | what it proves |
+|---|---|---|
+| `xkos/Qwen3-TTS-12Hz-1.7B-ONNX` (HF, updated 2026-03-03, apache-2.0) | FP16+FP32 ONNX, fp16/voice_clone + fp16/voice_design splits: `talker_decode.onnx`, `code_predictor.onnx`, `code_predictor_kv.onnx`, `code_predictor_embed_g0..15.onnx`, `text_embedding.onnx`, `codec_embedding.onnx`, shared `speaker_encoder.onnx`, `speech_tokenizer_encoder.onnx`, `speech_tokenizer_decoder.onnx`, `model_cache.npz` | every component of this architecture exports to ONNX, codec decoder included |
+| `zhangluoyang/qwen3-tts-onnxruntime` | export tooling + Python inference for Base/CustomVoice/VoiceDesign | the export is reproducible from the official checkpoints |
+| `Echo9Zulu/Qwen3-TTS-12Hz-CustomVoice-1.7B-INT8-OpenVINO`, `aurora2035/Qwen3-TTS-12Hz-0.6B-Base-OpenVINO-INT8` (2026-03) | OpenVINO IR, **INT8**: `talker`, `code_predictor`, `codec_embedding`, `text_model`, `openvino_speaker_encoder_model`, `speech_tokenizer/speech_decoder` | the talker and predictor INT8-quantize for an NPU-targeting stack |
+| `litert-community/Qwen3-TTS-12Hz-0.6B-Base` (2026-10-17, apache-2.0) | `qwen3_tts_int8.litertlm` / `_int4.litertlm` (LiteRT-LM `OmniEngine`), legacy split graphs `talker_int4.tflite` (prefill_32/prefill_128/decode signatures, KV 1024, blockwise-32 OCTAV INT4), `mtp_folded_int8.tflite` (16 inner steps × 5 layers folded into one graph, in-graph argmax+gather, GPTQ int8, ~5× faster), `codec_partA` (transformer, fp32) + `codec_partB` (conv upsampler, XNNPACK FORCE_FP16, ~2.5×, "ASR-identical") | an NPU/mobile compiler accepts the whole thing — at the price of graph splitting, static prefill buckets, and folding the per-frame inner loop |
+
+What the ports had to concede, and what any port inherits:
+
+- the generation structure does not fit existing engine loops: LiteRT-LM's Engine "does not
+  support this generation structure yet", so their first shape was **three graphs + a
+  host-side loop**, and the MTP step was "invoked 17× per frame" until they folded it;
+- per-step dispatch cost dominates: even on a Snapdragon 8 Elite Gen 5 they keep the MTP
+  loop, text projection, embedding gathers and the codec on CPU "avoiding 16 synchronous
+  CPU↔GPU round-trips per 80 ms frame", putting only the 28-layer talker on the GPU;
+- RTF on that hardware is **2.57–2.97 (Galaxy S26) and 5.65–8.48 (Pixel 10 Pro)** for the
+  0.6 B model, i.e. 2.6–8× slower than real time. Dankclaw's current CPU engine measures
+  **warm RTF 0.55–0.67 with TTFA 94–112 ms** on the 1.7 B model (fleet trial, 2026-10-18).
+  An NPU port is not "faster by default" for this workload; it has to beat that.
+- quantize the LM (int8/int4: proven), keep the codec's codebook lookups and SnakeBeta convs
+  in fp16/fp32 (the LiteRT split does exactly this).
+
+So: the answer to "can the fleet's payload be exported" is *yes — from the safetensors, not
+from the GGUF*, and the work is a pipeline rewrite (multi-graph host loop, per-frame fold,
+static prefill buckets), not a conversion. That is the same conclusion NeuTTS-Air's
+precompiled-graph route encodes: on AMD's Linux stack the graph is vendor-curated, so the
+fleet chooses the model that AMD already compiled rather than compiling Qwen3-TTS.
+
+### ANSWER B — is there any ggml backend for an AMD NPU? Not upstream; three out-of-tree, none for TTS
+
+Checked against the trees today (2026-10-19), because this was the crux:
+
+- **Nothing in-tree.** `ggml-org/llama.cpp` master (4,237 paths, GitHub tree API) contains no
+  `xdna`/`aie`/`vitis`/`ryzen` file or directory; the ggml backend set is CPU, BLAS, CUDA,
+  HIP, Metal, Vulkan, SYCL, OpenCL, WebGPU, MUSA, RPC, CANN (Ascend), **Hexagon** (Qualcomm),
+  OpenVINO (Intel CPU/GPU/NPU), ZenDNN, ZDNN, VirtGPU, ET. The deployed fork's registry
+  matches. The feature request "Feature Request: XDNA backend" (llama.cpp issue **#21725**,
+  opened 2026-04-10) is still **open**, 34 comments; the older #1499 was closed stale
+  2025-04-07. So Main's premise is confirmed: no ggml XDNA backend exists to flip a flag on.
+- **`Cyronius/ggml-xdna`** — a real ggml backend (`XDNA0`) loaded into *stock* llama.cpp via
+  `GGML_BACKEND_PATH`, no fork. It does bfp16 (8-value blocks sharing an exponent) GEMMs from
+  GGUF weights, and is **prefill-only**: it takes only matmuls in prompt chunks ≥ 1024 tokens
+  (`GGML_XDNA_MIN_BATCH` default 1024, `GGML_XDNA_MIN_MFLOP` 256), attention and every decode
+  token stay on the Vulkan device; pinned to llama.cpp release b10944; tested on Strix Point
+  (HX 370) only — Strix Halo "should pass, but nobody has run it on one yet"; Linux support
+  new and untested on hardware by the author; needs `libxrt-npu2`, `/dev/accel/accel0`,
+  unlimited `memlock`. Best measured prefill gain 1.88× on a 27 B model; models under
+  2,048-wide **lose** (0.77× at width 1,536).
+- **`BrandedTamarasu-glitch/OllamaAMDNPU`** — a llama.cpp fork with `-DGGML_XDNA=ON`,
+  explicitly targeting **Ryzen AI MAX / XDNA2 on Linux** (`/dev/accel/accel0`, XRT 2.21.75+,
+  amdxdna, unlimited memlock). It implements **only `GGML_OP_MUL_MAT`**, dispatched to
+  mlir-aie-compiled `.xclbin`s built per fixed tile shape (e.g. `M=2048 K=2048 N=64`,
+  int8 in / int32 out, per-row weight scales cached at first use), with a second slot for a
+  different K. Its own numbers on a Ryzen AI MAX 385 with Llama-3.1-8B Q4_K_M: NPU prefill
+  12–19 t/s vs **Vulkan iGPU 600–930 t/s**, NPU decode **0.65–1.40 t/s** ("Decode does not
+  use the NPU … M=1 per token, no xclbin covers it"), 0.6 % AIE utilization, and the stated
+  conclusion "**For LLaMA 8B on Strix Halo: use Vulkan exclusively**". Single-maintainer,
+  AI-assisted, self-measured — treat as a proof of concept, but it is the closest published
+  picture of what XDNA2 graph execution costs.
+- Also referenced in the same thread: `Atomic-Germ/OpenFlowLM-Next` (whole-model NPU engine,
+  LLM-only — and slower than the iGPU in the same comparison), `GaTmaNnes/xdna2-`,
+  `hardWorker254/Npu-Embeddings-XDNA1` (adds Whisper ASR ops, NPU1/Linux). None is a TTS path.
+
+Why none of them would carry this workload even if installed: the TTS hot loop is **batch-1
+by construction** (one talker step + 16 predictor steps + one codec frame per 80 ms), which
+sits under every dispatch gate those backends define (≥1024 tokens / ≥32768 M·N·K / tile M
+covering only prefill), and the codec half of the pipeline is conv, col2im, gather and
+elementwise ops — not `MUL_MAT` — so it would fall back to CPU op-by-op. The NPU case for
+this host is therefore *not* "run Qwen3-TTS on the NPU"; it is "run a vendor-compiled AR TTS
+graph on the NPU", which is exactly the NeuTTS-Air spike.
+
+### ANSWER C — would `qwentts.cpp` build for the gfx1151 iGPU? Yes, via **Vulkan**, not HIP
+
+- Upstream ships the build scripts and claims the backends: `buildvulkan.sh` ("AMD/Intel GPU
+  (Vulkan)"), `buildcuda.sh`, `buildsycl.sh`, `buildcpu.sh`, `buildall.sh` (all backends,
+  runtime dynamic loading), and README line 6 "runs on CPU, CUDA, Metal, Vulkan". CMake knobs
+  `GGML_VULKAN` / `GGML_HIP` / `GGML_SYCL` are the same ones the fleet's Containerfile turns
+  OFF, so the engine rebuild is a flag change: `-DGGML_VULKAN=ON` plus
+  `--device Vulkan0` on the same command line the unit runs today.
+- The codec's exotic ops are covered by the Vulkan backend, which is the thing that decides
+  whether a GPU build is coherent: `vulkan-shaders/col2im_1d.comp`, `snake.comp`,
+  `swiglu*.comp`, `cumsum*.comp` exist and `supports_op` handles `GGML_OP_COL2IM_1D` /
+  `CONV_TRANSPOSE_1D` / `CUMSUM`. By contrast `COL2IM_1D` appears in **no** HIP, OpenCL,
+  WebGPU, Hexagon or CANN source — a **HIP** (ROCm) build would shard the ConvNeXt/DAC
+  transposed convs back to the CPU per op. On an AMD iGPU the portable path is RADV/Vulkan.
+- `--device` also accepts a multi-device split (`-dev`-style naming is ggml's), and the
+  runtime's documented FP16 guard (`--clamp-fp16`) exists for sub-Ampere CUDA accumulation;
+  nothing in the tree is NVIDIA-specific.
+- Unverified by me: any vendor statement pairing this fork with gfx1151. Supporting evidence
+  that the hardware/driver combination is viable: this very host already runs a Vulkan
+  endpoint on the gfx1151 iGPU (fleet ground truth, 2026-10-19), and Mesa's Vulkan
+  implementation is officially supported on Radeon Software for Linux
+  (AMD release notes RN-AMDGPU-UNIFIED-LINUX-25.20.3). A gfx1151 Vulkan build of this fork is
+  a build-and-measure spike, not an unsupported port.
+- The real constraint is the one already in the fleet docs: the iGPU belongs to another
+  endpoint, and GPU+NPU concurrency has hung Strix Halo machines. An iGPU TTS build competes
+  with the text endpoints for the same silicon; the NPU route's only genuine advantage is
+  that it does not (the XDNA backend authors make the same argument: "Use NPU when the iGPU
+  is busy … no GPU contention").
+
+---
+
+## 4. What any replacement must reproduce, and what the voices cost
+
+Endpoint contract in production today (from the live unit/env, not the doc):
+`tts-server --model qwen-talker-1.7b-base-Q4_K_M.gguf --codec qwen-tokenizer-12hz-Q8_0.gguf
+--alias frank-tts-engine --host 0.0.0.0 --port 8880 --lang auto --max-batch 1`, no published
+port (the façade on :7772 dials the private bridge name), `GET /health`, `GET /v1/models`,
+`GET|POST /v1/audio/voices`, `DELETE /v1/audio/voices/{name}`, `POST /v1/audio/speech` with
+`response_format` wav|pcm (different audio at equal seed) and optional `seed`,
+`max_new_tokens`, `temperature`, `top_k`, `top_p`, `repetition_penalty`, `subtalker_*`.
+Voice registry is RAM-only, so boot warming is mandatory, and `--alias` keeps the quantization
+choice out of every consumer's config.
+
+Live voice census on Dankclaw (2026-10-19, `/home/dank/voices/`): three packs
+(`narrator1..3`), each carrying `reference.wav` (24 kHz mono crop), `reference16k.wav`,
+`manifest.json` (keys: name, kind, generated_utc, pipeline, source, forbidden_region_guard,
+silence_scan, crop, loudnorm, audio, acceptance, acceptance_passed) and ffmpeg analysis
+artifacts. **No `.spk` and no `.rvq` exist on the live host** — the repo's
+`build-voice-pack.sh` treats them as required-but-they-are-not-there, so today the engine
+re-extracts the x-vector (and the ICL codes when a transcript is supplied) from the WAV at
+registration time.
+
+That is the portable part, and it is the same shape AMD's NeuTTS flow wants
+(`encode_reference(reference.wav)` + a reference transcript): the fleet's voice identity is
+*the WAV plus its transcript*, processed at request/registration time by the model's own
+speaker encoder + codec — **never a trained per-voice artifact**. Qwen3-TTS cloning is
+inference-time conditioning (`generate_voice_clone(ref_audio=…, ref_text=…, x_vector_only_mode=…)`;
+`create_voice_clone_prompt` exists purely to avoid recomputing prompt features). So any
+replacement that consumes a WAV (+transcript) keeps all three narrators alive with no
+re-derivation, and if it wants precomputed latents instead, the LiteRT precedent shows what
+that looks like: a 4 KB x-vector `.npy` per voice, extracted offline
+(`extract_speaker_embedding.py`) — for the 1.7 B base checkpoint that vector is 2048-dim
+(`qwen3-tts.spk_enc.embedding_length = 2048` in the shipped GGUF; the LiteRT 0.6 B bundle
+ships a 1024-dim one).
+
+One trap for a mixed-runtime future: `.spk`/`.rvq` files, if they are ever written, are
+engine-specific encodings (ECAPA x-vector layout; 11-bit-packed RVQ stream) and are **not**
+interchangeable between runtimes. Keep `reference.wav` as the pack's source of truth and
+regenerate anything derived per engine — which is what the NeuTTS re-clone plan does anyway.
+
+## 5. Sources
+
+Runtime/payload (all read locally at pinned versions, 2026-10-19):
+
+- `github.com/ServeurpersoCom/qwentts.cpp` @ `5916bf7c79dbb4e66fc059cdc507a79a23a28bb4`
+  (commit date 2026-10-16; also upstream HEAD today): `README.md`, `docs/ARCHITECTURE.md`
+  (860 lines), `src/*`, `tools/*`, `docs/DOCKER.md`, `buildvulkan.sh`/`buildcpu.sh`/`buildall.sh`.
+- submodule `ServeurpersoCom/ggml` @ `cc5df49a6b878db2b68b9a7b45976e279594627d` (2026-10-16);
+  delta vs `ggml-org/ggml` master `a05c992` (2026-10-15) = ahead 2 / behind 0, per
+  https://github.com/ServeurpersoCom/ggml/compare/ggml-org:master...HEAD ;
+  `include/ggml.h` identical to upstream
+  https://raw.githubusercontent.com/ggml-org/ggml/master/include/ggml.h .
+- Live payload bytes: `/home/dank/models/Qwen3-TTS-GGUF/qwen-talker-1.7b-base-Q4_K_M.gguf`
+  (1,219,245,248 B) and `qwen-tokenizer-12hz-Q8_0.gguf` (291,150,624 B), headers parsed from
+  the host; sizes/sha256 match `scripts/download-frank-tts-engine-qwentts.sh`.
+- Live units/env/packs: `dankclaw/voiceservice/{Containerfile.engine,tts-engine-start.sh,tts-engine-warm.sh,quadlet/*}`,
+  `scripts/frank-tts-engine.env`, `scripts/frank-tts.env`, `/home/dank/voices/*`.
+
+Model provenance:
+
+- `https://huggingface.co/api/models/Qwen/Qwen3-TTS-12Hz-1.7B-Base` (+ CustomVoice,
+  VoiceDesign, 0.6B variants, `Qwen3-TTS-Tokenizer-12Hz`), apache-2.0, last-modified
+  2026-01-23 / 2026-01-29; safetensors param counts 1,928,677,440 (BF16, incl. embedded
+  `speech_tokenizer/`) and 170,557,441 (F32 codec).
+- `https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/raw/main/README.md` (clone API:
+  `generate_voice_clone(ref_audio, ref_text, x_vector_only_mode)`,
+  `create_voice_clone_prompt`; reference stack = PyTorch + `device_map="cuda:0"` + bf16 +
+  flash-attn-2, Python 3.12 recommended).
+- `https://github.com/QwenLM/Qwen3-TTS`.
+
+Portability precedents:
+
+- `https://huggingface.co/xkos/Qwen3-TTS-12Hz-1.7B-ONNX` (2026-03-03) ·
+  `https://github.com/zhangluoyang/qwen3-tts-onnxruntime` ·
+  `https://huggingface.co/Echo9Zulu/Qwen3-TTS-12Hz-CustomVoice-1.7B-INT8-OpenVINO` (2026-03-22) ·
+  `https://huggingface.co/aurora2035/Qwen3-TTS-12Hz-0.6B-Base-OpenVINO-INT8` (2026-03-18) ·
+  `https://huggingface.co/litert-community/Qwen3-TTS-12Hz-0.6B-Base` (2026-10-17) incl.
+  `legacy/README.md` (graph splits, MTP fold, codec partA/partB, RTF table).
+
+AMD NPU / ggml backend status:
+
+- `https://api.github.com/repos/ggml-org/llama.cpp/git/trees/master?recursive=1` (no
+  XDNA/AIE/Vitis backend, 2026-10-19) ·
+  `https://github.com/ggml-org/llama.cpp/issues/21725` (open since 2026-04-10) ·
+  `https://github.com/ggml-org/llama.cpp/issues/1499` (closed stale 2025-04-07).
+- `https://github.com/Cyronius/ggml-xdna` (prefill-only, gates, Linux requirements, results
+  measured 2026-10-05/06 on HX 370) ·
+  `https://github.com/BrandedTamarasu-glitch/OllamaAMDNPU` (XDNA2 ggml backend on Strix Halo,
+  phases 1–12, "use Vulkan exclusively", measured on Ryzen AI MAX 385).
+- `https://github.com/amd/RyzenAI-SW` (main, 669 paths, 2026-08-18): audio content is ASR
+  only (`Demos/ASR/Whisper`, `Demos/ASR/Parakeet-TDT`, `Transformer-examples/ASR/Whisper-AI`);
+  **no TTS example anywhere in the official Ryzen AI Software tree**.
+- `https://ryzenai.docs.amd.com/en/latest/relnotes.html` (Ryzen AI SW 1.8.0: ONNX-format
+  models, ONNX Runtime as the loader, power-of-two QDQ/QOP quantization).
+- Mesa/RADV on AMD iGPUs: `https://www.amd.com/en/resources/support-articles/release-notes/RN-AMDGPU-UNIFIED-LINUX-25-20.3.html`
+  (Mesa Vulkan driver officially supported).
+
+Status: COMPLETE for the assigned slice (payload architecture, runtime architecture,
+portability). `[INFERENCE]` marks: none above except the Mesa-version caveat in ANSWER C,
+which is flagged there.

report/tts-npu-spike-verdict-2026-10-19.md

@@ -0,0 +1,191 @@
+# TTS on the Strix Halo NPU — spike verdict (2026-10-19)
+
+Question: **how can the fleet's TTS engine run on hardware rather than CPU?** Answer in
+one line: not on the NPU — the loader, not the silicon, refuses it; the door that exists is
+the gfx1151 iGPU behind a build flag we already own.
+
+Spike evidence lives in `report/npu-tts-runtime-landscape-2026-10-19.md` (which runtimes can
+reach XDNA2 at all) and `report/qwen3-tts-payload-portability-2026-10-19.md` (what the
+payload is and where it can be moved). This page owns only the **measured spike result** and
+the **model-side contract** neither slice has. Throwaway work ran in containers under
+`/home/dank/spikes/npu-tts` on Dankclaw; nothing in the fleet was changed — `frank-tts` and
+`frank-tts-engine` still serve `narrator1|2|3` on CPU.
+
+Legend: **[host]** live census 2026-10-19 · **[spike]** measured in the spike containers ·
+**[vendor]** primary vendor source, with the version it holds for.
+
+## 1. Verdict
+
+**NO-GO on the NPU, and the wall is a loader.** The XDNA2 device is healthy by AMD's own
+instruments; the runtime that owns this model family opens the device and then declines the
+graph, silently. One run loaded successfully; byte-identical runs never reproduced it, and a
+path that cannot be reproduced is not a foundation. **[spike]**
+
+**Go on the iGPU, and the lever is in this repo.**
+`dankclaw/voiceservice/Containerfile.engine` builds `qwentts.cpp` (commit `5916bf7`) with
+`-DGGML_VULKAN=OFF` written out deliberately beside CUDA/HIP/METAL/SYCL/RPC — the running
+container carries `libggml-base.so.0.27.0` + `libggml-cpu.so.0.27.0` + `tts-server` and
+nothing else. So: `-DGGML_VULKAN=ON` + `--device Vulkan0`, plus `/dev/dri` passthrough in
+the engine quadlet (the ComfyUI pattern: `AddDevice` + `GroupAdd=keep-groups` + a
+world-writable udev rule). The target shape is **co-tenancy** — the engine resident on the
+gfx1151 *beside* `frank-dragoon`, not a handover. §5 carries the census and both gates:
+the **heap gate is PASSED** — RADV's `DEVICE_LOCAL` heap is 77.33 GiB with 65.16 GiB of
+budget while dragoon is loaded, so memory is not the constraint — which leaves the
+contention gate (dragoon's TTFT with a synthesis in flight) and one unproven piece:
+whether qwentts.cpp's codec ops have Vulkan kernels for every op. **[host]** **[spike]**
+
+Why the iGPU is worth doing at all, given CPU already works: the deployed payload needs
+only **12.5 codes/s** (§2), so this is latency headroom and freed cores, not reviving a
+dead service. The independent Strix Halo measurement in the portability slice ("use Vulkan
+exclusively"; NPU decode under 1.5 t/s for an 8B) points the same way.
+
+## 2. The two arms do not share a real-time bar
+
+|Arm|Talker|Codec|Codes per second of audio|Bar|
+|---|---|---|---|---|
+|**Deployed** (`frank-tts-engine`)|`qwen-talker-1.7b-base-Q4_K_M.gguf`|`qwen-tokenizer-12hz-Q8_0.gguf`|**12.5**|~13 tok/s **[host]**|
+|**Spike** (`amd/neutts-air_rai_1.8.0_npu_4K`)|NuTTS-Air, 24-layer Qwen2|NeuCodec `24000 / hop 480`|**50.00**|~50 tok/s **[spike]**|
+
+Measured from the staged NPU artifact — the contract any future arm must satisfy:
+
+|Fact|Value|
+|---|---|
+Vocab|`genai_config.json` `vocab_size` **217,652**; base BPE 151,643; specials 151,644–151,670 **[spike]**|
+Speech tokens|**65,536** `<|speech_N|>` entries, ids **151,671–217,206** contiguous; the vendor template maps a codec index straight to its id, `"".join(f"<|speech_{i}|>")`. A 10,001-code assumption silently truncates the space — encoded references really do reach the 60k range. **[spike]**|
+Stop token|`<|SPEECH_GENERATION_END|>` = 151,670, but the export's `model.eos_token_id` is `[151645, 151643]` (the *chat turn* stop) and `search.eos_token_ids` is unset — pass 151,670 explicitly or speech ends after one token **[spike]**|
+Input ceiling|4096 tokens **total**, input + output **[vendor]**|
+Frame rate|50.00 codes/s confirmed across three references (49.96 / 49.99 / 50.00) **[spike]**|
+Reference encode|`librosa.load(sr=16000, mono=True)` → `[1,1,T]` float → `codec.encode_code(audio_or_path=…)` (vendor `NeuTTS.encode_reference`); CPU RTF 0.28–0.38 on 8 threads, so codes are a cacheable one-time cost, never a per-request charge. `neucodec` also needs `torchtune`; install both `--no-deps --target <dir>` so a second torch never shadows the venv's. **[spike]**|
+Pre-encoded voices|`/spike/logs/codes-narrator{1,2,3}.npy` — 994 / 1,249 / 1,250 codes for 19.90 / 24.98 / 25.00 s **[spike]**|
+
+**Gap that any arm inherits:** the voice packs carry no transcript. `manifest.json` holds
+provenance and crop geometry (`crop.source_speech_run` 98.739–118.636 s of a 22,569 s
+audiobook) and no text, and Dankclaw has no ASR at all — `whisper`, `whisper.cpp`,
+`faster-whisper` all absent. NuTTS conditioning needs `ref_text`; do not invent it. **[host]**
+
+## 3. Exoneration chain — why the blame is not the hardware
+
+Every row is an instrument, not an inference.
+
+|Probe|Result|
+|---|---|
+`xrt-smi examine`|`[0000:c6:00.1] RyzenAI-npu5 aie2p 6x8`, XRT 2.25.37 (vendor extract), NPU fw 1.1.2.65, `amdxdna 7.0.0-38-generic` **[spike]**|
+`xrt-smi validate`|**PASSED**, including `throughput` **[spike]**|
+`pyxrt.device(0)` in-container|constructs — XRT enumerates the device **[spike]**|
+`strace -f -e trace=openat` on the failing load|`genai_config.json` read → `model.onnx` opened → `libxrt_driver_xdna.so.2` loaded → `/dev/accel/accel0` opened `O_RDWR` → GEM ioctls succeed → **EP declines, before touching `dd_metastate_*` or `cache/txn_bins.zip`** **[spike]**|
+`dmesg`, `fuser /dev/accel/accel0`|silent since boot, no holder — the decline never reaches the kernel **[spike]**|
+Hardlinked mirror dirs of the model tree (`rundir*`)|retired, no change — the "metastate shadowing" theory is dead **[spike]**|
+Every logging lever in that build: `GENAI_ORT_VERBOSE_LOGGING`, `GENAI_LOG_ORT_LIB`, `set_log_options`, `set_log_callback(callable)`, `RYZENAI_EP_TRACING`, `RYZENAI_LLM_PROFILE_INIT`|**zero lines** — the decline path is silent **[spike]**|
+Vendor `quicktest`|`Session successfully initialized` — proves the XRT/driver/device path, **not** NPU residency: that build falls back to CPU silently. Prove residency with `/sys/class/accel/accel0/device/mem_usage` during generation. **[spike] [vendor]**|
+
+No NPU token rate was ever measured, so none is quoted here. The single successful load
+(13:17, `[load] 0.84s`, finite logits, XRT banner at *generate* time) has not been
+reproduced by the byte-identical command since, with the model files pristine by inode and
+mtime. **[spike]**
+
+## 4. Host gates the spike proved (so they are not re-litigated)
+
+- **Rootless containers cannot read `/dev/accel/accel0`** (`root:render` 0660): the host
+  gid never enters the userns map. `--group-add keep-groups`, `--userns=keep-id`,
+  `--userns=keep-id:gid=990`, `--group-add render`, `--group-add gid:990` — all fail.
+  Doors: rootful podman (separate storage), or a world-writable udev rule, which the fleet
+  already does for `kfd`/`renderD*`. **[spike]**
+- **Never `dpkg -i` the vendor XRT deb** — it installs a DKMS `amdxdna` over the distro
+  driver. `dpkg-deb -x` the debs and bind-mount the tree at `/opt/xilinx/xrt`; the plugin
+  deb is what carries `libxrt_driver_xdna.so`. **[spike]**
+- In-tree `amdxdna 0.7.0-38` + `iommu=pt` is enough for context creation; a missing device
+  node is *not* the modern failure mode. **[host]**
+- `xrt-smi` inside a container needs `libboost-filesystem1.83.0` +
+  `libboost-program-options1.83.0` (**hyphenated** Ubuntu names, after `apt update`);
+  boost is needed by `xrt-smi` only, never by the OGA/EP path. **[spike]**
+- Headroom is tight and it is host RAM, not a separate pool — see §5 for tonight's census.
+  A multi-GB NPU partition plus an AIE compile workspace does not fit beside
+  `frank-dragoon`. Benign-looking `Failed to alloc dev bo memory, ret -28` is that
+  pressure. **[host]**
+
+## 5. Target shape: the TTS engine shares the gfx1151 with `frank-dragoon`
+
+The deployment Frank wants is **both resident at once**, not a handover. Measured 2026-10-19
+(read-only, nothing loaded or unloaded):
+
+|Measure|Value|Reading|
+|---|---|---|
+iGPU GTT in use|**18,523 MiB of 116,733 MiB**|≈96 GiB of aperture free; this is the real budget (fleet law) **[host]**|
+DRM "VRAM" carve-out|2,048 MiB total, 174 MiB used|what KFD/DRM calls dedicated — **not** the budget **[host]**|
+Host RAM|121 GiB total, 17 GiB available, 79 GiB buff/cache|**GTT pages are system pages**: an iGPU allocation is host RAM consumed, so the two tenants share one pool **[host]**|
+TTS engine today|3.84 GiB RSS, CPU|its device-resident replacement lands in that same pool **[host]**|
+
+**Two gates, in this order.**
+
+1. **Heap gate — PASSED.** RADV does **not** hand a container tenant the 2 GiB DRM
+   carve-out. `vulkaninfo` inside the engine's own base image (`ubuntu:24.04` plus
+   `mesa-vulkan-drivers vulkan-tools`, `--device /dev/dri`, rootless) sees
+   `Radeon 8060S Graphics (RADV GFX1151)`, driver `radv`, API 1.4.318, and reports:
+
+   |Heap|size|budget with dragoon loaded|flags|
+   |---|---|---|---|
+   |0|38.67 GiB|32.58 GiB|host-visible, not device-local|
+   |1|**77.33 GiB**|**65.16 GiB**|`DEVICE_LOCAL`|
+
+   The heaps sum to the ~116 GiB aperture in Mesa's ⅓/⅔ APU split — the same RAM twice:
+   a Vulkan allocation is a GTT mapping, so a budget is a ceiling on mappings, not a pile
+   of free silicon memory. ggml's Vulkan backend asks for `DEVICE_LOCAL`, which is heap 1,
+   and the TTS engine needs roughly 4 GiB of it. **Memory stops being the question.** **[spike]**
+   Probe traps: `vulkaninfo --batch` is not a flag in this `vulkan-tools` (it prints the
+   help page and exits 1 — use `vulkaninfo --text`), and the container has **no NVIDIA
+   ICD**, so Vulkan device 0 is the iGPU here. Do not trust that ordering — select by
+   device *name*, or an image that grows an `nvidia_icd.json` silently points TTS at the
+   3090 slot.
+2. **Contention gate — PASSED, measured 2026-10-19 after the cutover.** With a 16.9 s
+   synthesis occupying the device, `frank-dragoon` answered a streamed 124-token request at
+   **17 ms to first token and ~35 tok/s decode**, against 17 ms and ~33 tok/s for the same
+   request with the card idle. Compute-unit sharing showed up as nothing. One request per
+   arm, no ladder: these are the gate's evidence, not a throughput record, and the engine's
+   own shape lives in [[Frank TTS Engine]].
+
+**Change surface — as landed** (PR #34, `6248f02`; the CPU shape stays reachable as the
+rollback):
+
+1. `Containerfile.engine` build stage: add **`libvulkan-dev glslc spirv-headers`**. The
+   prediction above (`libvulkan-dev glslang-dev`) is wrong on two counts and the configure
+   says so: ggml looks for `glslc`, which Ubuntu's `glslang-tools` does **not** ship (it
+   ships `glslangValidator`), and `find_package(SPIRV-Headers CONFIG REQUIRED)` needs the
+   distro's `spirv-headers`, which does carry the CMake config. `glslc` is its own source
+   package on noble, at 2023.8 — old enough that only `GL_KHR_cooperative_matrix` compiles,
+   so the coopmat2 / integer-dot / bf16 / micro-float shader families are built out. Suspect
+   that compiler before the device if a Vulkan prefill ever reads as slow.
+2. `-DGGML_VULKAN=OFF` → `ON`: one word in the written-out flag list.
+3. Runtime stage: `mesa-vulkan-drivers` **and** `libvulkan1` — the ICD and its loader. An
+   image with a loader and no ICD enumerates zero devices.
+4. `frank-tts-engine.container`: `AddDevice=/dev/dri` + `GroupAdd=keep-groups` — the
+   DIRECTORY, never the node: on this hybrid box `renderD128` is the NVIDIA card and
+   `renderD129` the amdgpu iGPU (`readlink -f /sys/class/drm/renderD129/device/driver` →
+   `amdgpu`). On this host the udev rule already makes the nodes world-writable, so the
+   group is belt-and-braces, not the gate.
+5. Engine argv: `--device Vulkan0`, reached through `QWENTTS_DEVICE` in
+   `scripts/frank-tts-engine.env`. **An empty value passes no flag at all** — that is the
+   CPU rollback, and it is why the entrypoint's default is an `-v` test rather than `:=`,
+   which also fires on a set-but-empty value and would silently undo the rollback.
+
+Gate 1 recipe, already run and **PASSED**: `mesa-vulkan-drivers` + `vulkan-tools` in a
+throwaway container off the same base, `--device /dev/dri`, then `vulkaninfo --text` and read
+the `VkPhysicalDeviceMemoryProperties` heap sizes — no model, no load, no compute. The probe
+trap: `vulkaninfo --batch` is not a flag in Ubuntu 24.04's `vulkan-tools` — it prints the help
+page and exits 1, which reads like a driver failure. Every package involved has a candidate on
+the host's apt (`vulkan-tools` 1.4.341, `libvulkan-dev` 1.4.341, `glslang-dev`/`glslang-tools`
+16.2.0); the build stage runs on the digest-pinned Ubuntu 24.04 base, whose versions are older
+but present. Script: `/home/dank/spikes/npu-tts/vk_probe.sh` on Dankclaw.
+
+Constraints that do not relax for a co-tenant: `/dev/dri` passthrough needs the ComfyUI
+pattern (`AddDevice` + `GroupAdd=keep-groups` + the world-writable udev rule) because
+systemd user-unit spawns carry no supplementary groups; the 64 GiB swap at `/swap.img`
+exists to protect co-tenants, and paging a KV pool is worse than the OOM it prevents;
+`pp_dpm_fclk` reads `auto` (unheld), which only becomes a hazard if the NPU ever joins.
+
+## 6. Re-visit condition
+
+Retry the NPU when **a Linux loader claims the graph**: an `onnxruntime-genai-ryzenai`
+build whose RyzenAI adapter takes the NuTTS-Air Full-Fusion graph on `RyzenAI-npu5`, or that
+model appearing in a runtime's own supported catalog with a Linux runner. Then the
+prerequisites are already mapped and priced: the gated NeuCodec encoder for cloned voices,
+the `ref_text` transcripts, the 4096-token prompt budget, and 50 codes/s as the bar.

scripts/apply-protected-cast.sh

@@ -1,27 +1,45 @@
 #!/usr/bin/env bash
-# apply-protected-cast.sh — RUNS ON DANKSERVER. Rewrite the recipe's
-# PROTECTED-BEGIN/END section from Swift's OWN calibration grams: run
+# apply-protected-cast.sh — RUNS ON DANKSERVER. Rewrite a recipe's
+# PROTECTED-BEGIN/END section from the campaign's OWN calibration grams: run
 # derive-protected-experts.py over the calib dir, splice its cast lines into
 # the marked section of the recipe in place, keep the pre-edit recipe in the
-# campaign trash. Refuses to run twice (marker text says derived).
+# campaign trash. Refuses to run twice (marker text says derived), and refuses
+# when a requested target count was not reached. Run it on a COPY of the
+# template so the committed recipe stays the clean, reviewable baseline.
 set -euo pipefail
-CALIB=${1:-/home/dank/corpora/calib}
-RECIPE=${2:-/home/dank/serverAppConfigs-swift15/scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe}
-PY=/home/dank/venvs/radcorpus/bin/python
-DERIVE=/home/dank/serverAppConfigs-swift15/scripts/derive-protected-experts.py
+SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CALIB=${1:-/home/dank/campaign/qwen38-w5nl64/calib}
+RECIPE=${2:-$SELF_DIR/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe}
+TARGET_COUNT=${3:-0}
+PY=${PY:-/home/dank/venvs/radcorpus/bin/python}
+DERIVE=${DERIVE:-$SELF_DIR/derive-protected-experts.py}
 TS=$(date +%Y%m%d-%H%M%S)
+# OUT and TRASH_DIR are overridable so the seam can drive the whole script in a
+# scratch directory; on the box they are the campaign's usual places.
+OUT=${OUT:-/tmp/protected-cast-$TS.txt}
+TRASH_DIR=${TRASH_DIR:-/home/dank/.AGENTTRASH}
 
 grep -q "derive-protected-experts.py rewrote" "$RECIPE" && {
     echo "recipe already carries derived cast lines; refusing to re-derive" >&2; exit 1; }
 grep -q "PROTECTED-BEGIN" "$RECIPE" || { echo "no PROTECTED markers in $RECIPE" >&2; exit 1; }
 
-"$PY" "$DERIVE" "$CALIB" > "/tmp/protected-cast-$TS.txt"
-N=$(grep -c 'cast dtype=bf16' "/tmp/protected-cast-$TS.txt")
-[ "$N" -ge 2 ] || { echo "derive produced $N cast lines; refusing" >&2; exit 1; }
-echo "derived $N protected experts -> /tmp/protected-cast-$TS.txt"
+ARGS=("$DERIVE" "$CALIB")
+if [ "$TARGET_COUNT" -gt 0 ]; then
+    ARGS+=(--target-count "$TARGET_COUNT")
+fi
+"$PY" "${ARGS[@]}" > "$OUT"
+N=$(grep -c 'cast dtype=bf16' "$OUT")
+[ "$N" -ge 2 ] || { echo "derive produced $N cast lines; refusing (see $OUT)" >&2; exit 1; }
+if [ "$TARGET_COUNT" -gt 0 ] && [ "$N" -lt "$TARGET_COUNT" ]; then
+    echo "derive reached $N protected experts, target was $TARGET_COUNT; refusing to convert" >&2
+    echo "       (the bar sweep bottomed out: the recording is too thin or the target too high)" >&2
+    exit 1
+fi
+echo "derived $N protected experts -> $OUT"
 
-cp -f "$RECIPE" "/home/dank/.AGENTTRASH/$TS-qwen4exp-swift15-w4dn8-i8-hc8m.recipe"
-"$PY" - "$RECIPE" "/tmp/protected-cast-$TS.txt" <<'PYEOF'
+mkdir -p "$TRASH_DIR"
+cp -f "$RECIPE" "$TRASH_DIR/$TS-$(basename "$RECIPE")"
+"$PY" - "$RECIPE" "$OUT" <<'PYEOF'
 import sys
 recipe, castfile = sys.argv[1], sys.argv[2]
 lines = open(recipe).read().splitlines()
@@ -35,7 +53,7 @@ for l in lines:
     elif 'PROTECTED-END' in l:
         insec = False
         out.append(l)
-        out.append('# (derive-protected-experts.py rewrote this section from Swift\'s own grams)')
+        out.append('# (derive-protected-experts.py rewrote this section from the campaign grams)')
     elif not insec:
         out.append(l)
 open(recipe, 'w').write('\n'.join(out) + '\n')

scripts/authentik-grafana-sso-fix.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# Repair the Grafana OAuth2 relying-party SSO on Dankclaw's authentik.
+#
+# grafana.franksdank.cloud logs in through authentik as an OAuth2 relying party.
+# Two DB-side facts (the provider was created by hand via `ak shell`, never
+# repo-managed) silently broke that login, and BOTH had previously been
+# mis-recorded here as a single "the property-mapping evaluator is broken, so
+# userinfo returns only `sub`" myth. They are neither, and the evaluator is fine:
+#
+#   1. provider.authorization_flow was NULL. The authorize view builds a flow
+#      planner from it and crashes with
+#      `AttributeError: 'NoneType' object has no attribute 'slug'` — an HTTP 500
+#      "Server Error — Powered by authentik", before any claim is touched.
+#      Fix: point it at default-provider-authorization-implicit-consent, the
+#      flow every other provider in this instance already uses.
+#   2. The `grafana-email-scope` ScopeMapping's expression was the bare string
+#      `user.email`. authentik merges a scope mapping's result into the token as
+#      a MAPPING: a returned dict adds its keys as claims; a returned string is
+#      keyed under the mapping's NAME ("grafana-email-scope"), so no `email`
+#      claim is ever emitted. Grafana, finding no email, falls back to
+#      GET <api_url>/emails, authentik 404s that path, and the login dies with
+#      "Login failed: InternalError". Fix: return
+#      {"email": user.email, "email_verified": True} — the shape the stock
+#      OpenID 'email' mapping uses. The dict-valued `grafana-profile-scope`
+#      mapping has always worked, which is the proof the evaluator is fine.
+#
+# This is authentik DB state. It is NOT in any deployable file: a blueprint
+# re-import, or an ak-shell re-creation of the grafana provider, reverts BOTH
+# facts and the symptom returns verbatim. Re-run this after any such rebuild.
+# Every step is idempotent.
+#
+# Grafana's own half — SCOPES="openid email profile", a standard userinfo API_URL
+# with no `?api=1`, and ALLOW_SIGN_UP=true — lives in
+# dankclaw/observability/quadlet/grafana.container and deploys normally.
+#
+# Usage: authentik-grafana-sso-fix.sh          (override host: AUTHENTIK_HOST=…)
+set -euo pipefail
+HOST="${AUTHENTIK_HOST:-dankclaw}"
+
+ssh "$HOST" 'podman exec -i authentik-server ak shell' <<'PYEOF'
+from authentik.flows.models import Flow
+from authentik.providers.oauth2.models import OAuth2Provider
+
+p = OAuth2Provider.objects.filter(name="grafana").first()
+if not p:
+    raise SystemExit("no OAuth2Provider named 'grafana'")
+
+# 1. The authorization flow every other provider uses.
+want = Flow.objects.filter(slug="default-provider-authorization-implicit-consent").first()
+if not want:
+    raise SystemExit("default-provider-authorization-implicit-consent flow is missing")
+if p.authorization_flow_id != want.pk:
+    p.authorization_flow = want
+    p.save()
+    print(f"set grafana.authorization_flow -> {want.slug}")
+else:
+    print(f"grafana.authorization_flow already {want.slug}")
+
+# 2. The email scope mapping must RETURN A DICT, not a bare string.
+m = p.property_mappings.filter(name="grafana-email-scope").first()
+if not m:
+    raise SystemExit("grafana-email-scope mapping is missing")
+fixed = 'return {"email": user.email, "email_verified": True}'
+if m.expression.strip() != fixed:
+    m.expression = fixed
+    m.save()
+    print("repaired grafana-email-scope -> returns {email, email_verified}")
+else:
+    print("grafana-email-scope expression already correct")
+
+# 3. The id_token must carry the scope claims; Grafana reads identity from it.
+if not p.include_claims_in_id_token:
+    p.include_claims_in_id_token = True
+    p.save()
+    print("enabled include_claims_in_id_token")
+else:
+    print("include_claims_in_id_token already on")
+PYEOF
+
+echo "Done. Verify: sign in at https://grafana.franksdank.cloud via Authentik;"
+echo "the round trip must land on / and /api/user must answer 200 with an email."

scripts/authentik-set-login-session-duration.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Set the authentik login-session TTL on Dankclaw.
+#
+# The login-session lifetime is governed by the `default-authentication-login`
+# stage's `session_duration`, NOT a global setting and NOT SESSION_COOKIE_AGE
+# (that is the pre-login session, sessions.unauthenticated_age). Authentik does
+# not expose SESSION_SAVE_EVERY_REQUEST at all, so the session is an absolute
+# window from login, not a sliding one.
+#
+# The field is a STRING in authentik's own idiom ("days=7", "seconds=30",
+# "hours=12") — not a Python timedelta. Writing str(timedelta(days=7)) stores
+# "7 days, 0:00:00", which authentik's timedelta_from_string parser REJECTS
+# (ValueError), and login then crashes computing the session expiry. Always pass
+# the "key=value" form; this script validates it before writing.
+#
+# Existing sessions keep their old expiry; the new TTL applies from the next
+# login. This is DB state — it is NOT repo-managed and a blueprint re-import or
+# rebuild-from-scratch reverts it. Run this again after such a rebuild.
+#
+# Usage: authentik-set-login-session-duration.sh ["days=7"]   (default days=7)
+set -euo pipefail
+
+DUR="${1:-days=7}"
+STAGE="default-authentication-login"
+HOST="${AUTHENTIK_HOST:-dankclaw}"
+
+# Refuse anything that is not the safe key=value duration idiom.
+if ! printf '%s' "$DUR" | grep -Eq '^[A-Za-z]+=[0-9]+([ ]+[A-Za-z]+=[0-9]+)*$'; then
+    echo "invalid duration '$DUR' — use authentik form, e.g. 'days=7' or 'seconds=30'" >&2
+    exit 2
+fi
+
+ssh "$HOST" 'podman exec -i authentik-server ak shell' <<PYEOF
+from django.apps import apps
+from authentik.lib.utils.time import timedelta_from_string
+dur = "$DUR"
+timedelta_from_string(dur)  # validate the form before touching the row
+ULS = apps.get_model("authentik_stages_user_login", "UserLoginStage")
+s = ULS.objects.get(name="$STAGE")
+s.session_duration = dur
+s.save(update_fields=["session_duration"])
+back = ULS.objects.get(name="$STAGE").session_duration
+assert back == dur, "persisted value mismatch: %r" % back
+print("session_duration set to", repr(back), "->", timedelta_from_string(back))
+PYEOF
+
+echo "Login-session TTL for '$STAGE' is now $DUR (applies on next login)."

scripts/autoresearch-frank-archon.sh

@@ -29,10 +29,10 @@
 # The scratch mirrors the production container contract exactly (serve-frank-
 # primary.sh): --network host, --init, seccomp=unconfined (io_uring for the
 # n-gram table), memlock unbounded (the ~132 GiB pinned ledger), /dev/kfd +
-# /dev/dri, the .rad ro, the state dir rw, the tune cache mount. The ONLY
-# differences: the container name (arbench, so prod's identity is untouched)
-# and a FRESH prefix-cache disk tier (arbench/, wiped per run) so the cold
-# prefill is cold — the prod tier at kvcache/frank-archon is left intact.
+# /dev/dri, the .rad ro, the tune cache mount. The ONLY differences: the
+# container name (arbench, so prod's identity is untouched) and the prefix cache
+# switched OFF (--prefix-cache-mib 0), so every prefill is cold — the tier is
+# host RAM now, there is no per-run disk directory to wipe.
 set -uo pipefail
 
 ENV_FILE="${1:-/home/dank/serverAppConfigs/scripts/frank-archon.env}"
@@ -60,9 +60,8 @@ set -a; . "$ENV_FILE"; set +a
 sudo systemctl stop frank-archon >/dev/null 2>&1 || true
 podman rm -f "$SCRATCH" >/dev/null 2>&1 || true
 
-# --- fresh prefix-cache tier for a cold prefill (prod's tier untouched) ------
-FRESH_KV="$FRANK_ARCHON_STATE_DIR/kvcache/$SCRATCH"
-rm -rf "$FRESH_KV"; mkdir -p "$FRESH_KV"
+# --- the prefix cache is switched off below, so there is no scratch KV tree ---
+# to wipe; the cold prefill comes from --prefix-cache-mib 0.
 mkdir -p "$FRANK_ARCHON_STATE_DIR/tune"
 
 # --- boot the scratch: the production run line, scratch name + fresh KV ------
@@ -73,7 +72,6 @@ podman run -d --name "$SCRATCH" \
   --ulimit memlock=-1:-1 \
   --device /dev/kfd --device /dev/dri \
   -v "$FRANK_ARCHON_MODEL_DIR":/models:ro \
-  -v "$FRANK_ARCHON_STATE_DIR":/data \
   -v "$FRANK_ARCHON_STATE_DIR/tune":/opt/radiance/share/radiance/tune \
   -v "$FRANK_ARCHON_TEMPLATE":/templates/template.jinja:ro \
   "$FRANK_ARCHON_IMAGE" \
@@ -87,12 +85,9 @@ podman run -d --name "$SCRATCH" \
   --placement "$FRANK_ARCHON_PLACEMENT" \
   --host-pool-mib "$FRANK_ARCHON_HOST_POOL_MIB" \
   --gpu-headroom-mib "$FRANK_ARCHON_GPU_HEADROOM_MIB" \
-  --vram-weights-mib "$FRANK_ARCHON_VRAM_WEIGHTS_MIB" \
-  --vram-kv-mib "$FRANK_ARCHON_VRAM_KV_MIB" \
   --kv-cache-dtype "$FRANK_ARCHON_KV_DTYPE" \
-  --prefix-cache-host-mib "$FRANK_ARCHON_PREFIX_HOST_MIB" \
-  --prefix-cache-dir "/data/kvcache/$SCRATCH" \
-  --prefix-cache-disk-mib "$FRANK_ARCHON_PREFIX_DISK_MIB" \
+  --prefix-cache-mib 0 \
+  --busy-prefill-chunk "${FRANK_ARCHON_BUSY_PREFILL_CHUNK:-auto}" \
   --ngram-placement "$FRANK_ARCHON_NGRAM_PLACEMENT" \
   --num-speculative-tokens "$FRANK_ARCHON_SPEC_TOKENS" \
   --max-num-batched-tokens "$FRANK_ARCHON_MAX_BATCHED_TOKENS" \

scripts/axe-measurements.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""Destroy measurement ledgers in the surviving pages.
+
+Point of use (AGENTS.md → Documentation Policy): a rate/throughput figure for
+anything not running and not being compared is destroyed, not filed. This pass
+deletes
+
+  1. any section whose heading is a bench/throughput/rate/ladder/verdict ledger,
+  2. any table row or standalone bullet that REPORTS a rate (`123.4 t/s`,
+     `tok/s`, `ms/token`, `TTFT`, acceptance %, `llama-bench`, a bench arm),
+
+and it never touches a shape number (ctx, slots, MiB budgets, ports) or a bullet
+under a *Laws* / *Wire contract* / *Rollback* heading, where the sentence is the
+payload and the number is decoration. Dry run by default; `--apply` writes.
+
+Usage: axe-measurements.py [--apply] [--verbose]
+"""
+import pathlib
+import re
+import sys
+
+VAULT = pathlib.Path("serverAppConfigs-docs")
+RATE = re.compile(
+    r"\d[\d,.]*\s*(t/s|tok/s|tokens?/s|tok s|tps|ms/token|µs/token|GB/s|GiB/s)\b|"
+    r"\b(llama-bench|bench (arm|ladder|window)|decode ladder|prefill ladder)\b|"
+    r"\bacceptance (rate|of)\b|\bdraft_n\b|\bttft\b|\bServer-Timing\b",
+    re.I,
+)
+# A heading is a ledger only when the noun LEDS it and nothing instructional
+# follows: `## Throughput` yes, `## 6. The context ladder (find the max bootable
+# window by measurement)` no — that one is the method, which is point-of-use.
+LEDGER_HEAD = re.compile(
+    r"^#{2,4}\s+(?:\d+[.) ]\s*)?(bench(es|mark|marks)?|throughput|results?|timings?|"
+    r"measured (figures|numbers|results|rates|rates)|rates|verdict|acceptance|ladder)"
+    r"\b[^|]{0,40}$",
+    re.I,
+)
+INSTRUCTIONAL = re.compile(r"how|why|law|policy|method|procedure|find|by measurement|execution|flip|checklist", re.I)
+PROTECTED = re.compile(
+    r"^#{2,4}\s+.*(law|wire|rollback|contract|shape|eras|files|health|see also|related|"
+    r"boot|preflight|environment)",
+    re.I,
+)
+def blank_rate_cells(text):
+    """Kill throughput COLUMNS wherever they appear: a cell that reports a rate
+    becomes an em-dash, so the row keeps whatever else it records (the payload
+    name, the reason, the rollback) and loses only the dead number."""
+    out = []
+    for line in text.split("\n"):
+        st = line.strip()
+        if st.startswith("|") and st.endswith("|") and RATE.search(line):
+            cells = line.split("|")
+            for i, c in enumerate(cells):
+                if RATE.search(c):
+                    cells[i] = " — "
+            line = "|".join(cells)
+        out.append(line)
+    return "\n".join(out)
+
+
+def collapse_eras(text):
+    """One Eras row per era is fleet law; the NUMBERS in the old rows are not.
+    Keep the current era verbatim, fold everything older into one row that says
+    the numbers were destroyed and where the rollback actually lives (git)."""
+    lines = text.split("\n")
+    out = []
+    in_eras = False
+    ncol = 0
+    folded = 0
+    for line in lines:
+        if line.startswith("#"):
+            in_eras = bool(re.match(r"^#{2,4}\s+.*eras\b", line, re.I))
+            if folded:
+                out.append("| " + " | ".join(
+                    [f"≤ 2026-09 — {folded} earlier eras: payloads, shapes and every measured "
+                         "number destroyed 2026-10-19 as obsolete"] + ["—"] * max(0, ncol - 1)))
+                folded = 0
+            out.append(line)
+            continue
+        st = line.strip()
+        if in_eras and st.startswith("|"):
+            cells = [c.strip() for c in st.strip("|").split("|")]
+            if set("".join(cells)) <= set(":- "):
+                ncol = len(cells)
+                out.append(line)
+                continue
+            if ncol == 0:
+                ncol = len(cells)
+            if re.search(r"2026-1[01]|current|active|running", st, re.I):
+                out.append(line)
+            else:
+                folded += 1
+            continue
+        if folded and not st.startswith("|"):
+            out.append("| " + " | ".join(
+                [f"≤ 2026-09 — {folded} earlier eras: payloads, shapes and every measured number "
+                     "destroyed 2026-10-19 as obsolete"] + ["—"] * max(0, ncol - 1)))
+            folded = 0
+            ncol = 0
+        out.append(line)
+    if folded:
+        out.append("| " + " | ".join(
+            [f"≤ 2026-09 — {folded} earlier eras destroyed 2026-10-19"] + ["—"] * max(0, ncol - 1)))
+    return "\n".join(out)
+
+
+def main(argv):
+    apply = "--apply" in argv
+    verbose = "--verbose" in argv
+    total_lines = 0
+    pages = 0
+    for page in sorted(VAULT.rglob("*.md")):
+        s = str(page)
+        if ".AGENTTRASH" in s or ".obsidian" in s:
+            continue
+        text = page.read_text(errors="replace")
+        before = len(text.split("\n"))
+        if s.startswith("serverAppConfigs-docs/LLM/"):
+            text = collapse_eras(blank_rate_cells(text))
+        lines = text.split("\n")
+        out = []
+        skip = False
+        under_protected = False
+        cut = 0
+        for line in lines:
+            if line.startswith("#"):
+                under_protected = bool(PROTECTED.search(line))
+                skip = (bool(LEDGER_HEAD.match(line)) and not under_protected
+                        and not INSTRUCTIONAL.search(line))
+                if skip:
+                    cut += 1
+                    continue
+            if skip:
+                cut += 1
+                continue
+            stripped = line.strip()
+            out.append(line)
+            if under_protected:
+                # Eras rows are the rollback record, Law bullets are the payload:
+                # the number is decoration inside them, and the row/bullet is not
+                # a ledger entry. Leave the whole section alone.
+                continue
+            if stripped.startswith(("|", "-", "*")) and RATE.search(line):
+                # Destroy only when the line EXISTS to report a number: a table
+                # row, or a bullet whose verb is measurement.
+                if stripped.startswith("|") or re.search(
+                        r"\b(measured|measures|bench|benched|ran at|runs at|clocked|hit|"
+                        r"landed at|came in at|peak|peaks at|sustained|wall.clock)\b", line, re.I):
+                    out.pop()
+                    cut += 1
+        final = re.sub(r"\n{3,}", "\n\n", "\n".join(out))
+        saved = before - len(final.split("\n"))
+        if saved or cut:
+            pages += 1
+            total_lines += max(saved, cut)
+            print(f"  {s.replace('serverAppConfigs-docs/', '')}: -{max(saved, cut)}")
+            if apply:
+                page.write_text(final.rstrip("\n") + "\n")
+    print(f"{'APPLIED' if apply else 'dry run'}: {pages} pages, {total_lines} lines of ledger")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

scripts/build-qwen38-bf16-rad.sh

@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+# build-qwen38-bf16-rad.sh — RUNS ON DANKSERVER. rad-convert the vendor BF16
+# checkpoint /home/dank/models/Qwen3.8-Flash-Next into
+# /home/dank/models/rad/qwen38-bf16.rad with NO recipe and NO --reuse: a weight
+# no rule matches keeps the checkpoint's encoding, so every tensor stays the
+# bf16 it is in the safetensors and the container is the checkpoint's own
+# numbers behind a .rad header.
+#
+# This file is the campaign's measuring stick, twice over: scripts/
+# run-qwen38-bf16.sh kldref records the KLD reference logits from it (every
+# arm's --kld-ref is scored against that record) and the same container serves
+# with the calibration tap to write the Grams that derive the protected-expert
+# list and feed GPTQ. A reference that is not the vendor checkpoint, byte for
+# byte in bf16, silently moves every gate in the campaign — hence no recipe, no
+# --reuse, and the four refusal gates below.
+#
+# Why a container at all: a checkpoint dir has no mapped/file tiers, and a
+# weight only a host kernel reads (the 47.7 GiB PLE n-gram table) cannot be
+# served from one; rad-convert also bakes the tokenizer. The convert runs on
+# the HOST backend — no --device /dev/kfd, no /dev/dri — so declare resolves
+# kernels on CPU and production serving is untouched for the whole run (the
+# precedent that proved it: scripts/convert-swift15-w4dn8.sh).
+#
+# Gates, every one a refusal with exit 2 and a named reason, BEFORE anything is
+# written, cheapest first — the verifier runs last of them because its Hub gate
+# hashes 343 GiB and must not be paid for by a run that was going to refuse on
+# the output file anyway.
+#   checkpoint  the tree must be on disk; an absent dir names the command that
+#               makes one (scripts/download-qwen38-bf16.sh --fetch) and stops.
+#   free space  the convert writes ~343,323 MiB next to the ~343,323 MiB
+#               checkpoint; QWEN38_MIN_FREE_MIB (default 350000 = that figure
+#               plus 6.5 GiB of slack) is the floor. The volume that cannot
+#               hold both is discovered at minute 2, not hour 5.
+#   output      the campaign converts the reference ONCE. An existing
+#               qwen38-bf16.rad is refused; --force retires it to
+#               ~/.AGENTTRASH/<utc>-qwen38-bf16.rad (trash law: nothing is
+#               deleted, and the move is a rename on the same volume).
+#   verified    scripts/download-qwen38-bf16.sh's verify mode must pass. A
+#               failed verify exits 1, not 2: the tree being the wrong payload
+#               is a result, a missing environment is a refusal.
+#   plan        rad-convert --plan-only -v first — the oracle for "does this
+#               shape boot". A refused plan costs seconds and saves a convert.
+#
+# usage: scripts/build-qwen38-bf16-rad.sh [--dry-run] [--plan-only] [--force]
+#   --dry-run    print the argv it would exec (podman wrapping rad-convert),
+#               one shell-quoted argument a line, and exit 0. No gate runs, no
+#               container starts, nothing is written — the offline test seam.
+#               It does resolve the checkpoint and output paths through the
+#               /models mount, because an argv that carries host paths would
+#               not run either.
+#   --plan-only  run the plan and every gate except the disk and output ones —
+#               those two protect bytes the plan never writes, and this is the
+#               probe to run when the disk gate itself is what refuses
+#   --force      retire an existing output container to the trash dir first
+#
+# knobs: FRANK_QWEN38_MODELS   the host dir mounted at /models (default
+#        /home/dank/models); FRANK_QWEN38_CKPT (default $MODELS/
+#        Qwen3.8-Flash-Next, the same knob the verifier reads);
+#        FRANK_QWEN38_BF16_OUT (default $MODELS/rad/qwen38-bf16.rad);
+#        QWEN38_MIN_FREE_MIB; QWEN38_TRASH (default $HOME/.AGENTTRASH). The
+#        verifier's own census knobs (FRANK_QWEN38_SHARDS / _BYTES / _REPO)
+#        reach it through the environment untouched.
+# exit: 0 converted (or planned), 1 the checkpoint failed verification,
+#       2 refused before anything was written
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# ONE owner of the engine pin: the digest frank-archon serves on. The reference
+# must be built by the SAME engine build that scores it, or a KLD delta is
+# partly a version delta.
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/frank-archon.env"
+IMAGE="${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}"
+
+MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}"
+CKPT="${FRANK_QWEN38_CKPT:-$MODELS/Qwen3.8-Flash-Next}"
+OUT="${FRANK_QWEN38_BF16_OUT:-$MODELS/rad/qwen38-bf16.rad}"
+TRASH="${QWEN38_TRASH:-$HOME/.AGENTTRASH}"
+MIN_FREE_MIB="${QWEN38_MIN_FREE_MIB:-350000}"
+LOG="/home/dank/logs/qwen38_bf16_convert.log"
+
+DRY=0
+PLAN_ONLY=0
+FORCE=0
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --dry-run)   DRY=1 ;;
+        --plan-only) PLAN_ONLY=1 ;;
+        --force)     FORCE=1 ;;
+        *) echo "usage: ${0##*/} [--dry-run] [--plan-only] [--force]" >&2
+           exit 2 ;;
+    esac
+    shift
+done
+
+# The convert sees the tree only through the /models mount, so both paths must
+# be under it; the container-side spelling is what goes into the argv.
+case "$CKPT" in
+    "$MODELS"/*) CKPT_IN="/models/${CKPT#"$MODELS"/}" ;;
+    *) echo "REFUSED: checkpoint $CKPT is not under the mounted models dir $MODELS — the convert cannot see it" >&2
+       exit 2 ;;
+esac
+case "$OUT" in
+    "$MODELS"/*) OUT_IN="/models/${OUT#"$MODELS"/}" ;;
+    *) echo "REFUSED: output $OUT is not under the mounted models dir $MODELS — rad-convert could not write it" >&2
+       exit 2 ;;
+esac
+
+# Declare with NO recipe line — that is what keeps every tensor bf16 — and at the
+# bound this container will actually be served at. Both flags come off the pinned
+# image's own help: --kv-cache-dtype at convert "only picks the kernels it
+# checks" and the container is identical either way, so naming fp8 is free
+# insurance — if the fp8 KV kernels the fleet serves would not resolve, the
+# convert refuses here instead of the boot failing on a drained window.
+# --max-ctx is the bound declare is run at, whose default is the checkpoint's
+# training context (262,144 here, whose fp8 cache is 258.8 GiB against a 192 GiB
+# machine, so the default is not an option). 32,768 is set by the corpus, not by
+# taste: kld.jsonl's longest record is 38,263 characters — ~9,600 tokens at the
+# pessimistic chars/4, ~7,700 at the chars/4.97 this repo measured on the K3
+# corpus — so the window the rig boots at clears every record 2.6x over and the
+# reference boot cannot die on a sequence that will not fit. The rig's window is
+# one knob (FRANK_QWEN38_MAX_MODEL_LEN in scripts/run-qwen38-bf16.sh, default
+# 32,768) and that script refuses a corpus its window cannot hold, so raising one
+# without the other is caught before the drained window, not inside it. The
+# payload containers the gate scores declare at the training context because
+# production must hold the 200k shape; a declared bound is a declare-time check,
+# not a weight change, so the reference and the payloads carry the same numbers
+# whatever each declared at — which is why one can be measured against the other.
+CONVERT_ARGS=("$CKPT_IN" -o "$OUT_IN" --max-ctx 32768 --kv-cache-dtype fp8)
+PODMAN=(podman run --rm --entrypoint rad-convert -v "$MODELS:/models")
+PLAN_CMD=("${PODMAN[@]}" --name qwen38-bf16-plan "$IMAGE" "${CONVERT_ARGS[@]}" --plan-only -v)
+CONV_CMD=("${PODMAN[@]}" --name qwen38-bf16-convert "$IMAGE" "${CONVERT_ARGS[@]}")
+
+if [ "$DRY" = "1" ]; then
+    echo "dry run: no gate checked, no container started, nothing written"
+    echo "# $CKPT -> $OUT (the convert sees them as $CKPT_IN and $OUT_IN)"
+    echo "# plan argv:"
+    printf '%q\n' "${PLAN_CMD[@]}"
+    echo "# convert argv:"
+    printf '%q\n' "${CONV_CMD[@]}"
+    exit 0
+fi
+
+# ---- gate: the checkpoint is on disk ---------------------------------------
+if [ ! -d "$CKPT" ]; then
+    echo "REFUSED: no checkpoint at $CKPT — fetch and verify it first: scripts/download-qwen38-bf16.sh --fetch" >&2
+    exit 2
+fi
+
+# ---- gate: peak disk --------------------------------------------------------
+# The deepest existing ancestor is the right thing to ask: a fresh
+# /home/dank/models/rad has no free space of its own to report.
+OUT_DIR="$(dirname "$OUT")"
+PROBE="$OUT_DIR"
+while [ ! -d "$PROBE" ] && [ "$PROBE" != "/" ]; do PROBE="$(dirname "$PROBE")"; done
+FREE_MIB="$(df --output=avail -m "$PROBE" | tail -1 | tr -d ' ')"
+if ! [[ "$FREE_MIB" =~ ^[0-9]+$ ]]; then
+    echo "REFUSED: cannot read the free space under $PROBE (df said: $FREE_MIB)" >&2
+    exit 2
+fi
+if [ "$PLAN_ONLY" != "1" ] && [ "$FREE_MIB" -lt "$MIN_FREE_MIB" ]; then
+    echo "REFUSED peak-disk: $FREE_MIB MiB free under $PROBE, $MIN_FREE_MIB MiB wanted (QWEN38_MIN_FREE_MIB) — the bf16 container rewrites the checkpoint's own 343,323 MiB of tensors beside it" >&2
+    exit 2
+fi
+
+# ---- gate: one conversion ---------------------------------------------------
+if [ -e "$OUT" ]; then
+    if [ "$PLAN_ONLY" = "1" ]; then
+        echo "note: $OUT already exists; --plan-only writes nothing, so it stays"
+    elif [ "$FORCE" = "1" ]; then
+        mkdir -p "$TRASH"
+        DEST="$TRASH/$(date -u +%Y%m%dT%H%M%SZ)-$(basename "$OUT")"
+        mv "$OUT" "$DEST"
+        echo "TRASHED: retired the existing reference container $OUT -> $DEST"
+    else
+        echo "REFUSED: $OUT already exists — the campaign converts the reference once; re-run with --force to retire it to $TRASH first" >&2
+        exit 2
+    fi
+fi
+
+# ---- gate: the tree really is the checkpoint (the full verify) -------------
+echo "== verifying $CKPT (scripts/download-qwen38-bf16.sh, full gates)"
+rc=0
+bash "$SCRIPT_DIR/download-qwen38-bf16.sh" --dir "$CKPT" || rc=$?
+if [ "$rc" -ne 0 ]; then
+    if [ "$rc" -eq 2 ]; then
+        echo "REFUSED: the checkpoint verifier refused the environment (rc=2); nothing was converted" >&2
+        exit 2
+    fi
+    echo "REFUSED: $CKPT failed verification (rc=$rc) — a half tree converts to a half reference" >&2
+    exit 1
+fi
+
+# ---- plan, then convert ------------------------------------------------------
+mkdir -p "$(dirname "$LOG")"
+# rad-convert opens -o during declare as well, so the output directory exists
+# before the plan; that empty directory is all this creates ahead of it.
+mkdir -p "$OUT_DIR"
+echo "== rad-convert --plan-only -v (declare + plan; writes nothing)"
+"${PLAN_CMD[@]}" 2>&1 | tee -a "$LOG"
+rc=${PIPESTATUS[0]}
+if [ "$rc" -ne 0 ]; then
+    echo "REFUSED plan: rad-convert --plan-only exited rc=$rc — the engine refuses this encoding/shape, no container was written (log: $LOG)" >&2
+    exit 2
+fi
+if [ "$PLAN_ONLY" = "1" ]; then
+    echo "PLAN_OK: $CKPT_IN declares for $OUT_IN; no container written (log: $LOG)"
+    exit 0
+fi
+
+echo "== convert: $CKPT -> $OUT ($(date -Is))"
+"${CONV_CMD[@]}" 2>&1 | tee -a "$LOG"
+rc=${PIPESTATUS[0]}
+if [ "$rc" -ne 0 ]; then
+    echo "rad-convert rc=$rc; log: $LOG" >&2
+    exit "$rc"
+fi
+MIB=$(( $(stat -c %s "$OUT") / 1048576 ))
+echo "BUILD_DONE: $OUT is ${MIB} MiB (log: $LOG)"
+echo "NEXT: scripts/run-qwen38-bf16.sh kldref records the reference logits from it"

scripts/calib-push.py

@@ -22,6 +22,17 @@ Mechanics notes:
   prompts do not bias the Hessians (the corpus is drawn round-robin over
   source classes, so loss is class-neutral as long as failures are rare;
   a systematic 4xx is loud in the log and must be fixed, not ignored).
+- a round that seals no tap bytes is NORMAL and continues: RADIANCE_CALIB_TOKENS
+  is a pooled floor and this campaign's corpus is 10.03M tokens against a 12M
+  floor, so round 1 seals nothing by construction. A tap that LOSES bytes fails
+  the run, and a run that never sees the engine's completion line exits non-zero.
+  The first version failed on a flat round, which aborted exactly one round too
+  early (2026-10-29).
+- before any prompt, the tap must carry the rig's own marker
+  (`--tap-marker`, written by run-qwen38-bf16.sh calib) naming THIS
+  container, and that container must be running. Measured 2026-10-29:
+  with neither check, four rounds of 10M tokens went into restored
+  production after the rig's boot had died, and recorded nothing.
 
 Usage: calib-push.py [--port 7770] [--parallel 64] [--container arbench]
        [--corpus /home/dank/corpora/out/calib.jsonl] [--max-rounds 4]
@@ -29,11 +40,18 @@ Usage: calib-push.py [--port 7770] [--parallel 64] [--container arbench]
 import argparse
 import concurrent.futures
 import json
+import os
 import subprocess
 import sys
 import time
 import urllib.request
 
+_HERE = os.path.dirname(os.path.abspath(__file__)) if "__file__" in globals() else os.getcwd()
+if _HERE not in sys.path:
+    sys.path.insert(0, _HERE)
+
+import qwen38_common as qc  # noqa: E402 - sibling module, same directory
+
 DONE_MARK = "calibration wrote"
 
 
@@ -49,9 +67,9 @@ def tap_done(container):
     return DONE_MARK in (out.stdout + out.stderr)
 
 
-def push_one(url, prompt):
+def push_one(url, prompt, model):
     body = json.dumps({
-        "model": "frank-archon",
+        "model": model,
         "prompt": prompt,
         "max_tokens": 1,
         "temperature": 0,
@@ -68,38 +86,151 @@ def push_one(url, prompt):
         return False
 
 
-def main():
+def push_round(url, prompts, model, parallel):
+    """Push the whole corpus once; returns how many requests were accepted."""
+    ok = 0
+    with concurrent.futures.ThreadPoolExecutor(parallel) as ex:
+        for good in ex.map(lambda p: push_one(url, p, model), prompts):
+            ok += 1 if good else 0
+    return ok
+
+
+def tap_bytes(path):
+    """Bytes the tap has sealed so far; 0 when the directory is absent.
+
+    A tap whose directory is wrong, full or read-only stops writing while the
+    engine keeps answering 200 — the driver then re-pushes the whole corpus for
+    every round it is allowed and the loss is invisible until the hours are gone.
+    """
+    total = 0
+    try:
+        for root, _, files in os.walk(path):
+            for f in files:
+                try:
+                    total += os.path.getsize(os.path.join(root, f))
+                except OSError:
+                    pass
+    except OSError:
+        return 0
+    return total
+
+
+def round_verdict(before, after, done):
+    """What one round's tap bytes mean.
+
+    'done'   the engine logged that it sealed the tap — the run is over
+    'shrink' the tap LOST bytes: reset, pruned or replaced under us
+    'grew'   sealed bytes this round
+    'flat'   sealed nothing — which is legitimate: RADIANCE_CALIB_TOKENS is a
+             pooled floor, and a round that does not reach it has nothing to
+             write yet (the campaign's corpus is 10.03M tokens against a 12M
+             floor, so round 1 is always flat). Failing on it aborted the run
+             exactly one round too early.
+    """
+    if done:
+        return "done"
+    if after < before:
+        return "shrink"
+    if after == before:
+        return "flat"
+    return "grew"
+
+
+def require_rig(a):
+    """Refuse unless the rig that opened this tap is up and serving right now.
+
+    Three facts, in order: the runner wrote its marker into the tap dir, the
+    marker names the container this pusher targets, and that container is
+    running. A marker without a live container is a boot that died; a live
+    container without a marker is some other engine on the port — on 2026-10-29
+    it was restored production, which took four rounds of 10M tokens before the
+    old driver admitted it had recorded nothing.
+    """
+    marker = os.path.join(a.tap_dir, a.tap_marker)
+    if not os.path.isfile(marker):
+        qc.refuse(f"no rig marker at {marker}: this tap was not opened by "
+                  "run-qwen38-bf16.sh calib, so a push here would feed whatever "
+                  "answers the port and record nothing")
+    try:
+        lines = open(marker, encoding="utf-8", errors="replace").read().splitlines()
+    except OSError as exc:
+        qc.refuse(f"cannot read the rig marker {marker}: {exc}")
+    meta = dict(line.split("=", 1) for line in lines if "=" in line)
+    who = meta.get("container", "")
+    if who != a.container:
+        qc.refuse(f"the rig marker names container {who!r} and this pusher targets "
+                  f"{a.container!r} — wrong tap, or a stale marker")
+    try:
+        p = subprocess.run(["podman", "inspect", "-f", "{{.State.Running}}", a.container],
+                           capture_output=True, text=True, timeout=30)
+    except Exception as exc:
+        qc.refuse(f"cannot inspect the rig container {a.container}: {exc}")
+    if p.returncode != 0 or p.stdout.strip() != "true":
+        qc.refuse(f"the rig container {a.container} is not running — whatever is "
+                  "answering the port, it is not the calibration rig")
+
+
+def parse_args():
     ap = argparse.ArgumentParser()
     ap.add_argument("--port", type=int, default=7770)
     ap.add_argument("--parallel", type=int, default=64)
     ap.add_argument("--container", default="arbench")
+    ap.add_argument("--model", default="frank-archon")
+    ap.add_argument("--tap-dir", default="/home/dank/campaign/qwen38-w5nl64/calib")
+    ap.add_argument("--tap-marker", default=".rig-boot",
+                    help="the file run-qwen38-bf16.sh calib writes into the tap dir; without it the pusher refuses, because an unmarked tap is not a rig")
     ap.add_argument("--corpus", default="/home/dank/corpora/out/calib.jsonl")
     ap.add_argument("--max-rounds", type=int, default=4)
-    a = ap.parse_args()
-    url = f"http://127.0.0.1:{a.port}/v1/completions"
+    return ap.parse_args()
 
+
+def load_prompts(a) -> list[str]:
+    if not os.path.isfile(a.corpus):
+        qc.refuse(f"no corpus at {a.corpus}")
     prompts = []
-    with open(a.corpus) as f:
-        for line in f:
-            line = line.strip()
-            if line:
-                prompts.append(json.loads(line)["prompt"])
-    print(f"corpus: {len(prompts)} prompts", flush=True)
+    for lineno, rec in qc.iter_jsonl(a.corpus):
+        if not isinstance(rec.get("prompt"), str):
+            qc.refuse(f"{a.corpus}:{lineno} has no string 'prompt' field")
+        prompts.append(rec["prompt"])
+    if not prompts:
+        qc.refuse(f"the corpus at {a.corpus} holds no prompts")
+    return prompts
+
+
+def main():
+    a = parse_args()
+    url = f"http://127.0.0.1:{a.port}/v1/completions"
+    prompts = load_prompts(a)
+    require_rig(a)
+    chars = sum(len(p) for p in prompts)
+    print(f"corpus: {len(prompts)} prompts, ~{chars // 4} tokens (chars/4)",
+          flush=True)
 
     for rnd in range(1, a.max_rounds + 1):
         if tap_done(a.container):
             break
+        before = tap_bytes(a.tap_dir)
         t0 = time.time()
-        ok = 0
-        with concurrent.futures.ThreadPoolExecutor(a.parallel) as ex:
-            for good in ex.map(lambda p: push_one(url, p), prompts):
-                ok += 1 if good else 0
-        dt = time.time() - t0
+        ok = push_round(url, prompts, a.model, a.parallel)
+        dt = max(time.time() - t0, 1e-9)
         print(f"round {rnd}: {ok}/{len(prompts)} ok in {dt:.0f}s "
-              f"({sum(len(p) for p in prompts)/4/dt:.0f} chars/s est)",
-              flush=True)
+              f"({chars / 4 / dt:.0f} tokens/s est)", flush=True)
+        if ok == 0:
+            print("FAIL: not one prompt was accepted; fix the engine or the argv "
+                  "before spending another round", flush=True)
+            sys.exit(1)
         if tap_done(a.container):
             break
+        after = tap_bytes(a.tap_dir)
+        verdict = round_verdict(before, after, False)
+        if verdict == "shrink":
+            print(f"FAIL: the tap at {a.tap_dir} LOST bytes this round "
+                  f"({before} B before, {after} B after) — it was reset or replaced "
+                  "under us", flush=True)
+            sys.exit(1)
+        if verdict == "flat":
+            print(f"round {rnd}: the tap sealed nothing yet (byte floor not reached; "
+                  f"{before} B) — continuing", flush=True)
     else:
         print("max rounds exhausted, tap not done", flush=True)
         sys.exit(1)

scripts/census-frank-dragoon.sh

@@ -21,7 +21,7 @@
 # deployed knob has since dropped to 262144.
 set -euo pipefail
 
-IMG=ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a
+IMG=ghcr.io/peonist-ai/halogen-flash-server@sha256:e35133b206da3094cc4d70c78800b26235bd9124b9aeb0e029d4a7c153b9cff0
 CUR=/home/dank/models/halogen-flashnext-v2
 NEW=/home/dank/models/halogen-v2-abliterated
 CUR_CK=qwen38-flash-next-v2-swift15-abliterated.hgn

scripts/check-okf.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+"""OKF v0.1 conformance linter for a docs bundle (default serverAppConfigs-docs).
+
+Checks the hard rules of the Open Knowledge Format spec plus the maintenance
+guidance, so "conforms to OKF" is a test and not a claim:
+
+  FRONTMATTER  every non-reserved .md has parseable YAML frontmatter with a
+               non-empty `type` from the spec's value set
+  TYPE         the value is one of reference / howto / tutorial / explanation
+  DESC         `description` is present, non-blank, not the title, <= 120 chars,
+               and does not restate a fleet quantity
+  TIMESTAMP    `timestamp` present, ISO 8601, not in the future
+  LENGTH       lines between 1 and 500 inclusive
+  INDEX-FM     no index.md may carry frontmatter (index.md is reserved)
+  LOG-DATE     log.md date headings are `## YYYY-MM-DD` or `## YYYY-MM-DD — Title`,
+               one entry per date, newest first
+  INDEX-COVER  every index.md links each of its sibling pages; the root index
+               links every page in the bundle
+  INDEX-NUM    an index table row restates a fleet quantity — a digit run of
+               three or more, a digits-plus-unit figure, a port, or a
+               multiplier. Names (Qwen3.6, RDNA4, RTX 3090, PICO-8, P4) are
+               not quantities and are allowed
+  CITATIONS    informational: pages with factual claims and no Citations
+               section are counted, never failed
+
+  LENGTH       lines between 1 and 500 inclusive. A page may opt out of the
+               ceiling with `okf-exempt: length` in its frontmatter: AGENTS.md
+               makes one page per endpoint permanent, and an endpoint page with
+               a full Eras table cannot split without creating a second owner
+               of the same shape (tests/docs_completeness.bats enforces that)
+
+Exit 0 when there are no violations.
+"""
+import datetime
+import re
+import sys
+from pathlib import Path
+
+TYPES = {"reference", "howto", "tutorial", "explanation"}
+RESERVED = {"index.md", "log.md", "AGENTS.md"}
+
+# A quantity, as opposed to a name: a comma-grouped figure, a run of 3+ digits,
+# a unit-bearing figure, a port (":7770"), or a multiplier ("2x", "3 *").
+QUANTITY = re.compile(
+    r"\d{1,3}(?:,\d{3})+"          # 262,144
+    r"|\d{3,}"                     # 200000, 7770, 3090
+    r"|\d+(?:\.\d+)?\s*(?:GiB|MiB|GB|MB|TB|tok/s|t/s|x\b|%)"
+    r"|[:×]\s*\d"
+    r"|\d+\s*[x×]\s*\d"
+)
+# Product / model / standard names that legitimately contain digits.
+NAMES = re.compile(
+    r"Qwen[\d.]*[-\w]*|Gemma\s?\d[\w.]*|MiMo[-\w.\d.]*|Ornith[-\w.\d]*"
+    r"|Bonsai[-\w.\d]*|Ling[-\w.\d.]*|DeepSeek[-\w.\d]*|GLM[-\w.\d]*"
+    r"|PICO-\d+|RDNA\s?\d+|R\d{4}|RTX\s?\d+|XDNA\d|gfx\d+|CUDA\s?[\d.]+"
+    r"|\d+\s?GbE|Wi-?Fi\s?\d|v?\d+\.\d+(?:\.\d+)?|b\d+|P\d\b|A\d+\b"
+    r"|frank-\w*2026-\d\d-\d\d|2026-\d\d-\d\d",
+    re.I,
+)
+DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}")
+FM_KEY = re.compile(r"^([A-Za-z][\w-]*):\s*(.*)$")
+
+
+def split_frontmatter(text):
+    if not text.startswith("---"):
+        return None, text
+    end = text.find("\n---", 3)
+    if end < 0:
+        return None, text
+    body_start = text.find("\n", end + 1)
+    return text[3:end], text[body_start + 1 :] if body_start >= 0 else ""
+
+
+def fields(raw):
+    out = {}
+    if not raw:
+        return out
+    key = None
+    for line in raw.splitlines():
+        if not line.strip():
+            continue
+        m = FM_KEY.match(line)
+        if m:
+            key, val = m.group(1), m.group(2).strip().strip("\"'")
+            out[key] = val
+        elif line.startswith(" ") and key:
+            out[key] = (out[key] + " " + line.strip()).strip()
+    return out
+
+
+def strip_names(value):
+    return NAMES.sub(" ", value)
+
+
+def check_file(path: Path, root: Path, length_exempt=frozenset()):
+    v = []
+    rel = str(path.relative_to(root))
+    text = path.read_text(errors="replace")
+    lines = text.splitlines()
+    name = path.name
+    raw, body = split_frontmatter(text)
+    fm = fields(raw)
+
+    if name not in RESERVED:
+        if raw is None:
+            v.append(("FRONTMATTER", rel, "no parseable frontmatter"))
+        else:
+            t = fm.get("type", "")
+            if not t:
+                v.append(("TYPE", rel, "missing type"))
+            elif t not in TYPES:
+                v.append(("TYPE", rel, f"type '{t}' not in {sorted(TYPES)}"))
+            d = fm.get("description", "")
+            if not d:
+                v.append(("DESC", rel, "missing description"))
+            else:
+                if len(d) > 120:
+                    v.append(("DESC", rel, f"description is {len(d)} chars (max 120)"))
+                if d.strip("# ").lower() == fm.get("title", "\0").strip("# ").lower():
+                    v.append(("DESC", rel, "description equals title"))
+                if QUANTITY.search(strip_names(d)):
+                    v.append(("DESC", rel, "description restates a quantity"))
+            ts = fm.get("timestamp", "")
+            if not ts:
+                v.append(("TIMESTAMP", rel, "missing timestamp"))
+            else:
+                try:
+                    stamp = datetime.date.fromisoformat(ts[:10])
+                    if stamp > datetime.date.today():
+                        v.append(("TIMESTAMP", rel, f"timestamp {ts} is in the future"))
+                except ValueError:
+                    v.append(("TIMESTAMP", rel, f"unparseable timestamp {ts!r}"))
+
+    if name == "index.md" and raw is not None:
+        v.append(("INDEX-FM", rel, "index.md must not carry frontmatter"))
+
+    if name == "log.md":
+        dates = []
+        for line in lines:
+            m = re.match(r"^##\s+(\S.*?)\s*$", line)
+            if not m:
+                continue
+            head = m.group(1)
+            if not DATE_RE.match(head):
+                continue
+            dates.append(head[:10])
+            # OKF requires the heading to USE the date format; a title after an
+            # em-dash is allowed and costs zero lines, so `## 2026-08-17 — Why the
+            # pair was rebuilt` is conformant and `## 2026-08-17Rebuilt` is not.
+            if head != head[:10] and not re.match(r"^\d{4}-\d{2}-\d{2}\s+—\s+\S", head):
+                v.append(("LOG-DATE", rel,
+                          f"date heading must be the date alone or `date — Title`: '## {head}'"))
+        if len(dates) != len(set(dates)):
+            v.append(("LOG-DATE", rel, "duplicate date heading"))
+        if dates != sorted(dates, reverse=True):
+            v.append(("LOG-DATE", rel, "date headings not newest first"))
+
+    if raw is not None and name not in RESERVED:
+        n = len(lines)
+        body_lines = len([ln for ln in body.splitlines() if ln.strip()])
+        opted_out = "length" in fm.get("okf-exempt", "").lower()
+        if n > 500:
+            if rel in length_exempt:
+                v.append(("LENGTH-OK", rel,
+                          f"{n} lines — over the hint, pinned by the "
+                          "one-page-per-endpoint law"))
+            elif opted_out:
+                v.append(("LENGTH-OK", rel,
+                          f"{n} lines — over the hint, opted out by the page"))
+            else:
+                v.append(("LENGTH", rel, f"{n} lines (max 500)"))
+        elif body_lines < 15:
+            v.append(("LENGTH", rel, "stub: fewer than 15 body lines"))
+
+    # A second `type:` key inside the body means the page carries two copies of
+    # itself with a frontmatter block in the middle — the shape a naive
+    # frontmatter rewrite produces. It doubles the length, splits the wikilinks
+    # and makes the page's facts ambiguous, so it is a hard failure.
+    if raw is not None and name not in RESERVED:
+        dupes = [i + len((raw or "").splitlines()) + 2
+                 for i, ln in enumerate(body.splitlines())
+                 if re.match(r"^(type|title|description):", ln)]
+        if dupes:
+            v.append(("DOUBLED", rel,
+                      f"frontmatter keys inside the body at line(s) "
+                      f"{', '.join(str(d) for d in dupes[:5])} — the page "
+                      "contains a duplicated copy of itself"))
+
+    return v
+
+
+def index_rows(text):
+    rows = []
+    for line in text.splitlines():
+        if line.startswith("|") and "[[" in line:
+            rows.append(line)
+    return rows
+
+
+def check_links(root: Path):
+    v = []
+    pages = [
+        p for p in sorted(root.rglob("*.md"))
+        if ".AGENTTRASH" not in str(p) and ".obsidian" not in str(p)
+    ]
+    for index in [p for p in pages if p.name == "index.md"]:
+        text = index.read_text(errors="replace")
+        links = set(re.findall(r"\[\[([^\]|#]+)", text))
+        if index.parent == root:
+            targets = [p for p in pages if p.name not in RESERVED]
+        else:
+            targets = [p for p in index.parent.glob("*.md") if p.name != "index.md"]
+        for target in targets:
+            if target.stem not in links and target.name not in links:
+                v.append(("INDEX-COVER", str(index.relative_to(root)),
+                          f"does not link sibling {target.name}"))
+    for index in [p for p in pages if p.name == "index.md"]:
+        for row in index_rows(index.read_text(errors="replace")):
+            if QUANTITY.search(strip_names(row)):
+                v.append(("INDEX-NUM", str(index.relative_to(root)),
+                          "row restates a quantity: " + row[:80]))
+    return v
+
+
+def live_endpoints(root: Path):
+    """Pages pinned by AGENTS.md's one-page-per-endpoint law.
+
+    Derived, never hardcoded — the same derivation as
+    tests/docs_completeness.bats: an endpoint is live when scripts/ holds both
+    its env and its unit. Those pages carry the Eras table for their whole
+    history, so the OKF 500-line split hint cannot apply to them without
+    creating a second owner of the same shape.
+    """
+    scripts = root.parent / "scripts"
+    out = set()
+    if scripts.is_dir():
+        for envf in scripts.glob("frank-*.env"):
+            if (scripts / f"{envf.stem}.service").exists():
+                out.add(f"LLM/{envf.stem}.md")
+    return out
+
+
+def retired_era_pages(root: Path):
+    """Era pages of a retired endpoint: campaign ledgers, one owner each.
+
+    AGENTS.md lets a campaign ledger that exists nowhere else live on the page
+    that measured it, and a retired endpoint keeps exactly one page per era.
+    Splitting such a ledger to satisfy the 500-line hint would fragment one
+    campaign across two owners, so the tag opts the page out.
+    """
+    out = set()
+    for path in root.rglob("*.md"):
+        s = str(path)
+        if ".AGENTTRASH" in s or ".obsidian" in s:
+            continue
+        raw, _ = split_frontmatter(path.read_text(errors="replace"))
+        if raw is not None and "retired-endpoint" in fields(raw).get("tags", ""):
+            out.add(str(path.relative_to(root)))
+    return out
+
+
+def main(argv):
+    root = Path(argv[1] if len(argv) > 1 else "serverAppConfigs-docs")
+    exempt = frozenset(live_endpoints(root)) | retired_era_pages(root)
+    violations = []
+    notes = []
+    citation_gaps = 0
+    pages = 0
+    for path in sorted(root.rglob("*.md")):
+        s = str(path)
+        if ".AGENTTRASH" in s or ".obsidian" in s:
+            continue
+        pages += 1
+        for code, rel, detail in check_file(path, root, exempt):
+            (notes if code == "LENGTH-OK" else violations).append((code, rel, detail))
+        text = path.read_text(errors="replace")
+        if len(text.splitlines()) > 40 and not re.search(
+                r"^##\s+(Citations|Sources)\b", text, re.M):
+            citation_gaps += 1
+    violations += check_links(root)
+
+    for code, rel, detail in violations:
+        print(f"  {code:12s} {rel}: {detail}")
+    for code, rel, detail in notes:
+        print(f"  {code:12s} {rel}: {detail}")
+    print(f"{pages} concept pages, {len(violations)} OKF violations"
+          f" ({citation_gaps} pages without a Citations section — soft;"
+          f" {len(notes)} length exemptions honoured)")
+    return 1 if violations else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

scripts/check-vault-links.py

@@ -31,7 +31,12 @@ def strip_md(path: str) -> str:
 
 
 def build_index(vault: pathlib.Path):
-    pages = sorted(vault.rglob("*.md"))
+    # `.AGENTTRASH` is deleted-but-held state, not vault content: it must not be
+    # scanned (its relative links resolve against the trash dir, not the vault)
+    # and must not answer as a link target — a live page pointing into the trash
+    # is an unresolved link, not a resolved one.
+    pages = sorted(p for p in vault.rglob("*.md")
+                   if ".AGENTTRASH" not in str(p))
     stems: dict[str, list[pathlib.Path]] = {}
     for page in pages:
         stems.setdefault(page.stem.casefold(), []).append(page)

scripts/convert-qwen38-w5nl64.sh

@@ -0,0 +1,236 @@
+#!/usr/bin/env bash
+# convert-qwen38-w5nl64.sh — RUNS ON DANKSERVER. The CPU half of the Qwen3.8-Flash-Next five-bit
+# campaign: convert the vendor BF16 checkpoint into the campaign's arms, each one the whole
+# `w5nl64a8h` rung on BOTH expert projections with the protected experts kept bf16.
+#
+# Nothing here touches a GPU: the container is started without /dev/kfd or /dev/dri, so the
+# converter runs on the host backend and the fleet can keep serving through the whole script.
+# The engine pin is the digest that serves frank-archon, so a winning arm crosses no version
+# boundary between measurement and serving.
+#
+# Three arms, because the campaign's question is which search knob pays for its hours
+# (recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe):
+#
+#   cd3     the recipe as written — coordinate descent three sweeps. The control.
+#   cd6     six sweeps: the same search, twice the passes.
+#   ao1     act_order=1 instead of cd: GPTQ sweeps columns in activation-importance order and
+#           stores them permuted. This is a SEPARATE arm, not a fourth sweep — libquant refuses
+#           cd= together with act_order=1 ("cd= sweeps the columns in their stored order, and
+#           act_order=1 stores them permuted; one or the other", libquant/lq_registry.cpp), so
+#           the two knobs cannot be combined into a best-of-both container.
+#
+# The arms are generated from ONE template with `--quant` overrides, not three near-duplicate
+# recipes: --quant rules are inserted BEFORE the recipe file and the first rule whose glob matches
+# a weight decides it (core/format/recipe.cpp: Recipe::match), so an override cleanly replaces the
+# expert line and the other 60 lines cannot drift between arms.
+#
+# Protected experts: apply-protected-cast.sh derives the bf16 set from THIS campaign's grams
+# (--target-count 48, two per layer so each rank's 256-expert half stays even) and splices it into
+# a per-run COPY of the template; the committed recipe keeps its markers and stays reviewable.
+#
+# Gates, in the order they bite — each one is a lesson the swift15 campaign paid for:
+#   inputs        checkpoint (dir + index), recipe, grams, imatrix, tokenizer. A missing grams
+#                 directory converts UNCALIBRATED with one stderr note, which is silently worse
+#                 than failing here.
+#   reference     the 335.3 GiB BF16 reference container must be GONE before the converts start.
+#                 It and the checkpoint together eat the volume; its job (the KLD recording and
+#                 the calibration grams) is finished by then, and both survive on disk without the
+#                 container beside it. The FP8-native anchor is FETCHED after the converts (disk
+#                 sequencing), so nothing here waits on it.
+#   plan          --plan-only runs FIRST, per arm, and prints "about X would be written". The
+#                 number is the bill: free space must cover it plus the 128k-prompt overshoot
+#                 margin, and the finished artifact is checked against it.
+#   outputs       no arm overwrites anything; move the old one to trash.
+#
+# Usage: convert-qwen38-w5nl64.sh [--arm cd3|cd6|ao1]... [--dry-run] [--plan-only]
+#                                 [--target-count N] [--keep-recipe]
+set -u -o pipefail
+
+SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# THE TOOLCHAIN PIN. `:latest` on 2026-10-30 is v1.14.0 (revision 4a61d7296bc5),
+# the same image the tap producer boots: a tap must be converted by the line that
+# is current, and rad-convert's `gptq:calib=<dir>` input contract is version-stable
+# (RADGRAM2 + gram.r<rank>.<stem>.bin), so nothing else about the recipe moves with
+# it. Production stays on its own pin — this is the campaign's toolchain.
+IMAGE=${IMAGE:-stilldeadcode/radiance@sha256:69add58f7de1ed2bca2b1e20a4cc75717557df3f73569a178eca5aafe8c44093}
+CKPT=${CKPT:-/home/dank/models/Qwen3.8-Flash-Next}
+CAMPAIGN=${CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}
+RECIPE=${RECIPE:-$SELF/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe}
+CALIB=${CALIB:-$CAMPAIGN/calib}
+IMATRIX=${IMATRIX:-$CAMPAIGN/imatrix.gguf}
+OUTDIR=${OUTDIR:-/home/dank/models/rad}
+BF16_RAD=${BF16_RAD:-$OUTDIR/qwen38-bf16.rad}   # the reference container, not an arm output
+STATE=${STATE:-$CAMPAIGN}                      # the dir the container mounts read-only: the
+                                               # spliced recipe and the imatrix live here
+TARGET_COUNT=${TARGET_COUNT:-48}
+MIN_GRAMS=${MIN_GRAMS:-24576}                 # per-expert DOWN grams: 48 recorded layers x 512
+                                               # experts. The checkpoint has 49 MoE layers, but the
+                                               # MTP layer is never recorded, so its 512 experts are
+                                               # absent from the tap's output by design
+MARGIN_MIB=${MARGIN_MIB:-40960}               # the long-prompt overshoot, off the plan number
+TS=$(date +%Y%m%d-%H%M%S)
+
+ARMS=()
+DRY=0
+PLAN_ONLY=0
+KEEP_RECIPE=0
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+      --arm) [ "$#" -ge 2 ] || { echo "--arm needs a value (cd3|cd6|ao1)" >&2; exit 2; }
+             ARMS+=("$2"); shift 2 ;;
+      --dry-run) DRY=1; shift ;;
+      --plan-only) PLAN_ONLY=1; shift ;;
+      --target-count) [ "$#" -ge 2 ] || { echo "--target-count needs a value" >&2; exit 2; }
+             TARGET_COUNT="$2"; shift 2 ;;
+      --keep-recipe) KEEP_RECIPE=1; shift ;;
+      *) echo "unknown argument: $1" >&2; exit 2 ;;
+    esac
+done
+[ "${#ARMS[@]}" -gt 0 ] || ARMS=(cd3 cd6 ao1)
+for a in "${ARMS[@]}"; do
+    case "$a" in cd3|cd6|ao1) ;; *) echo "unknown arm: $a (cd3|cd6|ao1)" >&2; exit 2 ;; esac
+done
+
+# The expert rule's option set, spelled once. --quant replaces the template's line wholesale, so
+# every option has to be repeated here; `calib=$CALIB` is expanded by the recipe reader from the
+# container environment (core/format/recipe.cpp: expand refuses an unset name rather than taking
+# it for empty).
+expert_opts='table=w5nl,group=64,scale=fp8_e4m3,scale2=f32,block2=*x*,scale2_value=0.0001220703125,transform=fwht128,rule=search,calib=$CALIB'
+arm_quant() {
+    case "$1" in
+      # The control is the template itself: no override, so the recipe's own line decides.
+      cd3) return 0 ;;
+      cd6) echo "blk.*.ffn_*_exps.*.weight=gptq:${expert_opts},cd=6" ;;
+      # act_order=1 WITHOUT cd: libquant refuses the pair (lq_registry.cpp:105, "cd= sweeps the
+      # columns in their stored order, and act_order=1 stores them permuted; one or the other"), so
+      # this arm is the interleave knob alone, not cd=3 plus it.
+      ao1) echo "blk.*.ffn_*_exps.*.weight=gptq:${expert_opts},act_order=1" ;;
+    esac
+}
+
+# --- gates ------------------------------------------------------------------------------------
+[ -d "$CKPT" ] || { echo "no checkpoint at $CKPT; run scripts/download-qwen38-bf16.sh --fetch" >&2; exit 2; }
+[ -f "$CKPT/model.safetensors.index.json" ] || { echo "checkpoint at $CKPT has no safetensors index — a torn fetch" >&2; exit 2; }
+[ -f "$CKPT/tokenizer.json" ] || { echo "no tokenizer.json in $CKPT; the .rad cannot carry one without it" >&2; exit 2; }
+[ -f "$RECIPE" ] || { echo "no recipe at $RECIPE" >&2; exit 2; }
+GRAMS=$(find "$CALIB" -maxdepth 1 -name 'gram.r*.bin' 2>/dev/null | wc -l)
+[ "$GRAMS" -ge "$MIN_GRAMS" ] || {
+    echo "calib dir $CALIB holds $GRAMS gram files (<$MIN_GRAMS); run $0 after run-qwen38-bf16.sh calib" >&2; exit 2; }
+[ -f "$IMATRIX" ] || { echo "no imatrix at $IMATRIX; run scripts/grams-to-imatrix.py over $CALIB" >&2; exit 2; }
+# The container sees the campaign dir at one fixed mount point, so an imatrix kept elsewhere has
+# no path inside it: copy it in (or point IMATRIX at the campaign dir) rather than let the
+# converter fail halfway through a planned arm.
+[ "$(dirname "$IMATRIX")" = "$STATE" ] || {
+    echo "imatrix $IMATRIX is not in the campaign dir $STATE the container mounts; move it there" >&2; exit 2; }
+if [ -e "$BF16_RAD" ]; then
+    echo "the 335.3 GiB BF16 reference container $BF16_RAD is still on disk; its recording and" \
+         "the calibration grams are already taken, so move it to trash — the converts need its" \
+         "space (peak-disk law)" >&2
+    exit 2
+fi
+for a in "${ARMS[@]}"; do
+    [ -e "$OUTDIR/qwen38-w5nl64-$a.rad" ] && {
+        echo "$OUTDIR/qwen38-w5nl64-$a.rad already exists; move it to trash first" >&2; exit 2; }
+    # A killed run leaves its container behind (--rm only fires when podman itself
+    # exits), and the next arm then dies on "name already in use" AFTER the plan
+    # phase; refuse here, where the fix is one `podman rm -f`.
+    podman container exists "qwen38-convert-$a" && {
+        echo "container qwen38-convert-$a still exists (a killed run?);" \
+             "'podman rm -f qwen38-convert-$a' and re-run" >&2
+        exit 2; }
+done
+
+# --- the protected plane, spliced into a per-run copy of the template --------------------------
+mkdir -p "$STATE"
+WORK="$STATE/recipe-$TS.recipe"
+cp "$RECIPE" "$WORK"
+if [ "$DRY" = 1 ]; then
+    echo "would run: $SELF/apply-protected-cast.sh $CALIB $WORK $TARGET_COUNT"
+else
+    # Positional, because the tool reads its recipe and calib dir by position: passing them as
+    # environment variables would be silently ignored and it would rewrite the COMMITTED
+    # template in place instead of this run's copy.
+    "$SELF/apply-protected-cast.sh" "$CALIB" "$WORK" "$TARGET_COUNT" \
+        || { echo "the protected-expert derivation refused; the arms would convert unprotected" >&2; exit 1; }
+fi
+
+run_convert() { # run_convert ARM [extra flags...]
+    local arm="$1"; shift
+    local out="$OUTDIR/qwen38-w5nl64-$arm.rad" log="$STATE/convert-$arm-$TS.log"
+    local argv=(--rm --name "qwen38-convert-$arm"
+                -e "CALIB=/data/calib" -e "OMP_NUM_THREADS=$(nproc)"
+                -v "$(dirname "$CKPT"):/models:ro"
+                -v "$CALIB:/data/calib:ro"
+                -v "$STATE:/campaign:ro"
+                -v "$OUTDIR:/out")
+    local conv=(/models/"${CKPT##*/}" --recipe "/campaign/$(basename "$WORK")"
+                --tokenizer "/models/${CKPT##*/}/tokenizer.json"
+                --imatrix /campaign/"${IMATRIX##*/}"
+                # No --max-ctx: the context declare then runs at the checkpoint's training context
+                # (262144), which is what the shipped container did — rad-info --meta on it carries
+                # no override. A container declared at a smaller bound cannot be expected to serve
+                # the 200k window D2 holds, and the convert is not where to discover that.
+                # --kv-cache-dtype changes nothing in the artifact; it picks the kernels the declare
+                # checks, and the fleet serves fp8 KV, so an unresolvable fp8 kernel must be said no
+                # to by the convert, not by a boot on a drained window.
+                --kv-cache-dtype fp8 -o "/out/$(basename "$out")")
+    local q; q=$(arm_quant "$arm")
+    [ -n "$q" ] && conv+=(--quant "$q")
+    if [ "$DRY" = 1 ]; then
+        echo "podman run ${argv[*]} \$IMAGE --entrypoint /opt/radiance/bin/rad-convert ${conv[*]} $*"
+        return 0
+    fi
+    podman run "${argv[@]}" --entrypoint /opt/radiance/bin/rad-convert "$IMAGE" "${conv[@]}" "$@" \
+        2>&1 | tee "$log"
+    return "${PIPESTATUS[0]}"
+}
+
+# --- plan, then bill, then convert -------------------------------------------------------------
+overall=0
+for arm in "${ARMS[@]}"; do
+    echo "== arm $arm =="
+    if [ "$DRY" = 1 ]; then
+        # Both lines, in order: the plan runs before the convert, and the convert is what the
+        # operator is about to pay an afternoon for.
+        run_convert "$arm" --plan-only
+        run_convert "$arm"
+        continue
+    fi
+    rc=0; plan_log=$(run_convert "$arm" --plan-only) || rc=$?
+    [ "$rc" -eq 0 ] || { echo "arm $arm: --plan-only failed (rc=$rc); nothing was written" >&2; overall=1; continue; }
+    echo "$plan_log"
+    # "about 141.2 GiB would be written" is the converter's own bill; parse it rather than
+    # hardcoding a size, so the gate still works after any recipe edit.
+    want=$(printf '%s' "$plan_log" | sed -n 's/.*about \([0-9.]*\) \([KMGT]iB\) would be written.*/\1 \2/p' | tail -1)
+    [ -n "$want" ] || { echo "arm $arm: the plan printed no size; refusing to convert blind" >&2; overall=1; continue; }
+    need_mib=$(awk -v n="${want% *}" -v u="${want#* }" \
+        'BEGIN{ f = (u=="KiB") ? 2^-10 : (u=="MiB") ? 1 : (u=="GiB") ? 1024 : (u=="TiB") ? 1048576 : 0
+               if (f == 0) { exit 1 }; printf "%d", n*f + 0.5 }') || {
+        echo "arm $arm: the plan printed an unknown unit in '$want'" >&2; overall=1; continue; }
+    free_mib=$(df --output=avail -m "$OUTDIR" | tail -1 | tr -d ' ')
+    echo "plan: $want ($need_mib MiB); free ${free_mib} MiB on $OUTDIR (margin $MARGIN_MIB MiB)"
+    if [ "$free_mib" -lt $((need_mib + MARGIN_MIB)) ]; then
+        echo "arm $arm: needs ${need_mib} MiB + ${MARGIN_MIB} margin, ${free_mib} free; free space and re-run" >&2
+        overall=1; continue
+    fi
+    if [ "$PLAN_ONLY" = 1 ]; then echo "planned, not converted (--plan-only)"; continue; fi
+    rc=0; run_convert "$arm" || rc=$?
+    [ "$rc" -eq 0 ] || { echo "arm $arm: rad-convert rc=$rc; log: $STATE/convert-$arm-$TS.log" >&2; overall=1; continue; }
+    out="$OUTDIR/qwen38-w5nl64-$arm.rad"
+    [ -f "$out" ] || { echo "arm $arm: rad-convert exited 0 but wrote no $out; log: $STATE/convert-$arm-$TS.log" >&2; overall=1; continue; }
+    got_mib=$(( $(stat -c %s "$out") / 1048576 ))
+    # The receipt: the artifact is what the plan predicted, within the metadata the plan does not
+    # count. An arm that came out materially smaller lost a plane somewhere.
+    lo=$(( need_mib * 98 / 100 )); hi=$(( need_mib * 103 / 100 ))
+    if [ "$got_mib" -lt "$lo" ] || [ "$got_mib" -gt "$hi" ]; then
+        echo "arm $arm: $out is ${got_mib} MiB, the plan said ${need_mib} MiB (allowed ${lo}-${hi})" >&2
+        overall=1; continue
+    fi
+    echo "CONVERT_DONE $arm ${got_mib} MiB -> $out"
+done
+
+if [ "$KEEP_RECIPE" = 0 ] && [ "$DRY" = 0 ]; then
+    mkdir -p "$CAMPAIGN/.AGENTTRASH"
+    mv "$WORK" "$CAMPAIGN/.AGENTTRASH/$(basename "$WORK")" 2>/dev/null || true
+fi
+exit "$overall"

scripts/deploy-plan.py

@@ -34,12 +34,16 @@ SERVICE_ORDER = (
     "hermes",
     "hermes-dashboard",
     "hermes-webui",
-    "prometheus",
     "grafana",
     "loki",
     "alloy",
     "node-exporter",
+    "cadvisor",
+    "gpu-exporter",
     "alertmanager",
+    # The prober must answer before prometheus re-runs the services job.
+    "blackbox-exporter",
+    "prometheus",
     "authentik-postgres",
     "authentik-server",
     "authentik-worker",
@@ -92,7 +96,10 @@ DANKCLAW_SERVICES = {
     "loki",
     "alloy",
     "node-exporter",
+    "cadvisor",
+    "gpu-exporter",
     "alertmanager",
+    "blackbox-exporter",
     "authentik-postgres",
     "authentik-server",
     "authentik-worker",
@@ -124,6 +131,8 @@ DANKSERVER_SERVICES = {
     "readarr",
     "frank-archon",
     "node-exporter",
+    "cadvisor",
+    "gpu-exporter",
 }
 
 # FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on
@@ -146,13 +155,14 @@ DANKCLAW_COMPOSE_SERVICES = set()
 # Quadlet unit now — the last compose services were the hermes trio, whose units
 # have owned their containers since 2026-09-12 and whose compose blocks were
 # dead weight that could only create a duplicate container.
+# jellyfin left the Dankserver file on 2026-10-30 for the same reason: its Quadlet
+# has owned the container since the cutover, so a compose edit must not restart it.
 
 DANKSERVER_COMPOSE_SERVICES = {
     "mariadb",
     "redis",
     "romm",
     "copyparty",
-    "jellyfin",
     "calibre-web",
 }
 COMPOSE_NAME_MAP = {"mariadb": "romm-mariadb", "redis": "romm-redis"}
@@ -271,6 +281,8 @@ def services_for_path(path: str, target: str) -> set[str]:
             # Provisioned datasources and dashboards: git is the source of
             # truth, so a change here must restart the service that reads them.
             _add(services, "grafana")
+        elif group == "blackbox":
+            _add(services, "blackbox-exporter")
         elif group == "homepage":
             # Homepage is a Dankclaw-owned deployment. Shared homepage files
             # are intentionally not copied during a Dankserver-only run.
@@ -293,6 +305,17 @@ def services_for_path(path: str, target: str) -> set[str]:
         if len(parts) >= 2 and parts[1] in {"caddy", "comfyui", "loki", "alloy", "hermes", "anubis", "seerr"}:
             _add(services, parts[1])
             return services
+        if len(parts) >= 3 and parts[1] == "host":
+            # Host state a service needs at boot lives under dankclaw/host/.
+            # The fabric-clock hold is frank-dragoon's NPU precondition (the
+            # engine refuses to start with HALOGEN_NPU_MODELS set while the
+            # clock moves) and the deployer installs and gates it, so changing
+            # it re-dispatches that endpoint. `grub` is the tracked boot line,
+            # which moves only through scripts/set-dankclaw-bootline.sh and is
+            # a reboot decision — it plans no service.
+            if parts[2].startswith("halogen-fabric-clock"):
+                _add(services, "frank-dragoon")
+            return services
         if len(parts) >= 4 and parts[1] == "authentik" and parts[2] == "quadlet":
             name = parts[3].removesuffix(".container")
             if name in {"authentik-postgres", "authentik-server", "authentik-worker", "authentik-outpost", "authentik-outpost-ldap"}:
@@ -363,7 +386,7 @@ def services_for_path(path: str, target: str) -> set[str]:
             # consumer (prometheus before grafana, loki before alloy).
             if len(parts) >= 4 and parts[2] == "quadlet":
                 name = parts[3].removesuffix(".container")
-                if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "alertmanager"}:
+                if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "cadvisor", "alertmanager", "blackbox-exporter"}:
                     _add(services, name)
             return services
         if len(parts) >= 2 and parts[1] == "gitea":
@@ -399,6 +422,11 @@ def services_for_path(path: str, target: str) -> set[str]:
                 "tautulli": {"tautulli"},
             }
             _add(services, *quadlet_services.get(parts[1], set()))
+        if len(parts) >= 2 and parts[1] == "host":
+            # dankserver/host/systemd/media-meta-view.mount is the overlay view the
+            # media servers bind (scripts/media-view.sh installs it on the host),
+            # so editing it restarts jellyfin and plex and nothing else.
+            _add(services, "jellyfin", "plex")
         if len(parts) >= 2 and parts[1] == "observability":
             # Dankserver's node-exporter became a Quadlet unit on 2026-09-16
             # (serverAppConfigs-docs/Infrastructure/Observability.md): its
@@ -406,8 +434,8 @@ def services_for_path(path: str, target: str) -> set[str]:
             # node-dankserver job had nothing to scrape.
             if len(parts) >= 4 and parts[2] == "quadlet":
                 name = parts[3].removesuffix(".container")
-                if name == "node-exporter":
-                    _add(services, "node-exporter")
+                if name in {"node-exporter", "cadvisor"}:
+                    _add(services, name)
             return services
         # frank-archon's quadlet-era directory left the tree on 2026-09-26
         # (the Lucebox DS4.1 unit was retired for the llama.cpp root unit —
@@ -438,6 +466,21 @@ def services_for_path(path: str, target: str) -> set[str]:
             # change to it redeploying alertmanager is the only path that
             # reaches it on the host.
             _add(services, "alertmanager")
+        if name in {"gpu-exporter.py", "gpu-exporter.service"}:
+            # The host-process GPU exporter is installed directly by the
+            # gpu-exporter deploy on BOTH hosts, so its script and unit select
+            # that service (target_services then keeps it only on the host being
+            # planned). Editing the collector restarts the running exporter.
+            _add(services, "gpu-exporter")
+        if name == "media-view.sh":
+            # The installer is what creates, mounts and reaps the view both media
+            # servers read, so a change to it is a restart of that pair.
+            _add(services, "jellyfin", "plex")
+        if name == "jellyfin-providers.sh":
+            # The deployer runs this before jellyfin restarts, so a change to
+            # the installer is a change to the provider state Jellyfin loads at
+            # start-up: it selects jellyfin and nothing else.
+            _add(services, "jellyfin")
         if name in {"frank-zealot.env", "frank-zealot.service", "serve-frank-zealot.sh"}:
             # llama.cpp frank-zealot (2026-09-16) keeps its unit, env and serve
             # files here; each one selects frank-zealot only. The ik_llama.cpp

scripts/deploy-remote.sh

@@ -86,7 +86,7 @@ service_is_allowed() {
         dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
         dankclaw:comfyui|dankclaw:hermes|\
         dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
-        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\
+        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:alertmanager|dankclaw:blackbox-exporter|\
         dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
         dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
@@ -97,7 +97,7 @@ service_is_allowed() {
         dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
         dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
         dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
-        dankserver:frank-archon|dankserver:node-exporter|\
+        dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|dankserver:gpu-exporter|\
         frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
         frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
         frankbook-pro:readarr)
@@ -252,16 +252,24 @@ install_signal_relay() {
     # The relay binds the host's LAN IP (the one address a bridge container
     # can reach as host.containers.internal), so keep the unauthenticated
     # endpoint off the rest of the LAN: admit loopback, the host itself
-    # (its own health probe) and the podman bridge subnet only. Idempotent:
-    # -C before -I.
+    # (its own health probe) and the podman bridge subnet only.
+    # ORDER IS THE GATE: `iptables -I` inserts at position 1, so the old
+    # shape (-C/-I x3 ACCEPT, then -C/-I the DROP) landed the blanket DROP
+    # ABOVE the ACCEPTs and every probe timed out beside a LISTENing socket
+    # (live 2026-10-21: 159 packets on the DROP, 0 on each ACCEPT; the whole
+    # dankclaw drain aborted at the alertmanager gate twice before the cause
+    # was read off `nft list ruleset`). Rebuild the tail instead of inserting
+    # it: delete any blanket DROP wherever it sits, keep the three ACCEPTs
+    # at the head (-C before -I, unchanged), then APPEND the DROP so it is
+    # last and the ACCEPTs decide first. Converges on poisoned hosts.
+    run_privileged iptables -D INPUT -p tcp --dport 18093 -j DROP 2>/dev/null || true
     run_privileged iptables -C INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT 2>/dev/null \
         || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT
     run_privileged iptables -C INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT 2>/dev/null \
         || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT
     run_privileged iptables -C INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT 2>/dev/null \
         || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT
-    run_privileged iptables -C INPUT -p tcp --dport 18093 -j DROP 2>/dev/null \
-        || run_privileged iptables -I INPUT -p tcp --dport 18093 -j DROP
+    run_privileged iptables -A INPUT -p tcp --dport 18093 -j DROP
     # The interpreter needs a moment to bind after restart; a one-shot curl
     # raced the socket on 2026-09-17 and failed a healthy deploy.
     local i
@@ -275,14 +283,81 @@ install_signal_relay() {
     return 1
 }
 
+# Host-process GPU metrics exporter (amdgpu sysfs + nvidia-smi). Not a quadlet:
+# it runs python3 from the checkout on the host so it can read the DRM/cgroup
+# sysfs and shell out to nvidia-smi. Same two-address bind posture as
+# node-exporter/cadvisor (loopback + this host's LAN), so it is never a wildcard.
+deploy_gpu_exporter() {
+    local unit_dir="$HOME/.config/systemd/user" lan dropin i
+    case "$TARGET" in
+        dankclaw) lan=192.168.0.143 ;;
+        dankserver) lan=192.168.0.235 ;;
+        *) echo "gpu-exporter: unknown target '$TARGET'" >&2; return 2 ;;
+    esac
+    mkdir -p "$unit_dir"
+    cp scripts/gpu-exporter.service "$unit_dir/gpu-exporter.service"
+    dropin="$unit_dir/gpu-exporter.service.d"
+    mkdir -p "$dropin"
+    printf '[Service]\nEnvironment=GPU_EXPORTER_BINDS=127.0.0.1,%s\n' "$lan" \
+        > "$dropin/bind.conf"
+    systemctl --user daemon-reload
+    systemctl --user enable gpu-exporter.service
+    systemctl --user restart gpu-exporter.service
+    # The interpreter needs a moment to bind after restart (the same race that
+    # once failed a healthy signal-relay deploy).
+    for i in $(seq 1 15); do
+        curl -fsS -m 5 http://127.0.0.1:9401/metrics >/dev/null && return 0
+        sleep 1
+    done
+    echo "gpu-exporter is not answering on 127.0.0.1:9401/metrics" >&2
+    return 1
+}
+
 copy_loki_config() {
     run_privileged mkdir -p /opt/loki/config
     run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
 }
 
 copy_alloy_config() {
-    run_privileged mkdir -p /opt/alloy/config
-    run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
+    # The repo config carries the token GRAFANA_HOST_LABEL; the installed copy
+    # carries the host's name, so Loki streams say WHICH machine's journal a
+    # line came from without a second config file per host.
+    # *_CONF_ROOT are test seams: the bats cases execute these copiers for
+    # real against a sandbox with a stubbed run_privileged.
+    local root="${ALLOY_CONF_ROOT:-/opt/alloy}" tmp
+    tmp="$(mktemp)"
+    sed "s/GRAFANA_HOST_LABEL/$TARGET/g" shared/alloy/config.alloy > "$tmp"
+    if ! grep -q "host = \"$TARGET\"," "$tmp" || grep -q "GRAFANA_HOST_LABEL" "$tmp"; then
+        echo "alloy config: host-label substitution failed" >&2
+        rm -f "$tmp"
+        return 1
+    fi
+    # Syntax gate: alloy's config language has no schema — a misplaced
+    # attribute (e.g. `on_error` outside the json stage) parses only at RUN
+    # time, and the unit then crash-loops with a healthy-looking deploy.
+    # Validate against the very image that will run it, before installing.
+    # Skipped only in the bats sandbox (no image there).
+    if [ "$root" = "/opt/alloy" ] || podman image exists localhost/franksdank/alloy:latest 2>/dev/null; then
+        if ! podman run --rm -v "$tmp:/c.alloy:ro" \
+                --entrypoint /usr/bin/alloy localhost/franksdank/alloy:latest \
+                validate /c.alloy; then
+            echo "alloy config: validate FAILED against the pinned image" >&2
+            rm -f "$tmp"
+            return 1
+        fi
+    fi
+    run_privileged mkdir -p "$root/config"
+    run_privileged cp "$tmp" "$root/config/config.alloy"
+    rm -f "$tmp"
+}
+
+copy_blackbox_config() {
+    # Single-file mount target: the unit binds this exact path into
+    # /etc/blackbox_exporter/config.yml. cp (not cp -r) keeps it one file.
+    # BLACKBOX_CONF_ROOT is the same test seam.
+    local root="${BLACKBOX_CONF_ROOT:-/opt/blackbox}"
+    run_privileged mkdir -p "$root"
+    run_privileged cp shared/blackbox/blackbox.yml "$root/blackbox.yml"
 }
 
 copy_grafana_provisioning() {
@@ -291,14 +366,25 @@ copy_grafana_provisioning() {
     # datasource would be invisible to review, lost with the data volume, and
     # unknown to anyone reading the repository. The unit already mounts
     # /opt/grafana/provisioning read-only, so this only has to populate it.
-    run_privileged mkdir -p /opt/grafana/provisioning/datasources \
-        /opt/grafana/provisioning/dashboards
+    # GRAFANA_PROV_ROOT is the test seam: point it at a sandbox and the whole
+    # copy runs unprivileged (the bats case executes this function for real).
+    local root="${GRAFANA_PROV_ROOT:-/opt/grafana}"
+    run_privileged mkdir -p "$root/provisioning/datasources" \
+        "$root/provisioning/dashboards"
     run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \
-        /opt/grafana/provisioning/datasources/prometheus.yml
+        "$root/provisioning/datasources/prometheus.yml"
     run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \
-        /opt/grafana/provisioning/dashboards/provider.yml
-    run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \
-        /opt/grafana/provisioning/dashboards/llm-fleet.json
+        "$root/provisioning/dashboards/provider.yml"
+    # EVERY dashboard JSON rides, not a hand-listed subset: the fleet provider
+    # turns each file in the directory into a dashboard, so a copier that names
+    # files one by one silently drops new boards (measured 2026-10-21: the
+    # containers board existed in the repo since its commit and was NEVER
+    # provisioned — the panel-census browser check could not find it while the
+    # hand-placed older boards happened to be on disk). Same rule as the quadlet
+    # tier: the directory is the contract, the copy is generated state.
+    for dash in shared/grafana/provisioning/dashboards/*.json; do
+        run_privileged cp "$dash" "$root/provisioning/dashboards/$(basename "$dash")"
+    done
 }
 
 render_caddyfile() {
@@ -487,13 +573,15 @@ seed_grafana_env() {
         || { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; }
 }
 
-# The five observability containers left compose on 2026-09-11 (they were
-# started by hand with podman-compose and never came back after a reboot).
-# They are ordinary rootless Quadlet units now: same images, same mounts, same
-# arguments, same container names, so nothing that addresses them by name —
-# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
-# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
-# compose services published on 0.0.0.0 and none of them authenticates.
+# The observability containers left compose on 2026-09-11 (they were started by
+# hand with podman-compose and never came back after a reboot). Five are ordinary
+# rootless Quadlet units: same images, same mounts, same arguments, same
+# container names, so nothing that addresses them by name — Caddy, Prometheus'
+# own scrape config — changed. Only the published ports did: the three HTTP UIs
+# bind 127.0.0.1, because the compose services published on 0.0.0.0 and none of
+# them authenticates. Alloy is the exception: a ROOT system service on
+# Network=host, because reading the host journal needs privileges a rootless
+# container cannot get. It is handled by deploy_alloy, not the loop below.
 deploy_observability() {
     local service="$1"
     # Observability units live under <target>/observability/quadlet: Dankclaw
@@ -512,16 +600,17 @@ deploy_observability() {
             copy_loki_config
             build_compose_image loki
             ;;
-        alloy)
-            copy_alloy_config
-            build_compose_image alloy
-            ;;
         grafana)
             copy_grafana_provisioning
             seed_grafana_env || return 1
             ;;
         node-exporter)
             ;;
+        cadvisor)
+            ;;
+        blackbox-exporter)
+            copy_blackbox_config
+            ;;
         *)
             echo "unknown observability service: $service" >&2
             return 2
@@ -534,11 +623,75 @@ deploy_observability() {
         alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
         grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
         loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
-        alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
         node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
+        cadvisor) wait_for_health cadvisor http://127.0.0.1:8080/healthz 90 ;;
+        # Loopback: the exporter binds 127.0.0.1 + LAN (the gateway address
+        # is not bindable from the host netns — see the unit header).
+        blackbox-exporter) wait_for_health blackbox-exporter http://127.0.0.1:9115/-/healthy 60 ;;
     esac
 }
 
+# Mirror an image the deployer built rootless into the ROOT image store. A root
+# quadlet reads root storage; a root build (build_compose_image is rootless) is
+# not possible on this host — the build container's resolver fails DNS — so the
+# only way to get a root-readable copy is save-and-load. Compares image ids so a
+# rebuilt alloy actually re-lands in root storage instead of leaving a stale tag.
+mirror_image_to_root() {
+    local ref="$1" rid
+    rid="$(podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" || {
+        echo "ERROR: rootless image $ref is not built; cannot mirror to root" >&2
+        return 1
+    }
+    if [[ "$(run_privileged podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" == "$rid" ]]; then
+        return 0
+    fi
+    local tar; tar="$(mktemp /var/tmp/alloy-mirror.XXXXXX.tar)"
+    podman save -o "$tar" "$ref"
+    run_privileged podman load -i "$tar"
+    rm -f "$tar"
+}
+
+# Install a quadlet into the SYSTEM generator dir (root manager) instead of the
+# user dir. Boot persistence is the unit's own [Install] section (multi-user
+# target); the v1 system generator creates the .wants on daemon-reload, so —
+# like the user units — no `systemctl enable` (generated units cannot be enabled
+# here).
+install_system_quadlet() {
+    local source_dir="$1" unit="$2"
+    run_privileged mkdir -p /etc/containers/systemd
+    run_privileged cp "$source_dir/$unit.container" /etc/containers/systemd/
+    run_privileged systemctl daemon-reload
+}
+
+# Alloy as a ROOT system service on Network=host. See the unit file: a rootless
+# container cannot read the 0640 journal files, and it cannot reach Loki's
+# 127.0.0.1:3100 publish from a bridge netns. Root + host network fixes both;
+# the price is that this one container runs as root and on the host network.
+deploy_alloy() {
+    copy_alloy_config
+    build_compose_image alloy                      # rootless build (resolver works)
+    mirror_image_to_root localhost/franksdank/alloy:latest
+
+    # Retire the rootless unit BEFORE the system one starts: both managers
+    # generate alloy.service, and two containers named "alloy" both binding
+    # 127.0.0.1:12345 cannot coexist. Move the user quadlet out of the generator
+    # dir (it regenerates from there) and stop the user unit.
+    if [[ -f "$quadlet_unit_dir/alloy.container" ]] ||
+       systemctl --user is-active --quiet alloy.service 2>/dev/null; then
+        systemctl --user stop alloy.service 2>/dev/null || true
+        if [[ -f "$quadlet_unit_dir/alloy.container" ]]; then
+            local trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-alloy-root-cutover"
+            mkdir -p "$trash"
+            mv -- "$quadlet_unit_dir/alloy.container" "$trash/alloy.container"
+            echo "retired rootless alloy.container -> $trash"
+        fi
+        systemctl --user daemon-reload
+    fi
+
+    install_system_quadlet dankclaw/observability/quadlet alloy
+    run_privileged systemctl restart alloy.service
+    wait_for_health alloy http://127.0.0.1:12345/-/healthy 60
+}
 build_compose_image() {
     local service="$1" image cfhash oldhash containerfile
     # Every file the image COPIES IN, beside the Containerfile. The skip below
@@ -1049,7 +1202,42 @@ deploy_romm_stack() {
     systemctl --user restart "$service.service"
 }
 
+ensure_media_view() {
+    # Both media servers read their library through the SSD-backed overlay view
+    # (scripts/media-view.sh, dankserver/host/systemd/media-meta-view.mount): the
+    # array below stays read-only and every sidecar Jellyfin writes lands on the
+    # NVMe.
+    # Raise it BEFORE either container starts — a start against a bare view
+    # directory would let a scan clear the library database. The units also carry
+    # ConditionPathIsMountPoint, so a missing view skips the start rather than
+    # damaging anything. A relayout must be followed by `media-view.sh reap`
+    # (see the Media Relayout page), which this is not.
+    run_privileged bash scripts/media-view.sh install
+    bash scripts/media-view.sh verify
+}
+
+ensure_provider_keys() {
+    # Jellyfin's metadata-provider keys live in the container-owned plugin
+    # configuration under the data root, never in git; scripts/jellyfin-providers.sh
+    # is what places them, reading the per-service host env file (the fleet
+    # convention: grafana, caddy, pihole, copyparty, romm, seerr) that holds
+    # TMDB_API_KEY= and TVDB_API_KEY=, mode 600.
+    # Missing keys mean a keyless provider — a degraded library, not a broken
+    # deploy — so this warns rather than failing the restart. But when the keys
+    # ARE placed it must happen before the restart: the plugin and its keys are
+    # read at start-up, so a restart first would boot the old state.
+    local keys=/home/dank/Containers/jellyfin/provider-keys
+    if [ -f "$keys" ]; then
+        bash scripts/jellyfin-providers.sh install
+    else
+        echo "WARN: $keys is absent - provider keys NOT placed"
+        echo "WARN: create it (TMDB_API_KEY=, TVDB_API_KEY=, chmod 600) and re-deploy jellyfin"
+    fi
+}
+
 deploy_jellyfin() {
+    ensure_media_view
+    ensure_provider_keys
     cp "dankserver/jellyfin/quadlet/jellyfin.container" "$quadlet_unit_dir/"
     systemctl --user daemon-reload
     if ! systemctl --user is-active -q jellyfin.service; then
@@ -1137,6 +1325,7 @@ deploy_copyparty() {
 }
 
 deploy_plex() {
+    ensure_media_view
     cp "dankserver/plex/quadlet/plex.container" "$quadlet_unit_dir/"
     systemctl --user daemon-reload
     if ! systemctl --user is-active -q plex.service; then
@@ -1209,11 +1398,17 @@ deploy_authentik() {
     # generated". Boot persistence comes from [Install] WantedBy=default.target
     # inside each .container file, which the generator honors at user-manager
     # start (linger must stay on for the deploy user).
-    systemctl --user start authentik-postgres.service
+    # RESTART, not start: `start` on an already-active unit is a no-op, so an
+    # image bump would install the new digest and keep serving the old
+    # container while every gate stayed green (measured 2026-10-30: server and
+    # worker stayed on the previous release beside an outpost that moved,
+    # because only the outpost handler restarts). The generator has already
+    # re-rendered the unit from the .container, so a restart is what applies it.
+    systemctl --user restart authentik-postgres.service
     wait_container_healthy authentik-postgres 120 || return 1
-    systemctl --user start authentik-server.service
+    systemctl --user restart authentik-server.service
     wait_container_healthy authentik-server 300 || return 1
-    systemctl --user start authentik-worker.service
+    systemctl --user restart authentik-worker.service
 }
 
 deploy_authentik_outpost() {
@@ -1310,9 +1505,10 @@ deploy_anubis() {
 # conversion (2026-10-11) of the abliterated Qwen3.8-Flash-Next —
 # qwen38-flash-next-v2-abliterated.hgn (66,687,678,432 B, ~62 GiB resident)
 # + the REQUIRED n-gram lookup table + the vision tower — MTP spec decode
-# (HALOGEN_SPEC_ADAPT=0), 2 slots over ONE 401,408-position KV pool (2 × ctx,
-# raised 2026-10-18 so BOTH slots can hold a maximum-length request) at the
-# ctx 200704 ceiling, image input — on the Strix Halo iGPU. It stays a
+# (HALOGEN_SPEC_ADAPT=0), 2 slots over ONE 401,408-position KV pool at the
+# native ctx 262144 ceiling (raised 2026-10-21 from 200704 so the harness can
+# run a 200,000 window with its ~45k fit gap; the pool still carries two
+# 200k conversations at once), image input — on the Strix Halo iGPU. It stays a
 # ROOT unit because the serve script drives `podman run` directly (the frank-archon pattern):
 # relink_unit points /etc/systemd/system/frank-dragoon.service at the repo
 # file, boot persistence is that unit's [Install], and LimitMEMLOCK=infinity
@@ -1328,7 +1524,10 @@ deploy_anubis() {
 # artifact would leave a restart-looping unit and a dark endpoint. The
 # port gate is the same as every cutover since 2026-09-16: nothing
 # starts until 7770 is released.
-LLM_DRAGOON_IMAGE="${LLM_DRAGOON_IMAGE:-ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a}"
+LLM_DRAGOON_IMAGE="${LLM_DRAGOON_IMAGE:-ghcr.io/peonist-ai/halogen-flash-server@sha256:e35133b206da3094cc4d70c78800b26235bd9124b9aeb0e029d4a7c153b9cff0}"
+# The payload dir is its own twin because the NPU lane's containment gate below
+# reads it; every artifact path under it stays a full literal as usual.
+LLM_DRAGOON_MODEL_DIR="${LLM_DRAGOON_MODEL_DIR:-/home/dank/models/halogen-v2-abliterated}"
 LLM_DRAGOON_CKPT="${LLM_DRAGOON_CKPT:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-v2-abliterated.hgn}"
 LLM_DRAGOON_CKPT_BYTES="${LLM_DRAGOON_CKPT_BYTES:-66687678432}"
 LLM_DRAGOON_NGRAM="${LLM_DRAGOON_NGRAM:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-ngram.hgn}"
@@ -1337,6 +1536,18 @@ LLM_DRAGOON_VISION="${LLM_DRAGOON_VISION:-/home/dank/models/halogen-v2-abliterat
 LLM_DRAGOON_VISION_BYTES="${LLM_DRAGOON_VISION_BYTES:-897916416}"
 LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-v2-abliterated/tokenizer}"
 LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}"
+# NPU lane (2026-10-19). HALOGEN_NPU_MODELS is the whole switch and the parity
+# twin of the env file's line: empty here AND in the env = the pre-lane
+# endpoint. The gates below mirror serve-frank-dragoon.sh, and the vendor's
+# fabric-clock host unit (dankclaw/host/, verbatim at tag v0.16.2) is installed
+# ONLY while the lane is named — on Strix Halo the iGPU and the NPU co-running
+# hang the machine while the fabric clock moves, so halogen refuses to boot with
+# the lane armed unless that clock is held.
+LLM_DRAGOON_NPU_MODELS="${LLM_DRAGOON_NPU_MODELS:-qwen3-embedding-0.6b}"
+LLM_DRAGOON_NPU_DIR="${LLM_DRAGOON_NPU_DIR:-/home/dank/models/halogen-v2-abliterated/npu}"
+LLM_DRAGOON_DEV_ACCEL="${LLM_DRAGOON_DEV_ACCEL:-/dev/accel/accel0}"
+LLM_DRAGOON_XRT_DIR="${LLM_DRAGOON_XRT_DIR:-/usr/lib/x86_64-linux-gnu}"
+LLM_DRAGOON_NETWORK="${LLM_DRAGOON_NETWORK:-dankclaw_default}"
 
 retire_halogen_dragoon_quadlet() {
     local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
@@ -1415,6 +1626,77 @@ deploy_llm_dragoon() {
         fi
     done
 
+    # The container network has to exist before the run line names it (a typo
+    # there is a restart-looping unit, not a config error anyone reads).
+    if [[ -n "$LLM_DRAGOON_NETWORK" ]] && ! podman network exists "$LLM_DRAGOON_NETWORK"; then
+        echo "ERROR: podman network $LLM_DRAGOON_NETWORK does not exist" >&2
+        echo "frank-dragoon: create it or blank FRANK_DRAGOON_NETWORK (pasta default)" >&2
+        return 1
+    fi
+
+    # --- NPU lane: the same predicates the serve script checks --------------
+    if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then
+        # One door into the container: the read-only /models mount of the
+        # payload dir. A staging dir outside it needs a nested bind whose
+        # mountpoint runc cannot create through a read-only parent, so the
+        # parity twin of the serve gate refuses the shape before the restart.
+        if [[ "$LLM_DRAGOON_NPU_DIR/" != "$LLM_DRAGOON_MODEL_DIR"/* ]]; then
+            echo "ERROR: NPU staging dir $LLM_DRAGOON_NPU_DIR is outside the payload dir $LLM_DRAGOON_MODEL_DIR — the lane's files reach the container only through the read-only /models mount" >&2
+            return 1
+        fi
+        if [[ ! -e "$LLM_DRAGOON_DEV_ACCEL" ]]; then
+            echo "ERROR: NPU node $LLM_DRAGOON_DEV_ACCEL absent — amdxdna is not bound (is iommu=pt still on the boot line?)" >&2
+            return 1
+        fi
+        # Access, not existence: a rootless container cannot open a root:render
+        # 0660 node (its namespace maps one uid/gid, and keep-groups cannot grant
+        # an unmapped gid — measured 2026-10-19). Install the mode rule, re-trigger
+        # the node, then PROVE the node is other-accessible before the restart.
+        run_privileged install -m 0644 dankclaw/udev/98-halogen-npu-access.rules \
+            /etc/udev/rules.d/98-halogen-npu-access.rules
+        run_privileged udevadm control --reload-rules
+        run_privileged udevadm trigger --sysname-match="$(basename "$LLM_DRAGOON_DEV_ACCEL")"
+        # (`-o FILE` is not "other-writable" — it means "exists", and `-o` is also
+        # the OR operator inside test), so the bit is read straight off the mode.
+        if [[ $(( 8#$(stat -c '%a' "$LLM_DRAGOON_DEV_ACCEL") & 8#2 )) -eq 0 ]]; then
+            echo "ERROR: $LLM_DRAGOON_DEV_ACCEL is still $(stat -c '%a %U:%G' "$LLM_DRAGOON_DEV_ACCEL") after installing the access rule — the lane cannot open it rootless" >&2
+            return 1
+        fi
+        for xl in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
+            if [[ ! -e "$LLM_DRAGOON_XRT_DIR/$xl" ]]; then
+                echo "ERROR: XRT NPU library $LLM_DRAGOON_XRT_DIR/$xl absent (Ubuntu libxrt2 + libxrt-npu2)" >&2
+                return 1
+            fi
+        done
+        for nm in ${LLM_DRAGOON_NPU_MODELS//,/ }; do
+            if [[ ! -d "$LLM_DRAGOON_NPU_DIR/$nm" ]]; then
+                echo "ERROR: NPU model $nm not staged in $LLM_DRAGOON_NPU_DIR" >&2
+                echo "frank-dragoon: stage it: bash scripts/download-frank-dragoon-npu.sh" >&2
+                return 1
+            fi
+        done
+        # The hold must exist BEFORE the engine starts and survive every boot
+        # (a resume hands the clock back too — the unit's WantedBy covers it).
+        run_privileged install -m 0755 dankclaw/host/halogen-fabric-clock \
+            /usr/local/sbin/halogen-fabric-clock
+        run_privileged install -m 0644 dankclaw/host/halogen-fabric-clock.service \
+            /etc/systemd/system/halogen-fabric-clock.service
+        run_privileged systemctl daemon-reload
+        run_privileged systemctl enable --now halogen-fabric-clock.service
+        if ! run_privileged /usr/local/sbin/halogen-fabric-clock status; then
+            echo "ERROR: fabric clock is not held — halogen refuses to start with HALOGEN_NPU_MODELS set" >&2
+            echo "frank-dragoon: journalctl -u halogen-fabric-clock.service -n 40" >&2
+            return 1
+        fi
+    elif [[ -x /usr/local/sbin/halogen-fabric-clock ]]; then
+        # Lane rolled back: hand the clock back to the driver and drop the boot
+        # hook. `disable` deletes the INSTALLED copy of this unit — unlike the
+        # tracked LLM unit symlinks the fleet's disable law protects, this file
+        # is repo-owned and the next lane-on deploy reinstalls it.
+        run_privileged systemctl disable halogen-fabric-clock.service 2>/dev/null || true
+        run_privileged /usr/local/sbin/halogen-fabric-clock release || true
+    fi
+
     retire_halogen_dragoon_quadlet
     # The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
     # the container) before waiting on the port. The stray sweep is keyed
@@ -1434,6 +1716,33 @@ deploy_llm_dragoon() {
     run_privileged systemctl enable frank-dragoon.service
     run_privileged systemctl restart frank-dragoon.service
     wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
+    # The lane's own gate. /health proves the engine is up, nothing about the
+    # NPU programs; one real embedding is the observable output. The first
+    # request after a boot is a warmup artifact (fleet law), so retry before
+    # calling a booted lane dead.
+    if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then
+        for nm in ${LLM_DRAGOON_NPU_MODELS//,/ }; do
+            lane_ok=0
+            for _try in 1 2 3 4 5; do
+                if curl -fsS -m 60 -H 'content-type: application/json' \
+                        -d "{\"model\":\"$nm\",\"input\":[\"frank-dragoon npu lane gate\"]}" \
+                        http://127.0.0.1:7770/v1/embeddings 2>/dev/null \
+                   | python3 -c 'import json,sys
+v = json.load(sys.stdin)["data"][0]["embedding"]
+raise SystemExit(0 if isinstance(v, list) and len(v) and all(isinstance(x, (int, float)) for x in v) else 1)'; then
+                    lane_ok=1
+                    break
+                fi
+                sleep 10
+            done
+            if [[ "$lane_ok" != 1 ]]; then
+                echo "ERROR: NPU model $nm answered no vector on /v1/embeddings" >&2
+                echo "frank-dragoon: read the boot ledger: tail -100 /home/dank/logs/frank_dragoon.log" >&2
+                return 1
+            fi
+            echo "frank-dragoon: NPU lane verified — $nm embedded"
+        done
+    fi
 }
 
 # deploy_llm_zealot — frank-zealot is llama.cpp (PrismML fork build) on the
@@ -1463,6 +1772,12 @@ LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-prism-b10
 # 868.32 MiB worst-case VRAM (595.29 model + 248.10 compute in the boot ledger)
 # lands AFTER the engine's fit check, so it is not inside the q8_0 budget.
 LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf}"
+# The refusal adapter is part of the payload identity now, so the deployer
+# checks it beside the other two files — but it has its OWN fetcher, so a
+# missing adapter must not send the operator to the bonsai9 script. Integrity
+# (bytes + sha256) is the fetcher's job at download time; this gate only asks
+# whether the file the env names exists, exactly like the two above.
+LLM_ZEALOT_LORA="${LLM_ZEALOT_LORA:-/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf}"
 
 deploy_llm_zealot() {
     # Refuse rather than restart-loop: a unit whose runtime or payload is
@@ -1484,6 +1799,13 @@ deploy_llm_zealot() {
             return 1
         fi
     done
+    if [[ ! -f "$LLM_ZEALOT_LORA" ]]; then
+        echo "ERROR: frank-zealot refusal adapter absent at $LLM_ZEALOT_LORA" >&2
+        echo "frank-zealot: run scripts/download-frank-zealot-uncensored-lora.sh first;" >&2
+        echo "frank-zealot: the env names the adapter, so the unit would refuse to boot" >&2
+        echo "frank-zealot: without it — starting the unit anyway is a crash loop." >&2
+        return 1
+    fi
     relink_unit frank-zealot
     run_privileged systemctl daemon-reload
     run_privileged systemctl enable frank-zealot.service
@@ -1815,7 +2137,9 @@ deploy_service() {
         copyparty) deploy_copyparty ;;
         plex) deploy_plex ;;
         tautulli) deploy_tautulli ;;
-        prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
+        alloy) deploy_alloy ;;
+        prometheus|grafana|loki|node-exporter|cadvisor|alertmanager|blackbox-exporter) deploy_observability "$service" ;;
+        gpu-exporter) deploy_gpu_exporter ;;
         comfyui) deploy_comfyui ;;
         *) deploy_compose_service "$service" ;;
     esac

scripts/deploy.sh

@@ -133,13 +133,15 @@ fi
 valid_service() {
     local service="$1"
     case "$TARGET:$service" in
+        # dankclaw:searxng-network is client-only: a network quadlet with no
+        # remote dispatch handler, so the remote gate does not (must not) list it.
         dankclaw:caddy|dankclaw:gluetun|dankclaw:searxng-network|dankclaw:searxng-valkey|\
         dankclaw:searxng|dankclaw:marreta|dankclaw:pihole|dankclaw:open-webui|dankclaw:gitea-postgres|\
         dankclaw:gitea-server|dankclaw:homepage|\
         dankclaw:comfyui|dankclaw:hermes|dankclaw:hermes-dashboard|\
         dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|dankclaw:loki|dankclaw:alloy|\
-        dankclaw:alertmanager|\
-        dankclaw:node-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
+        dankclaw:alertmanager|dankclaw:blackbox-exporter|\
+        dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|dankclaw:seerr|\
         dankclaw:open-notebook|\
         dankclaw:frank-tts-engine|dankclaw:frank-tts|\
@@ -147,8 +149,8 @@ valid_service() {
         dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
         dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
         dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\
-        dankserver:lidarr|dankserver:readarr|dankserver:node-exporter|\
-        dankserver:frank-archon|\
+        dankserver:lidarr|dankserver:readarr|dankserver:node-exporter|dankserver:cadvisor|dankserver:gpu-exporter|\
+        dankserver:prowlarr|dankserver:frank-archon|\
         frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
         frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
         frankbook-pro:readarr)

scripts/derive-protected-experts.py

@@ -20,9 +20,16 @@ of the Gram is its down-input energy. Rank halves: 0-255, 256-511
 Output: the cast-rule lines (recipe format) on stdout, a report on stderr.
 
 Usage: derive-protected-experts.py <calib dir> [--min-share 0.03]
-       [--min-route-mult 3.0]
+       [--min-route-mult 3.0] [--target-count 0]
+       --target-count N fills the byte bill: the bars pick the base set exactly as
+       the bar-only pass does, then the energy ranking admits experts one at a time
+       (each with its parity partner) until the count lands. The energy bar NEVER
+       relaxes, so asking for 48 bf16 experts never quietly buys 80; the ratio bar
+       walks down only when the ranking cannot produce that many candidates.
+       0 takes the bars as given.
 """
 import argparse
+import collections
 import glob
 import os
 import re
@@ -32,6 +39,7 @@ import sys
 import numpy as np
 
 NAME_RE = re.compile(r"^gram\.r(\d)\.blk\.(\d+)\.ffn_down_exps\.(\d+)\.bin$")
+CAST_RE = re.compile(r"^blk\.(\d+)\.ffn_\*_exps\.(\d+)\.weight")
 
 
 def read_gram(path):
@@ -49,27 +57,8 @@ def read_gram(path):
     return k, rank, int(rows), tr
 
 
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("calib_dir")
-    ap.add_argument("--min-share", type=float, default=0.03)
-    ap.add_argument("--min-route-mult", type=float, default=3.0)
-    a = ap.parse_args()
-
-    # layer -> global expert -> (rank, rows, energy)
-    data = {}
-    for path in sorted(glob.glob(os.path.join(
-            a.calib_dir, "gram.r*.ffn_down_exps.*.bin"))):
-        m = NAME_RE.match(os.path.basename(path))
-        if not m:
-            continue
-        rank, layer, expert = int(m.group(1)), int(m.group(2)), int(m.group(3))
-        k, _, rows, tr = read_gram(path)
-        data.setdefault(layer, {})[expert] = (rank, rows, tr)
-
-    if not data:
-        raise SystemExit("no per-expert down grams found")
-
+def select(data, min_share, min_route_mult):
+    """(cast lines, report lines, count) for one pair of bars."""
     lines, report = [], []
     total = 0
     for layer in sorted(data):
@@ -83,7 +72,7 @@ def main():
         for e, (rank, rows, tr) in experts.items():
             es, rs = tr / tot_e, rows / max(tot_r, 1)
             rec = (e, rank, es, rs)
-            if es >= a.min_share and es >= a.min_route_mult * rs:
+            if es >= min_share and es >= min_route_mult * rs:
                 qual.append(rec)
             elif es > 0:
                 pool.append(rec)
@@ -107,10 +96,148 @@ def main():
                           f"energy {es*100:5.2f}%, routing {rs*100:5.3f}%, "
                           f"ratio {es/max(rs,1e-12):6.1f}")
         total += len(chosen)
+    return lines, report, total
+
+
+def parity_repair(chosen, by_half, report, only=None):
+    """Add the next-best candidate to every half — or to `only` — holding an odd count.
+
+    The four-bit GEMM takes each half's remainder as two equal tables, so an odd count
+    of bf16 experts on a half is not a style problem: the two tables would not match.
+    `by_half` is energy-descending, so the first unchosen candidate is the next most
+    energetic, which is the expert worth keeping. Returns how many were added.
+    """
+    added = 0
+    for key in ([only] if only is not None else sorted(by_half)):
+        layer, half = key
+        n = sum(1 for (l, x) in chosen if l == layer and x // 256 == half)
+        if n % 2 == 0:
+            continue
+        for e, es, rs in by_half.get(key, []):
+            if (layer, e) not in chosen:
+                chosen[(layer, e)] = (es, rs)
+                added += 1
+                report.append(f"blk.{layer}: half {half} odd -> added e{e} "
+                              f"(energy {es*100:.2f}%)")
+                break
+        else:
+            report.append(f"blk.{layer}: half {half} holds {n} and no candidate is "
+                          f"left to make it even")
+    return added
+
+
+def select_to_count(data, min_share, min_route_mult, target):
+    """(lines, report, total, cutoff) for the set that LANDS the count.
+
+    The bars decide the base set; the count is then filled from the energy ranking,
+    one expert at a time, each addition taking its half's next-best candidate with it
+    so every rank's half stays even. Only the RATIO bar walks down to find candidates
+    — it is the law that says an expert is disproportionate and not merely busy, and
+    the caller's byte bill has to be filled — while the ENERGY bar holds, because that
+    bar is what the extra GiB of bf16 is supposed to buy. `cutoff` is the energy share
+    the set bottoms out at: the bar this campaign actually converted with.
+    """
+    report = []
+    shares, ranked = {}, []
+    for layer in sorted(data):
+        experts = data[layer]
+        tot_e = sum(v[2] for v in experts.values())
+        tot_r = sum(v[1] for v in experts.values())
+        if tot_e <= 0 or tot_r <= 0:
+            report.append(f"blk.{layer}: empty recording, skipped")
+            continue
+        for e, (_rank, rows, tr) in experts.items():
+            es, rs = tr / tot_e, rows / max(tot_r, 1)
+            if es <= 0:
+                continue
+            shares[(layer, e)] = (es, rs)
+            ranked.append((es, layer, e, rs))
+    ranked.sort(key=lambda r: (-r[0], r[1], r[2]))
+    by_half = {}
+    for es, layer, e, rs in ranked:
+        by_half.setdefault((layer, e // 256), []).append((e, es, rs))
+
+    chosen = {k: v for k, v in shares.items()
+              if v[0] >= min_share and v[0] >= min_route_mult * v[1]}
+    total = len(chosen)
+    total += parity_repair(chosen, by_half, report)
+
+    mult = min_route_mult
+    while True:
+        pool = [(es, l, e, rs) for es, l, e, rs in ranked
+                if es >= mult * rs and (l, e) not in chosen]
+        if len(pool) + total >= target or mult <= 0.0:
+            break
+        mult -= 0.5
+        report.append(f"ratio bar {mult + 0.5:g} leaves {len(pool)} fill candidates "
+                      f"for {target - total} slots; relaxing toward 0")
+    for es, layer, e, rs in pool:
+        if total >= target:
+            break
+        if (layer, e) in chosen:
+            continue
+        chosen[(layer, e)] = (es, rs)
+        total += 1
+        report.append(f"blk.{layer}: count fill -> added e{e} (energy {es*100:.2f}%)")
+        total += parity_repair(chosen, by_half, report, only=(layer, e // 256))
+    lines, cutoff = [], None
+    for (layer, e) in sorted(chosen):
+        es, rs = chosen[(layer, e)]
+        cutoff = es if cutoff is None else min(cutoff, es)
+        lines.append(f"blk.{layer}.ffn_*_exps.{e}.weight         cast dtype=bf16")
+    return lines, report, total, cutoff
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("calib_dir")
+    ap.add_argument("--min-share", type=float, default=0.03)
+    ap.add_argument("--min-route-mult", type=float, default=3.0)
+    ap.add_argument("--target-count", type=int, default=0)
+    a = ap.parse_args()
+
+    # layer -> global expert -> (rank, rows, energy)
+    data = {}
+    for path in sorted(glob.glob(os.path.join(
+            a.calib_dir, "gram.r*.ffn_down_exps.*.bin"))):
+        m = NAME_RE.match(os.path.basename(path))
+        if not m:
+            continue
+        rank, layer, expert = int(m.group(1)), int(m.group(2)), int(m.group(3))
+        k, _, rows, tr = read_gram(path)
+        data.setdefault(layer, {})[expert] = (rank, rows, tr)
+
+    if not data:
+        raise SystemExit("no per-expert down grams found")
+
+    if a.target_count > 0:
+        lines, report, total, cutoff = select_to_count(
+            data, a.min_share, a.min_route_mult, a.target_count)
+        bar = f"energy cutoff {cutoff:.3g}" if cutoff is not None else "no candidates"
+    else:
+        lines, report, total = select(data, a.min_share, a.min_route_mult)
+        bar = f"min-share {a.min_share:.3g}"
+
+    # The emitted lines ARE the contract with the four-bit GEMM: every rank's half of
+    # every layer must hold an even number of bf16 experts. Judged on the output and
+    # not on the internal sets, so a slip in either pass is caught here rather than at
+    # engine load time.
+    halves = collections.Counter()
+    for rule in lines:
+        m = CAST_RE.match(rule)
+        if m:
+            halves[(int(m.group(1)), int(m.group(2)) // 256)] += 1
+    odd = {key: n for key, n in halves.items() if n % 2}
+    if odd:
+        print("PARITY LAW BROKEN: " + ", ".join(
+            f"blk.{lay} half {half} holds {n} protected experts"
+            for (lay, half), n in sorted(odd.items())), file=sys.stderr)
+        raise SystemExit(2)
 
     print("\n".join(lines))
     print("\n".join(report), file=sys.stderr)
-    print(f"total protected: {total}", file=sys.stderr)
+    print(f"total protected: {total} (target {a.target_count or 'none'}, {bar}, "
+          f"min-route-mult {a.min_route_mult:g})", file=sys.stderr)
 
 
 if __name__ == "__main__":

scripts/destroy-vault.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Destroy the vault's dead weight: era ledgers, retired runtimes, dead experiments.
+
+`AGENTS.md` was treating a measurement as a record worth keeping forever. It is not: a
+throughput figure for a payload that no longer runs, on a runtime that no longer exists,
+is a liability — the next agent spends context on a number that AI progress has already
+obsolete. This pass moves those pages to `.AGENTTRASH/` (repo law: never `rm`) and then
+deletes every reference to them from the surviving pages, including the sentence that
+carried the link when the link was the sentence's only content.
+
+Usage: destroy-vault.py <list-file>
+List file: one vault-relative path per line; a directory path destroys the whole tree.
+"""
+import pathlib
+import re
+import shutil
+import sys
+
+VAULT = pathlib.Path("serverAppConfigs-docs")
+TRASH = VAULT / ".AGENTTRASH" / "20261019-destruction"
+
+
+def targets(list_file):
+    out = []
+    for line in pathlib.Path(list_file).read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        p = VAULT / line
+        if p.is_dir():
+            out += [f for f in sorted(p.rglob("*.md"))]
+        elif p.is_file():
+            out.append(p)
+        else:
+            print(f"  SKIP (missing): {line}")
+    return out
+
+
+def main(argv):
+    if len(argv) < 2:
+        print(__doc__)
+        return 2
+    doomed = targets(argv[1])
+    # Stems come from the trash too, so re-running the pass (or running it after a
+    # partial destruction) still knows which names are dead and keeps scrubbing.
+    stems = {p.stem for p in doomed}
+    if TRASH.is_dir():
+        stems |= {p.stem for p in TRASH.rglob("*.md")}
+    moved = 0
+    for p in doomed:
+        dest = TRASH / p.relative_to(VAULT)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.move(str(p), str(dest))
+        moved += 1
+    print(f"destroyed {moved} pages ({sum(len(p.read_text(errors='replace').splitlines()) for p in TRASH.rglob('*.md'))} lines staged in trash)")
+
+    # Repoint-or-delete: a link to a destroyed page is itself dead information.
+    link_wl = re.compile(r"\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|([^\]]*))?\]\]")
+    link_md = re.compile(r"\[[^\]]*\]\(([^)#]*\.md)(?:#[^)]*)?\)")
+    pages_cleaned = links_dropped = lines_dropped = 0
+    for page in sorted(VAULT.rglob("*.md")):
+        s = str(page)
+        if ".AGENTTRASH" in s or ".obsidian" in s:
+            continue
+        text = page.read_text(errors="replace")
+        lines = text.split("\n")
+        out = []
+        hits = 0
+
+        def dead(target):
+            # NOT pathlib.Path(...).stem: that strips a real suffix off names like
+            # `Lemonade and sd.cpp`, so a link to a destroyed page survives and the
+            # vault then fails its own link check.
+            t = target.strip()
+            if t.endswith(".md"):
+                t = t[:-3]
+            return t in stems or pathlib.PurePosixPath(t).name in stems
+
+        for line in lines:
+            found = [t for t in link_wl.findall(line) if dead(t[0])]
+            for m in link_md.finditer(line):
+                if dead(m.group(1)):
+                    found.append(m.group(1))
+            if not found:
+                out.append(line)
+                continue
+            hits += len(found)
+            stripped = link_wl.sub(
+                lambda m: m.group(2) if (m.group(2) and not dead(m.group(1))) else "",
+                line)
+            stripped = link_md.sub(
+                lambda m: "" if dead(m.group(1)) else m.group(0), stripped)
+            # What is left of the line after the dead links are gone?
+            residue = re.sub(r"[\s\-*•·|:;,.\u2014\u2013]+", "", stripped)
+            if residue:
+                out.append(re.sub(r"\s+$", "", re.sub(r"^(\s*[-*]\s*)[\s:;,.\u2014-]+", r"\1", stripped)))
+            else:
+                lines_dropped += 1
+        if hits:
+            page.write_text("\n".join(out))
+            pages_cleaned += 1
+            links_dropped += hits
+    print(f"cleaned {pages_cleaned} surviving pages: {links_dropped} dead links dropped, "
+          f"{lines_dropped} lines left empty and deleted")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

scripts/download-frank-dragoon-npu.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# download-frank-dragoon-npu.sh — stage the NPU half of frank-dragoon: the
+# vendor-precompiled AIE programs halogen-flash-server runs on the XDNA2 NPU
+# when HALOGEN_NPU_MODELS names them. These are NOT model weights you can
+# quantize yourself: XDNA2 executes only programs compiled by AMD's AIE
+# compiler, so the only source is peonist-ai's per-model NPU repo, and the
+# engine checks every file's size and sha256 against its own catalog at boot
+# (/opt/halogen/npu/models.txt inside the image) and refuses to load a model
+# whose files do not match.
+#
+# The manifest below IS that catalog block (dumped from the pinned 0.18.20
+# image on 2026-10-30; the 0.16.2 block this replaced named different device
+# programs — 10 `u*.elf` and a 5,086,496 B consts.hnpb — so a version bump
+# re-fetches the set or the entrypoint refuses the boot), so the files land in the layout the entrypoint
+# expects — <dir>/<model-id>/devices/*, <dir>/<model-id>/<model-id>.hnpw and
+# <dir>/<model-id>/tokenizer/tokenizer.json — and the endpoint can keep
+# /models read-only and open no outbound connection at boot (HALOGEN_DOWNLOAD
+# stays unset; the vendor's own README: with it unset the container makes no
+# outbound connections at all).
+#
+# The fetch law, same as every sibling script: curl -f (an access error must
+# not become a "successful" 153-byte artifact), the exact byte count and the
+# sha256 from the manifest, --verify-only to re-check a staged set.
+#
+# Adding a second NPU model: read its block out of the image's catalog and add
+# a case below —
+#   podman exec frank-dragoon-halogen grep "^file <model-id> " /opt/halogen/npu/models.txt
+#   podman exec frank-dragoon-halogen grep "^model <model-id> " /opt/halogen/npu/models.txt
+# and add the model id to HALOGEN_NPU_MODELS in frank-dragoon.env.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
+# The env file owns the staging path (it must sit inside the payload dir, which
+# is the container's only /models mount); an exported FRANK_DRAGOON_NPU_DIR wins
+# so the bats seam can stage somewhere disposable.
+DEST="${FRANK_DRAGOON_NPU_DIR:-}"
+if [ -z "$DEST" ] && [ -f "$ENV_FILE" ]; then
+    DEST="$(sed -nE 's/^FRANK_DRAGOON_NPU_DIR=(.*)$/\1/p' "$ENV_FILE" | tail -1)"
+fi
+: "${DEST:?FRANK_DRAGOON_NPU_DIR is not set and $ENV_FILE names no FRANK_DRAGOON_NPU_DIR}"
+HF="https://huggingface.co"
+
+MODELS="${HALOGEN_NPU_MODELS:-}"
+if [ -z "$MODELS" ] && [ -f "$ENV_FILE" ]; then
+    MODELS="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE" | tail -1)"
+fi
+
+VERIFY_ONLY=0
+case "${1:-}" in
+    --verify-only) VERIFY_ONLY=1 ;;
+    "") ;;
+    *) echo "usage: ${0##*/} [--verify-only]   (models come from HALOGEN_NPU_MODELS)" >&2; exit 2 ;;
+esac
+[ -n "$MODELS" ] || { echo "no NPU model named — set HALOGEN_NPU_MODELS or run with it in the environment" >&2; exit 1; }
+
+# manifest <model-id> -> repo revision + one "path bytes sha256" per file.
+manifest() {
+    case "$1" in
+      qwen3-embedding-0.6b)
+        NPU_REPO="peonist-ai/halogen-npu-qwen3-embedding-0.6b"
+        NPU_REV="8fd44c9f18861618a509bea75c51ff8a37e3bd5d"
+        FILES=(
+        "devices/NOTICE.md 3248 0f267b3d2523ddf14997ee58660a492a9eaf10ccf6b08bf19dfdc85b9a478b91"
+        "devices/consts.hnpb 8495512 5af7c1f2f24d45a9054872bba882461cf7f6ffda832e785b700643d6467ad88b"
+        "devices/devices.hnpm 35465 85d870f6b900de1829659955f0dc701d57984580f5e5ec251add5d6969f03e3d"
+        "devices/u0.elf 2975008 f74df12eb8c79e4e90cbedcc1dd8f142219772d400e3a142fde8a95378b105eb"
+        "devices/u1.elf 3266848 ac5a3f89f934951b315b62ab53e1d0a74dbe6396134192641c405d08da332754"
+        "devices/u2.elf 4015328 5e4f0d3342f262816289866d7e8c966f56a9f938c2e12e7341718cc7312aa480"
+        "devices/u3.elf 4326976 94cf5d269fa286d91c26bf4654cedf101c047d23c4495b74c9c129cdc33b210f"
+        "devices/u4.elf 4953792 cbd2796050d4c675f5cb3da117111087d510173e26f262f1b36aab66dd49dc91"
+        "devices/u5.elf 4940112 7bf6a4470546bb888da02af395c07d92a24fb6d43aef27eb2151b3a9eda19e4d"
+        "devices/u6.elf 6174544 10b9d50b635993e1f9c23775c0b94901d922c1e433cd10910e9f97174235f5ac"
+        "devices/u7.elf 2983776 e0b06063b7379d05c85b3f435b819d96fda1223d8025d6d15958854b5711ace9"
+        "devices/u8.elf 3298656 3bad204f782a68b97e27c1172530b13d7fbb0b495fa95f9504530a1048560644"
+        "devices/u9.elf 2983776 875e313280358ce116e2a51a29a2c42794b5a9fadb3d48db702c90d18d838c2c"
+        "devices/u10.elf 3298656 ef2e0f399f149a5476955bcf233c6fa2e896fcef1f86ecb0d8113c952c41a8b2"
+        "devices/u11.elf 3298656 68a2519eee2d6a61ccc4ca2007e3186d44c9eb4e81162d742fe2f1756ec5a528"
+        "devices/u12.elf 3298656 2bdc6abfa58cac1d5795a3ef52fe8a9ab4e7c2a4792822d615499a913401f728"
+        "devices/u13.elf 3298656 4a3ce14f7ba15987b9bcbb4a010083e8aef68cd87c20cbe43403a98263179fe2"
+        "qwen3-embedding-0.6b.hnpw 813032248 b9f9901429df7b46ec17153aa3899f738adbea8e4356080da0c6e6586fb750d1"
+        "tokenizer/tokenizer.json 11423705 def76fb086971c7867b829c23a26261e38d9d74e02139253b38aeb9df8b4b50a"
+        ) ;;
+      *)
+        echo "unknown NPU model '$1' — add its catalog block to manifest() in this script" >&2
+        return 1 ;;
+    esac
+}
+
+for MODEL in ${MODELS//,/ }; do
+    manifest "$MODEL" || exit 1
+    echo "== $MODEL  ($NPU_REPO@$NPU_REV) =="
+    for entry in "${FILES[@]}"; do
+        read -r f want sha <<<"$entry"
+        path="$DEST/$MODEL/$f"
+        mkdir -p "$(dirname "$path")"
+        if [ "$VERIFY_ONLY" -eq 0 ]; then
+            if [ -f "$path" ] && [ "$(stat -c %s "$path")" = "$want" ]; then
+                echo "HAVE  $f ($want bytes) — skipping fetch"
+            else
+                echo "FETCH $f -> $path"
+                # HF answers /resolve/ with a 302 to the Xet CDN and curl
+                # DROPS the Range header across redirects, so resolve the
+                # effective URL first and resume against the CDN (the sibling
+                # 62 GiB fetch hit this on 2026-10-18).
+                url=$(curl -sfIL -o /dev/null -w '%{url_effective}' "$HF/$NPU_REPO/resolve/$NPU_REV/$f") \
+                    || { echo "cannot resolve $NPU_REPO/$f" >&2; exit 1; }
+                # Resume ONLY into a genuinely partial file. A staged file of a
+                # DIFFERENT size is an artifact of another revision (a version
+                # bump re-cuts the device programs), and `-C -` against it
+                # appends the tail to the old bytes: the result is the right
+                # BYTE COUNT with the wrong content, which reads as a corrupt
+                # program, not as a fetch fault (measured 2026-10-30 on
+                # consts.hnpb). Fetch those from zero.
+                resume=()
+                if [ -f "$path" ] && [ "$(stat -c %s "$path")" -lt "$want" ]; then
+                    resume=(-C -)
+                fi
+                curl -fL --retry 5 "${resume[@]+"${resume[@]}"}" -o "$path" "$url"
+            fi
+        fi
+        [ -f "$path" ] || { echo "MISSING $path (run without --verify-only)" >&2; exit 1; }
+        if [ -L "$path" ]; then
+            echo "SYMLINK $path — the serve gate refuses symlinks; replace with a real file" >&2
+            exit 1
+        fi
+        got="$(stat -c %s "$path")"
+        [ "$got" = "$want" ] || { echo "SIZE $path: $got != $want" >&2; exit 1; }
+        got="$(sha256sum "$path" | cut -d' ' -f1)"
+        [ "$got" = "$sha" ] || { echo "SHA256 $path: $got != $sha" >&2; exit 1; }
+        echo "OK   $f ($want bytes, sha256)"
+    done
+done
+echo "DOWNLOAD_DONE $DEST ($MODELS)"

scripts/download-frank-zealot-uncensored-lora.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# Fetch the refusal-direction adapter that frank-zealot applies at load:
+# AtomicChat/Ternary-Bonsai-2-9B-Uncensored-LoRA-GGUF — a **3.2 MB rank-1 LoRA**
+# for prism-ml/Ternary-Bonsai-2-9B, not a checkpoint. Public repo: no auth
+# header, and never a token on the command line (a Bearer token is
+# world-readable in /proc/<pid>/cmdline).
+#
+# IT IS AN EDIT INSIDE THE COMPUTE GRAPH, NOT A SECOND MODEL. The card: B = -r
+# (the refusal direction, row 23 of its own probe matrix), A = P^T(s *
+# H(r^T W_fold)) built from the base pack's folded input basis, applied at every
+# transformer layer (`target_layers = all`, `sites = 64`), with the embedding
+# deliberately untouched because an embedding LoRA crashes this fork's graph.
+# `--lora-scaled <file>:<scale>` is the flag pair: scale 1 is the card's
+# "held-out refusals 0 %" point, scale 0 is the plain model again.
+#
+# THE RUNTIME IS A GATE, NOT A PREFERENCE. The card verified the adapter on
+# `prism-b10777-e3de715` — the exact build frank-zealot runs — and says older
+# forks (e.g. `prism-b10709`) refuse these packs with `unsupported
+# prism.hadamard.version: 2`. The fork is already pinned for the payload's
+# custom tensor type 142; this adapter adds a second reason it cannot move.
+#
+# PROVENANCE ASYMMETRY, STATED HONESTLY. The GGUF's own metadata names the pack
+# the direction was fitted against:
+#   prism.abliterate.base_gguf   = Ternary-Bonsai-2-9B-PTQ1_0.gguf
+#   prism.abliterate.base_sha256 = 3561df7ac7d26a71ffa991ac2db740b10841c3a350aa2676ccd3ff5d428b4e8b
+# frank-zealot serves the PQ2_0 + MTP + Q8_0-embedding packing (its sha256 is
+# c79fcdfa…, see download-frank-zealot-bonsai9.sh), which is NOT that file and is
+# not on this host. The adapter still applies — same tensor names, and the
+# metadata is provenance rather than a checksum gate — but its strength was
+# calibrated on a sibling quantization. What proves the effect here is the
+# behaviour of the endpoint (a refusal probe before and after the swap), not the
+# metadata and not the card's table, which was measured on PTQ1_0.
+#
+# Fresh downloads, no resume; -f/--fail-with-body so an HTTP error is not a
+# success, then byte count and sha256 (this repo is NOT LFS — the file is a
+# plain git object, so the sha below is the content hash of the Hub's bytes as
+# fetched, not an LFS oid). Idempotent: a complete file is skipped.
+#
+# Usage:
+#   scripts/download-frank-zealot-uncensored-lora.sh [DEST_DIR] [--verify-only]
+set -euo pipefail
+
+DEST=""
+VERIFY_ONLY=0
+for arg in "$@"; do
+    case "$arg" in
+        --verify-only) VERIFY_ONLY=1 ;;
+        *) DEST="$arg" ;;
+    esac
+done
+DEST="${DEST:-/home/dank/models/Bonsai2-9B-MTP}"
+
+REPO="AtomicChat/Ternary-Bonsai-2-9B-Uncensored-LoRA-GGUF"
+# name | bytes | sha256 of the Hub's bytes
+FILES=(
+  "Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf|3155424|861b9c099fb5cd81438ba85bad30590847c777b1e18112586be7f4f9ae5fb07b"
+)
+
+mkdir -p "$DEST"
+status=0
+for entry in "${FILES[@]}"; do
+  IFS='|' read -r name size sha <<<"$entry"
+  url="https://huggingface.co/${REPO}/resolve/main/${name}"
+  f="$DEST/$name"
+  have=0
+  if [[ -f "$f" ]] && [[ "$(stat -c%s "$f")" == "$size" ]]; then
+    if sha256sum -c --quiet <(printf '%s  %s\n' "$sha" "$f"); then
+      have=1
+    fi
+  fi
+  if (( have )); then
+    echo "OK        $name"
+    continue
+  fi
+  if (( VERIFY_ONLY )); then
+    echo "MISSING   $name"
+    status=1
+    continue
+  fi
+  echo "FETCHING  $name (from $REPO)"
+  tmp="$f.part"
+  curl -fsSL --fail-with-body -o "$tmp" "$url"
+  mv "$tmp" "$f"
+  if [[ "$(stat -c%s "$f")" != "$size" ]]; then
+    echo "SIZE FAIL $name ($(stat -c%s "$f") != $size)"
+    status=1
+    continue
+  fi
+  if ! sha256sum -c --quiet <(printf '%s  %s\n' "$sha" "$f"); then
+    echo "SHA FAIL  $name"
+    status=1
+    continue
+  fi
+  echo "VERIFIED  $name"
+done
+if (( status == 0 )); then
+  echo "DOWNLOAD_DONE $DEST"
+else
+  echo "DOWNLOAD_FAILED $DEST" >&2
+fi
+exit "$status"

scripts/download-qwen38-bf16.sh

@@ -0,0 +1,249 @@
+#!/usr/bin/env bash
+# download-qwen38-bf16.sh — RUNS ON DANKSERVER. Fetch and verify the vendor
+# BF16 checkpoint Qwen/Qwen3.8-Flash-Next: the source the five-bit campaign
+# converts, and the source the KLD reference is recorded from.
+#
+# Why the base and not a finetune: the campaign converts to compare against the
+# publisher's own iq4r container, and that container was built from this
+# checkpoint. Base-in, base-out; anything else confounds the gate. (The
+# predecessor campaign ran ukisai/Swift-1.5-Qwen3.8-Flash-Next-BF16, whose key
+# set is identical to this repo's but whose weights are not — none of its
+# calibration grams, KLD reference or protected list carries over.)
+#
+# Why re-fetch instead of reusing: every BF16 copy of this family was destroyed
+# in the 2026-10-15 storage purge — only the 113.6 GiB served .rad survived.
+# 335.3 GiB of safetensors, 131 shards, ungated (the predecessor campaign
+# verified the key set against this repo, so no gate letter is needed).
+#
+# Verification is four gates deep, because the failure this repo actually lived
+# through is a fetch that reported success: `curl -o FILE URL` exits 0 on an
+# HTTP 401/403 and writes the error body as the artifact — three shards of
+# identical tiny size and a cheerful DONE in 20 seconds for 104 GiB.
+#
+#   1. shard census: the file count must equal the expected shard count, so a
+#      partially fetched fragment tree (the .partNofM trap) cannot pass.
+#   2. per-shard completeness, from the shard's OWN safetensors header: the
+#      header declares where the tensor data ends, so every shard's true size is
+#      known offline. A torn tail is named here, by file, instead of surfacing
+#      as a load error five hours later.
+#   3. index coverage: every shard model.safetensors.index.json references
+#      exists and is named.
+#   4. total bytes against the number the Hub declared, then, unless --no-hub,
+#      every file's sha256 against the Hub's LFS record — including the shards
+#      already on disk, so a previous partial download is never trusted.
+#
+# The token never enters a command line (the 2026-09-16 leak law): it is read
+# into HF_TOKEN in the environment, where a local process listing is the only
+# reader. This repo is ungated, so the fetch simply runs without one when no
+# token file exists. scripts/hfget.sh is curl+jq only — no Python, no venv, no
+# hf_transfer — and it drops the pickle/ONNX weight duplicates by itself.
+#
+# usage: download-qwen38-bf16.sh [--fetch] [--no-hub] [--dir DIR]
+#                                [--shards N] [--bytes B] [--repo ID]
+#   --fetch    download first (resumable: re-run to continue), then verify
+#   --no-hub   skip the Hub-side sha256 sweep; the other gates always run
+# exit: 0 verified, 1 verification failed, 2 environment / usage error
+set -euo pipefail
+
+REPO="${FRANK_QWEN38_HF_REPO:-Qwen/Qwen3.8-Flash-Next}"
+LOCAL_DIR="${FRANK_QWEN38_CKPT:-/home/dank/models/Qwen3.8-Flash-Next}"
+# Hub census of 2026-10-27: 144 files, 131 of them .safetensors, together
+# 360,023,351,514 bytes; the shards alone are 360,000,192,888. This gate counts
+# shards only, so a config file edited on the Hub cannot fail it.
+EXPECTED_SHARDS="${FRANK_QWEN38_SHARDS:-131}"
+EXPECTED_BYTES="${FRANK_QWEN38_BYTES:-360000192888}"
+MIN_FREE_MIB="${FRANK_QWEN38_MIN_FREE_MIB:-350000}"
+
+FETCH=0
+HUB=1
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --fetch) FETCH=1 ;;
+        --no-hub) HUB=0 ;;
+        --dir)    LOCAL_DIR="$2"; shift ;;
+        --shards) EXPECTED_SHARDS="$2"; shift ;;
+        --bytes)  EXPECTED_BYTES="$2"; shift ;;
+        --repo)   REPO="$2"; shift ;;
+        *) echo "usage: ${0##*/} [--fetch] [--no-hub] [--dir DIR] [--shards N] [--bytes B] [--repo ID]" >&2
+           exit 2 ;;
+    esac
+    shift
+done
+
+SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+FAILS=0
+note() { echo "$@"; FAILS=$((FAILS + 1)); }
+
+if [ "$FETCH" = "1" ]; then
+    # Every fetch gate is a hard stop: skipping the download silently and then
+    # verifying a half tree is how a campaign ships a stale payload.
+    HFGET="$SELF_DIR/hfget.sh"
+    if [ ! -f "$HFGET" ]; then
+        echo "REFUSED: the fetch helper is not here: $HFGET" >&2
+        exit 2
+    fi
+    if ! command -v jq >/dev/null 2>&1; then
+        echo "REFUSED: hfget.sh needs jq on PATH" >&2
+        exit 2
+    fi
+    mkdir -p "$LOCAL_DIR"
+    # The campaign's peak-disk law starts at the fetch: the checkpoint and the
+    # reference container built from it are ~335 GiB each, and a tree that does
+    # not fit is discovered hours into a transfer, when it costs the most.
+    free_mib=$(df -m --output=avail "$LOCAL_DIR" | tail -1 | tr -dc '0-9')
+    if [ "${free_mib:-0}" -lt "$MIN_FREE_MIB" ]; then
+        echo "REFUSED: $free_mib MiB free at $LOCAL_DIR; the fetch wants $MIN_FREE_MIB MiB free" >&2
+        echo "         (retire the peak-disk predecessors first — nothing is downloaded, so" >&2
+        echo "         nothing is left half-written)" >&2
+        exit 2
+    fi
+    echo "fetch: $REPO -> $LOCAL_DIR ($free_mib MiB free)"
+    if [ -z "${HF_TOKEN:-}" ]; then
+        CREDSTORE="$HOME/.config/huggingface/credentials/https://huggingface.co"
+        if [ -f "$CREDSTORE" ]; then
+            HF_TOKEN="$(awk '/^ OauthToken: / {print $2; exit}' "$CREDSTORE")"
+        elif [ -f "$HOME/.cache/huggingface/token" ]; then
+            # The legacy single-line token file the hub library writes.
+            HF_TOKEN="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")"
+        elif [ -f "$HOME/.hf_token" ]; then
+            HF_TOKEN="$(tr -d '[:space:]' < "$HOME/.hf_token")"
+        fi
+    fi
+    if [ -n "${HF_TOKEN:-}" ]; then
+        export HF_TOKEN
+    fi
+    bash "$HFGET" "$REPO" "$LOCAL_DIR" main
+    unset HF_TOKEN
+fi
+
+if [ ! -d "$LOCAL_DIR" ]; then
+    echo "REFUSED: $LOCAL_DIR does not exist (run with --fetch, or pass --dir)" >&2
+    exit 1
+fi
+INDEX="$LOCAL_DIR/model.safetensors.index.json"
+if [ ! -f "$INDEX" ]; then
+    echo "REFUSED: $INDEX is missing — the checkpoint tree is not addressable without it" >&2
+    exit 1
+fi
+
+echo "== gate 1: shard census ($LOCAL_DIR)"
+SHARDS=$(find "$LOCAL_DIR" -maxdepth 1 -type f -name '*.safetensors' | wc -l)
+if [ "$SHARDS" != "$EXPECTED_SHARDS" ]; then
+    note "FAIL SHARD_COUNT: $SHARDS safetensors files on disk, expected $EXPECTED_SHARDS"
+else
+    echo "  ok: $SHARDS shards"
+fi
+
+echo "== gate 2: every shard is whole, judged by its own header"
+# A safetensors file is [u64 header length][JSON header][tensor data]. The JSON
+# declares where the data ends, so the file's true size is computable offline:
+# 8 + header + the largest end offset. Truncation and error bodies both fall
+# here, and each is reported with the file that caused it.
+while IFS= read -r f; do
+    state=$(python3 - "$f" <<'PY'
+import json, os, struct, sys
+path = sys.argv[1]
+size = os.path.getsize(path)
+try:
+    with open(path, "rb") as fh:
+        (hlen,) = struct.unpack("<Q", fh.read(8))
+        if hlen == 0 or hlen > size:
+            print("NOT_SAFETENSORS")
+            raise SystemExit
+        header = json.loads(fh.read(hlen))
+except Exception:
+    print("NOT_SAFETENSORS")
+    raise SystemExit
+if not header:
+    print("NOT_SAFETENSORS")
+    raise SystemExit
+end = max((v["data_offsets"][1] for v in header.values()
+           if isinstance(v, dict) and "data_offsets" in v), default=0)
+want = 8 + hlen + end
+print("OK" if want == size else f"TRUNCATED want={want} have={size}")
+PY
+)
+    case "$state" in
+        OK) ;;
+        NOT_SAFETENSORS*) note "FAIL NOT_SAFETENSORS: $(basename "$f") is not a safetensors file (an error body written as an artifact?)" ;;
+        TRUNCATED*) note "FAIL TRUNCATED: $(basename "$f") $state" ;;
+    esac
+done < <(find "$LOCAL_DIR" -maxdepth 1 -type f -name '*.safetensors' | sort)
+if [ "$FAILS" -eq 0 ]; then
+    echo "  ok: every shard's data extent matches its size on disk"
+fi
+
+echo "== gate 3: the index's shards are all present"
+INDEX_SHARDS=$(python3 - "$INDEX" <<'PY'
+import json, sys
+map_ = json.load(open(sys.argv[1]))["weight_map"]
+print("\n".join(sorted(set(map_.values()))))
+PY
+)
+while IFS= read -r name; do
+    [ -n "$name" ] || continue
+    if [ ! -f "$LOCAL_DIR/$name" ]; then
+        note "FAIL MISSING: $name is named by the index and is not on disk"
+    fi
+done <<< "$INDEX_SHARDS"
+if [ "$FAILS" -eq 0 ]; then
+    echo "  ok: $(printf '%s\n' "$INDEX_SHARDS" | grep -c .) referenced shards on disk"
+fi
+
+echo "== gate 4: total bytes vs the Hub's declaration"
+TOTAL=$(find "$LOCAL_DIR" -maxdepth 1 -type f -name '*.safetensors' -printf '%s\n' | awk '{s+=$1} END{print s+0}')
+if [ "$TOTAL" != "$EXPECTED_BYTES" ]; then
+    note "FAIL TOTAL_BYTES: shards total $TOTAL bytes, expected $EXPECTED_BYTES"
+else
+    echo "  ok: $TOTAL bytes"
+fi
+
+if [ "$HUB" = "1" ]; then
+    echo "== gate 5: every file's sha256 vs the Hub's LFS record"
+    # size = file size; lfs.sha256 = sha256 of the file itself (LFS or not).
+    TMPD="$(mktemp -d)"
+    trap 'rm -rf "$TMPD"' EXIT
+    MANIFEST="$TMPD/tree.json"
+    PLAN="$TMPD/plan"
+    curl -sf "https://huggingface.co/api/models/$REPO/tree/main?recursive=true&expand=true" \
+        -o "$MANIFEST" || { echo "FAIL: the Hub API call failed — refusing to report success" >&2; exit 1; }
+    python3 - "$MANIFEST" > "$PLAN" <<'PY'
+import json, sys
+for e in json.load(open(sys.argv[1])):
+    if e.get("type") != "file":
+        continue
+    sha = (e.get("lfs") or {}).get("sha256") or e.get("sha") or ""
+    if sha:
+        print(e["path"], e["size"], sha)
+PY
+    BAD=0
+    while read -r rel size sha; do
+        [ -n "${rel:-}" ] || continue
+        if [ ! -f "$LOCAL_DIR/$rel" ]; then
+            note "FAIL MISSING: $rel is on the Hub and not on disk"
+            continue
+        fi
+        have=$(stat -c %s "$LOCAL_DIR/$rel")
+        if [ "$have" != "$size" ]; then
+            note "FAIL SIZE: $rel is $have bytes, Hub says $size"
+            continue
+        fi
+        got=$(sha256sum "$LOCAL_DIR/$rel" | awk '{print $1}')
+        if [ "$got" != "$sha" ]; then
+            note "FAIL SHA256: $rel got $got want $sha"
+            BAD=$((BAD + 1))
+        fi
+    done < "$PLAN"
+    if [ "$BAD" -eq 0 ] && [ "$FAILS" -eq 0 ]; then
+        echo "  ok: sha256 matches for every file"
+    fi
+    rm -rf "$TMPD"
+    trap - EXIT
+fi
+
+if [ "$FAILS" -gt 0 ]; then
+    echo "VERIFY_FAILED: $FAILS problem(s) in $LOCAL_DIR" >&2
+    exit 1
+fi
+echo "VERIFY_OK: $SHARDS shards, $TOTAL bytes of safetensors in $LOCAL_DIR"
+echo "NEXT: build the BF16 reference container with scripts/build-qwen38-bf16-rad.sh"

scripts/frank-archon.env

@@ -1,306 +1,157 @@
-# frank-archon.env — radiance (StillDeadcode, Apache-2.0) serving the
-# PUBLISHER's 4-bit-codebook container of the official Qwen/Qwen3.8-Flash-Next
-# (MODEL_FILE below, fetched by scripts/download-frank-archon-iq4r.sh) on
-# BOTH R9700s (TP=2), dankserver :7770.
-# 2026-10-15 PAYLOAD SWAP: the model moved from the in-house w4dn8 conversion
-# of the Swift1.5 finetune to the engine author's published container
-# qwen3.8-next-flash-fp8-iq4r-moe.rad (116,322 MiB, sha256-verified at fetch —
-# the w4dn8 measured 116,416 MiB, so this shape carries over with 94 MiB to
-# spare). NO argument changed: every number below is the 2026-10-13 shape,
-# kept verbatim. The swap commit is the rollback (the w4dn8 payload stays on
-# disk).
-
+# frank-archon.env — radiance (StillDeadcode) on BOTH R9700s (TP=2),
+# Dankserver :7770, publisher's 4-bit-codebook container of the official
+# Qwen3.8-Flash-Next.
 #
-# SHAPE (2026-10-13): THREE decode slots x 200,000 tokens, fp8 (quantized) KV,
-# with the per-card budget spent on resident experts: 25,600 MiB of slab +
-# 5,120 MiB of pool = 30,720 MiB, inside the 30,969 MiB ceiling the engine's own
-# budget table allows (card 32,624 MiB, settle floor 209 MiB). The pool is sized
-# to the contract and nothing more (3 x 200,000 needs 5,063 MiB at fp8; 5,120
-# predicts 608,027 tokens, 1.3 % slack), because every MiB taken from the slab
-# is prefill and TTFT (2026-10-11 A/B: slab -14.8 % cost prefill -29 %, TTFT
-# +25 % at the full window) while KV beyond the contract buys nothing.
-# Consequence, deliberate: a FOURTH full-height conversation does not fit this
-# pair, and the engine refuses it — it never truncates. See the KV block for the
-# arithmetic and the measured dtype law. The w4dn8 payload is the swap
-# rollback and keeps its trio under the w4dn8 runbook; the fp8-native payload
-# that came before it was DELETED from disk 2026-10-15 (Frank's call, to free
-# the volume for this download — rebuildable via the fetch + build scripts
-# named in the fp8-native runbook).
-#
-# 2026-10-05 cutover from FreeToken (PR #587 ROCm, one card, experts in
-# pinned host RAM; git tag frank-archon-freetoken-pr587-20261005 is the
-# pre-cutover state — its unit/env/serve trio and Dockerfile are all in the
-# tag and rebuildable). Before that: tcclaviger vLLM fork (tag
-# frank-archon-vllm-shapeG-20261003).
-#
-# WHY RADIANCE: the engine is C++/HIP with libr4d kernels built for
-# gfx1201, drives `podman run` directly from the digest-pinned
-# stilldeadcode/radiance image (287 MiB; host needs only the amdgpu driver),
-# and its qwen4exp plugin reads the modelopt FP8 encoding NATIVELY
-# (`X.weight_scale_inv` is the scale plane — checkpoint.h). The PLE n-gram
-# table (51B params, 47.7 GiB) is served from pinned host RAM via io_uring
-# reads — Frank's call: "Native FP8. Experts from ram. Pin the PLE to ram."
-# THE 4-bit-codebook publisher shape is the DEPLOYED payload since
-# 2026-10-15. The card's own run line (--host-pool-mib 12288,
-# --expert-vs-cache-ratio 0.82, seqs 8, spec 3) is the publisher's
-# suggestion on 32 GB cards; this file's shape is the fleet's measured one.
-#
-# PAYLOAD: /home/dank/models/rad/qwen3.8-next-flash-fp8-iq4r-moe.rad — the
-# official Qwen3.8-Flash-Next bf16 checkpoint rad-converted by the engine's
-# own author (recipe qwen4exp-w4nl64-i8-hc8m: experts as 4-bit codes into a
-# 16-level non-uniform codebook after a 128-point Walsh–Hadamard rotation,
-# GPTQ + 3 coordinate-descent sweeps on a 10M-token calibration, ten
-# bf16-protected experts per layer, int8 W8A8 trunk + lm_head, fp8 HC
-# mixtures, PLE n-gram fp8 fixed-scale, MTP head, bf16 vision tower; card KL
-# 0.0789 all-tokens / top-1 92.85 % vs bf16). Fetched byte-exact + sha256 by
-# scripts/download-frank-archon-iq4r.sh; the recipe/LICENSE stay in the
-# staging dir beside it as provenance. ROLLBACK: revert the swap commit — the
-# in-house qwen4exp-swift15-w4dn8-i8-hc8m.rad (scripts/convert-swift15-w4dn8.sh,
-# KLD-gated) stays on disk. Behind it, the fp8-native
-# qwen4exp-fp8-native.rad was DELETED from disk 2026-10-15 (Frank's call,
-# one-time rm, to free the volume for this payload); rebuild =
-# scripts/download-frank-archon.sh + scripts/build-frank-archon-rad.sh — a
-# NO-RECIPE rad-convert copy of the official 131-shard checkpoint under the
-# descriptor law (no shipped plugin claims (qwen4exp, "fp8"), so the staging
-# config.json drops quantization_config; per-weight encodings are data, not
-# the descriptor).
-#
-# SERVING FACTS (docs/GUIDE.md, docs/DOCKER.md, compose/flashnext.yaml):
-# - The HTTP listener binds AFTER the full load (main.cpp: engine.start()
-#   then engine.serve()) — during load the port refuses connections, so a
-#   bare curl /health gate is honest. /health == /ping: 200 JSON while
-#   serving, 503 only while stopping (core/server/admin.cpp:385).
-# - Reasoning: --reasoning-format defaults to auto, emitting thinking in
-#   `reasoning_content` (config.cpp). Tool calls: the qwen3.8 dialect, from
-#   the chat format. Both need no parser flags.
-# - Sampling: the OpenAI endpoint accepts temperature, top_p, top_k, min_p,
-#   typical_p, presence_penalty, frequency_penalty, repetition_penalty,
-#   repeat_last_n, chat_template_kwargs, reasoning_effort,
-#   enable_thinking, preserve_thinking (core/server/oai.cpp:194-205) —
-#   models.yml's extraBody set is fully inside it. Unknown params are
-#   refused (400), which is why nothing unverified rides the wire.
-# - MTP: the .rad carries the model's mtp.* head (the publisher's recipe
-#   names the mtp.fc_hidden/fc_embedding/draft_head rules; card default
-#   depth 3). Served at the depth named by FRANK_ARCHON_SPEC_TOKENS below —
-#   the depth this shape's decode ladder was measured at.
-# - The fused TP all-reduce is declared over max-num-seqs x
-#   (1 + --num-speculative-tokens) rows; 3 x (1+2) = 9 rows, under the 25-row
-#   exactness bound of the wht6 wire.
-# Engine pin: Codeberg tag v1.0.13 (commit 53282fa), Docker Hub tag 1.0.13,
-# index digest below. REJECTED UPGRADE: 1.2.0 (sha256:665775ed1d90de6dbb
-# 25558e07fc60d20328f1346f96f3c6e00845eb763f01fa, tag 1.2.0, commit
-# e95d397) was benched 2026-10-06 on this exact shape (fp8-native rad,
-# vram pair, spec 2, seqs 3, 43.6k greedy probe): decode 29.21 t/s and
-# draft acceptance 0.7123 vs the 1.0.13 control's 30.08 and 0.7549.
-# Acceptance on the greedy harness is deterministic (the control
-# reproduced the recorded 0.755 exactly), so the drop is a real numerics
-# change — the 1.2.0 MoE weight-rotation change (92b8aaf) is the named
-# suspect; decode loss tracks the acceptance ratio (0.943 vs 0.945).
-# The vram-pair translation itself is exact (1.2.0 boot ledger took
-# vram_weights 25.46 GiB / vram_kv 3.46 GiB per card byte-identical to
-# the 1.0.13 ledger at EVC 0.82). Re-test 1.2.0+ when the rotation change
-# is untangled; 1.2.0 also REMOVED --expert-vs-cache-ratio in favour of
-# the --vram-weights-mib / --vram-kv-mib pair (core/config.cpp) — this
-# file already uses the pair, which BOTH pins accept (rollback symmetry).
-FRANK_ARCHON_IMAGE=stilldeadcode/radiance@sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e
-# The .rad container FILE (mounted under /models) and the state dir for the
-# prefix-cache disk tier (mounted rw at /data). The model dir holds ONLY
-# .rad files; the raw HF download and its staging dir live beside it and
-# are never mounted.
+# ONE OWNER: shape, payload identity, eras, laws and every measured number live
+# in serverAppConfigs-docs/LLM/frank-archon.md. This file carries the knobs plus
+# only the reason a reader editing THIS line cannot recover from the page.
+# Where the two disagree, this file wins on values; the boot ledger arbitrates
+# the engine shape.
+FRANK_ARCHON_IMAGE=stilldeadcode/radiance@sha256:d590e300234bf6d2f942a0b8d456d183153fd808f28820e6c2fa69c9389178ed
+# 1.8.1 (2026-10-18), pinned by digest. Two CLI breaks sit between it and the
+# 1.0.13 line this endpoint ran for two eras, and both are boot refusals, not
+# warnings: --vram-weights-mib/--vram-kv-mib were REMOVED at 1.3.0 (the card is
+# now split by the engine's budget resolver — see below), and the prefix cache
+# lost its disk tier. Draft acceptance has to be re-scored on any pin past 1.2.0:
+# 1.2.0's MoE weight rotation measured 0.7123 against the 0.7549 control, and the
+# suspect is untangled (the page's Dead ends section names it).
+# The .rad file mounts under /models; the tune cache mounts separately below.
+# The model dir holds ONLY .rad files — the HF download and staging stay beside it.
 FRANK_ARCHON_MODEL_DIR=/home/dank/models/rad
 FRANK_ARCHON_MODEL_FILE=qwen3.8-next-flash-fp8-iq4r-moe.rad
 FRANK_ARCHON_STATE_DIR=/home/dank/radiance-state
-# The state dir's tune/ subdirectory holds rad-tune's kernel-selection
-# cache (the engine's selector reads $RADIANCE_HOME/tune/<machine>.tune
-# at boot; $RADIANCE_HOME defaults to /opt/radiance, in the image, so the
-# cache must ride a host mount — serve script mounts it, see the runbook's
-# "Tuning levers").
-# Container identity is the SERVICE name (Frank's call, 2026-10-05): one
-# name across systemctl and podman ps. The runtime changes; the name does
-# not. The legacy list retires every predecessor identity that could hold
-# the GPUs or the port.
+# state dir's tune/ subdir carries rad-tune's kernel-selection cache; the engine
+# reads $RADIANCE_HOME/tune/<machine>.tune and that defaults to /opt/radiance
+# (inside the image), so the cache only survives on a host mount.
+# Container identity is the SERVICE name: one name across systemctl and
+# `podman ps`; the runtime changes, the name does not. The legacy list retires
+# every predecessor identity that could hold the GPUs or the port.
 FRANK_ARCHON_CONTAINER=frank-archon
 FRANK_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-probe"
-# Served model id — Protoss name (wire rename 2026-10-09). The container,
-# unit, scripts and this file keep the frank-archon name above; only the
-# alias on the wire moved. Radiance validates the requested model name, so
-# every client (harness, bench scripts, probes) MUST send this value.
+# Served id. Radiance VALIDATES the requested model name, so every client must
+# send exactly this value (unlike llama.cpp and halogen, which ignore it).
 FRANK_ARCHON_SERVED_NAME=frank-archon
 FRANK_ARCHON_PORT=7770
-# Frank's chat-template call: froggeric's fixed Qwen3.8 template (the
-# repo's root chat_template.jinja, vendored here; the checkpoint's own
-# template is baked into the .rad by rad-convert and this overrides it).
-# Radiance derives the reasoning/tool-call reply format from it.
+# Frank's chat-template call: froggeric's fixed Qwen3.8 template, vendored in
+# the repo; it overrides the template baked into the .rad, and radiance derives
+# the reasoning and tool-call reply format from it.
 FRANK_ARCHON_TEMPLATE=/home/dank/serverAppConfigs/scripts/qwen38-froggeric-v22.5.jinja
 
 # --- engine shape (the boot ledger arbitrates every number below) ---------
-# Tensor parallel: BOTH R9700s. The board swap (2026-09-20, ProArt B850
-# x8/x8 on root ports) made collectives legal on this host; radiance's TP
-# wire is libr4d's own, not RCCL.
+# TP=2 needs the 2026-09-20 board (x8/x8 on root ports): a collective cannot be
+# dispatched on a card whose PCIe chain lacks atomics. Radiance's wire is
+# libr4d's own, not RCCL.
 FRANK_ARCHON_TP=2
-# Lossy WHT 6-bit wire for large all-reduces (publisher's production choice:
-# +5.6-6.3% prefill, ~-1.3% 8-stream decode at TP2). min-kb stays the
-# engine default.
+# Lossy WHT 6-bit wire for large all-reduces (publisher's production choice). It
+# stays EXACT for single-sequence decode and any step of <= 25 rows — see
+# MAX_SEQS x (1 + SPEC_TOKENS) below.
 FRANK_ARCHON_TP_WIRE=wht6
-# Decode slots: THREE, so the served window is a CAPACITY CONTRACT — three
-# concurrent conversations of --max-model-len fit the KV pool (see the KV
-# block below). The "seqs 2 cannot boot" law is NOT radiance's: it was the
-# tcclaviger vLLM fork's torch.compile shape specialization on
-# query_start_loc (ConstraintViolationError, measured 2026-10-02, retired
-# with that runtime). Radiance booted seqs 2 at the fp8-native first boot,
-# and the 2026-10-06 decode sweep measured seqs 2 and 3 statistically tied
-# (30.9-33.1 t/s, inside the +-3% noise floor) — slot count buys concurrency,
-# not speed, so a slot costs KV, never decode.
-# Cost of each extra slot: 110 MiB of per-slot recurrent state inside the KV
-# pool, and the fused TP all-reduce is declared over seqs x (1 + spec) rows —
-# 3 x 3 = 9 here, under the 25-row bound at which the wht6 wire stays EXACT
-# (single-sequence decode and any step of <= 25 rows is bit-exact; only busier
-# steps are lossy).
-# The pool is sized to this contract and nothing more (see the KV block):
-# 3 x 200,000 at fp8 = 5,063 MiB, pool 5,120, and the freed 1,792 MiB went into
-# the resident expert slab. The trade is explicit and it is Frank's call:
-# a FOURTH full-height conversation no longer fits (5,120 MiB of pool backs
-# ~608k tokens; 4 x 200,000 needs 6,601), so a fourth 200k request is refused
-# as too large to fit — it is NOT squeezed, paged, or truncated. Third-party
-# traffic that wants four full-height sessions is a --vram-kv-mib change, not
-# a code change.
-FRANK_ARCHON_MAX_SEQS=3
-# Served window. Native is 262,144; 200,000 is the publisher's production
-# value (compose) and what models.yml's contextWindow must quote — a longer
-# prompt is refused by the engine, never truncated. The engine treats
-# `kv group tokens >= max-model-len` as a HARD limit ("ONE request longer
-# than N tokens is accepted by the API and then refused as too large to ever
-# fit — this is a hard limit, not a backing ratio"), so the pool is sized
-# for seqs x maxlen, not for one sequence.
+# Decode slots. A slot costs KV and pinned RAM, never decode: the cache target is
+# exactly MAX_SEQS x MAX_MODEL_LEN and the checkpoint store asks (1 + what the
+# retention policy keeps) per slot. Ordered 2026-10-20: FOUR slots over a pool of at
+# least 700,000 fp8 tokens, rounding UP if the resolver cannot land it exactly. The
+# target is therefore 4 x 200,000 = 800,000 tokens and HOST_POOL_MIB decides whether
+# the cache gets it (expert_vram_floor, core/place/planner.cpp:100-104: the card must
+# keep expert-plane-minus-host-room). The 2-slot shape was the answer to a
+# 365,204-token pool that backed 1.8 full-height sessions; at this pool the third and
+# fourth conversations are backed by the cache instead of over-committing it.
+FRANK_ARCHON_MAX_SEQS=4
+# Served window. The engine treats `kv group tokens >= max-model-len` as a HARD
+# limit, and models.yml contextWindow MUST quote this number (over-length is a
+# refusal, not a truncation). The pool is sized for seqs x maxlen, not one
+# sequence.
 FRANK_ARCHON_MAX_MODEL_LEN=200000
 FRANK_ARCHON_PLACEMENT=expert_tiered
-# Pinned host pool for the expert rows that do not fit VRAM. vram_weights and
-# this pool serve the per-rank resident set TOGETHER: at the fp8-KV pair
-# (26071/3543) the ledger showed the VRAM slab filled to 259 KiB of slack,
-# i.e. the pair was exactly sized. Cutting vram-weights by 3.8 GiB to pay for
-# the bf16 KV pool SPILLS ~3.8 GiB/rank here, so the pool moves with it —
-# under-sizing it is a boot failure (36864 refused the boot in the fp8 era).
-# The boot ledger's coverage line remains the referee, not this arithmetic.
-FRANK_ARCHON_HOST_POOL_MIB=44928
-# VRAM left unclaimed per card (publisher default 96 MiB).
+# Pinned host pool for expert rows that do not fit VRAM, PINNED ONCE PER RANK: the
+# host must pin tp x this number, and the refusal that names it is
+# core/engine_bringup.cpp:250-265. Under the budget resolver this is ALSO the KV
+# lever -- the card keeps expert-plane-minus-this-room, so every MiB added moves one
+# MiB of experts off the card and into the cache. Derived from the boot ledger (a
+# 23.73 GiB floor at 44,928 room puts plane + 2 units at 67.60 GiB a rank), 49,152
+# drops the floor to 19.6 GiB and leaves the cache ~6.9 GiB, which is enough to back
+# the whole 4 x 200,000 target; the leftover returns to the experts. Cost is paid
+# twice over: pinned RAM (tp x 48 GiB, against the TTM target below) and expert
+# residency -- ~3.2 GiB a card of experts now lives in host RAM and is read over the
+# x8 link when the router picks it. The ledger's coverage line is the referee, not
+# this file: if vram_kv comes back under 700,000 tokens, raise this 1024 and redeploy.
+# 2026-10-29 A/B, measured on both payloads and folded back the same day: at 66,560 the
+# endpoint printed the identical `elastic 26.73 GiB / experts 20.72 / kv cache 5.64 /
+# vram_weights 24.30` it had printed at 49,152, opening the bigger pool with `0 B used`;
+# the campaign rig (bf16, same step) moved `vram_kv 269.99 -> 271.25 MiB`, a rounding
+# step — its window is fixed by its own n-gram table (95.37 GiB) and expert plane, not
+# by this pool, so the refused calibration documents were fixed by cutting them to the
+# window (scripts/rechunk-kld-corpus.py), not by widening the pool.
+FRANK_ARCHON_HOST_POOL_MIB=49152
+# VRAM left unclaimed per card (publisher default 96 MiB); the engine's settle
+# floor is built from it.
 FRANK_ARCHON_GPU_HEADROOM_MIB=96
-# VRAM split of claimed memory per card, MiB (0 = derive from the card).
-# THE CEILING, read off this engine's own VRAM-budget table (boot 2026-10-13,
-# pair 24000/5120): card 32,624 MiB; budgets claimed 29,120; `3.42 GiB is
-# neither budgeted nor accounted — the driver's context, the activation arena
-# and --gpu-headroom-mib come out of it; anything past those is throughput on
-# the floor`; and `card 0 settled at 2.01 GiB free against 209.19 MiB (the
-# 96.00 MiB --gpu-headroom-mib and 113.19 MiB kept for the static region)`.
-# So the reserve above the ENGINE's floor is 2,058 - 209 = 1,849 MiB and the
-# ceiling is 29,120 + 1,849 = 30,969 MiB. The 2026-10-11 bound of 29,120
-# ("31,856 - 1,710 outside both budgets - keep 1 GiB free") was MY reserve, not
-# the engine's; the pair below spends the difference and lands 458 MiB above
-# the engine's floor. `weights + kv <= 30,969` is the gate.
-# THE PRICE, and it is paid twice: on this engine idle KV VRAM is ALSO the
-# expert cache's spillover (`mover: 2275 slots can be lent out of the KV
-# cache (2.69 GiB)`, `arena: lends 740 MiB while steps are <= 256 tokens`),
-# so a bigger KV pool steals weight residency from the slab AND from the
-# lendable cache. Priors on this card/model: stealing 2.14 GiB of weights for
-# a bigger pool (MAX_MODEL_LEN 262144) cost 2.5% decode; the EVC ladder's
-# other direction (0.82 -> 0.84, +2.5 GiB weights) cost 8.9%. The 2026-10-11
-# exact-KV A/B measured the slab's own price (22208 vs the fp8-era 26071):
-# prefill -29 %, TTFT at the full window +25 %, decode inside the noise band on
-# 2 of 3 passes, draft acceptance UP 0.661 -> 0.740.
-# 2026-10-13 (late): THREE slots, so the pool is cut to the contract (5,120)
-# and every spare MiB goes to the slab: 22208 -> 24000 (the third slot's
-# surplus KV), then 24000 -> 25600 (the card's settling slack). The direction
-# the A/B data points: prefill is slab-limited, KV beyond the contract is not.
-# The margin is thin by design (608,027 predicted tokens against the 600,000
-# contract), so the boot ledger's `backs N sequences all at 200000 tokens at
-# once` line is the referee for THIS pair and the concurrent probe is its
-# regression test.
-# VERDICT on the 25,600 arm (measured 2026-10-13, scripts/probe-frank-archon-kv.py):
-# the threshold claim is FALSIFIED — the slab is a SLOPE at this end. A-cold
-# 43.5k single-stream: 22,208 -> 3,397, 24,000 -> 3,290, 25,600 -> 3,584 t/s;
-# the 3-way 197k arm's aggregate prefill 3,051 -> 3,589 t/s (wall
-# 193.7 -> 164.5 s). So +1,600 MiB of slab = +8.9 % single-stream and
-# +17.6 % concurrent, both outside the +-3 % noise floor. The 22,208/24,000
-# pair landed inside noise of EACH OTHER, which is what made it read as a
-# plateau; two points are not a curve.
-# The cliff that remains is between 25,600 (3,584) and 26,071 (4,379, the
-# 2026-10-11 fp8 control at seqs 3 / pool 3,543) — +22 % for +471 MiB. At THREE
-# slots that is only reachable by breaking the ceiling (26,112 + 5,120 = 31,232
-# vs 30,969, ~214 MiB above the engine's floor); at TWO slots it is free:
-# pool 3,072 + slab 27,897 = exactly the ceiling, slab above the cliff.
-FRANK_ARCHON_VRAM_WEIGHTS_MIB=25600
-FRANK_ARCHON_VRAM_KV_MIB=5120
-# KV cache QUANTIZED (fp8). This engine's flag takes ONLY bf16|fp8 — there is
-# ONE kv_dtype for the whole cache, so "exact K, quantized V" (llama.cpp's
-# --cache-type-k/--cache-type-v pair, which is what frank-zealot runs) is
-# UNSERVABLE here: the rung below bf16 is fp8 on both halves.
-# Measured per-rank cost, read out of the boot ledger's group table (fp8 boot
-# 2026-10-08, bf16 boot 2026-10-11):
-#   kv_attn     13 layers, 4-token blocks, page 26.00 KiB (fp8) -> 52.00 KiB
-#   kv_qsa_bkey 13 layers, 4-token blocks, page  3.25 KiB at BOTH dtypes —
-#               the QSA block key is an index, not attention state, and does
-#               NOT follow --kv-cache-dtype
-#   => 7.3125 KiB/token fp8, 13.8125 KiB/token bf16: bf16 costs 1.89x fp8,
-#               not 2x, precisely because the bkey plane is dtype-fixed.
-# The dtype-INDEPENDENT terms also live in the pool: kv_gdn_state 54 MiB x 2
-# copies per sequence (+ conv/tail/ple ~= 110 MiB a sequence) and the prefix
-# checkpoint region 8 x 55.99 = 448 MiB. So:
-#   pool_mib >= tokens x 7.3125 KiB + 110 x seqs + 448
-#   3 x 200,000 at fp8 needs 4,285 + 330 + 448 = 5,063 MiB; the pool is 5,120,
-#   and it predicts (5120 - 330 - 448) x 1024 / 7.3125 = 608,027 tokens: the
-#   three-slot contract with 1.3 % slack, every MiB above 5,063 having been
-#   handed to the expert slab instead. The ledger line is the referee ("backs N
-#   sequences all at 200000 tokens at once") — a pair this tight is read off the
-#   boot, never assumed.
-#   At bf16 the same three slots cost 8,093 + 778 = 8,871 MiB, which would cut
-#   the slab to 20,249 MiB (-9 %); at FOUR slots bf16 costs 10,791 + 888 =
-#   11,679 MiB and the slab falls to 17,441 (-33 %). fp8 keeps 22,208 at both
-#   slot counts. The 2026-10-11 bf16 shape (6,912 / seqs 2) backed 462,752
-#   tokens = 2 x 200,000; the historical fp8 pool of 3,543 gave 386,904 (1.93
-#   sessions), and the four-slot fp8 boot of 2026-10-13 19:21 backed 843,276.
-# Secondary saving: the prefix-cache entry shrinks with the attention plane
-# (one entry 110.50 KiB at bf16 -> 58.50 KiB at fp8, measured on the two boot
-# ledgers' `kv tier transfer` lines), so the host tier below parks twice as
-# many conversations as it did at bf16 — no re-size needed to park three slots.
+# There is no VRAM pair any more. --vram-weights-mib/--vram-kv-mib were removed
+# at 1.3.0: the engine takes capacity - already-held - GPU_HEADROOM - arena,
+# puts the static (non-expert) weights off the top, then divides the rest between
+# the expert plane and the KV cache — experts first when the cache has no idle
+# blocks to lend, and never less than ONE MAX_MODEL_LEN sequence for the cache.
+# What moves the split from here is HOST_POOL_MIB (which experts stay resident),
+# MAX_MODEL_LEN/MAX_SEQS (the cache bound) and GPU_HEADROOM_MIB. The slab size
+# the prefill-slope law cares about is a boot-ledger line now, not a knob: read
+# it after every resize instead of asserting it here.
+# fp8 is the ONLY rung below bf16 on this engine: one kv_dtype for the whole
+# cache, so llama.cpp's exact-K/quantized-V split (frank-zealot) is unservable
+# here. bf16 costs 1.89x fp8, not 2x — the QSA block-key plane is dtype-fixed.
 FRANK_ARCHON_KV_DTYPE=fp8
-# Spec decode: the checkpoint's own MTP head, served at depth 2 — the
-# publisher's card ships depth 3 as its peak; this fleet's ladder was
-# measured at 2 and the swap keeps every argument, so 2 stays (rows
-# 3 x (1+2) = 9, inside the wht6 25-row exactness bound).
+# The checkpoint's own MTP head at depth 2: this fleet's ladder was measured at
+# 2, and 3 x (1+2) = 9 rows stays inside the wht6 25-row exactness bound.
 FRANK_ARCHON_SPEC_TOKENS=2
 FRANK_ARCHON_MAX_BATCHED_TOKENS=2048
-# Prefix cache tiers (publisher pattern). The host tier is sized in ENTRIES,
-# and one entry tracks the cache dtype (58.50 KiB at fp8, 117 KiB at bf16, per
-# the ledger's `kv tier transfer` line): at fp8, 8192 MiB parks ~5.8 sessions
-# of 200k (at bf16 the same tier parked ~2.9), so four live slots plus a full
-# parked set fits with the tier as it stands — at RAM the host has (104 GiB
-# available with the service loaded).
-FRANK_ARCHON_PREFIX_HOST_MIB=8192
-FRANK_ARCHON_PREFIX_DISK_MIB=65536
-# Frank's pin law: the PLE n-gram table (47.7 GiB) serves from pinned host
-# RAM the cards read. `ram` REFUSES the start if the memlock limit is too
-# small — the unit's LimitMEMLOCK=infinity is the other half of this law.
+# Step sharing. A prefill chunk that shares a step with decoding sequences holds
+# the device for the chunk's whole time, and every decode row in that step waits
+# that long for its next token: measured at this shape, decode-only steps 15.5 ms
+# against chunk steps 166 ms, which is the harness's ~146 t/s falling to ~11 t/s
+# while another session prefills (law and census: [[frank-archon]]).
+# `auto` is a PREFILL-SPEED dial, not a decoder dial: it reserves one row plus the
+# draft window per decoding sequence (3 tokens) and an eighth of the budget per
+# waiting request or waiting prefill, and the source states it will not halve for
+# decode rows alone. Live proof: prefill_chunks_capped 5,018 across 62,386 chunk
+# steps. A number is the only value that applies whenever the chunk shares a step,
+# and it must not go under the prefill stager's smallest pass -- 1025 rows here,
+# named on the boot line "prefill staging: ... a step of more than 1024 tokens" --
+# because below it every routed layer reads its pooled experts synchronously, so
+# 512 is a trap and the vendor measures the whole job slower there. Interleave
+# inserts N decode-only steps after a chunk step taken beside decoders: it protects
+# their cadence and holds a waiting request out of those steps. That is the decoder
+# dial. ARMED at 1 on 2026-10-20: at N=1 a stream takes its token from the chunk step
+# AND one from the held step, ~2 tokens per 182 ms instead of one per 166 ms; the
+# price falls on the prefilling prompt (its chunks lose every held step) and on any
+# request still waiting to enter. `--prefill-span` re-funds a prefill only when it is
+# ALONE, so it cannot buy this back while decoders run. Meter:
+# radiance:engine_steps_held_total must leave zero, and the §9 buckets must move
+# before N rises.
+FRANK_ARCHON_BUSY_PREFILL_CHUNK=auto
+FRANK_ARCHON_PREFILL_INTERLEAVE=1
+# Prefix cache: pinned host memory only. The disk tier was removed outright
+# (--prefix-cache-dir and --prefix-cache-disk-mib are refusals), so what used to
+# be restored from NVMe is now re-prefilled. The tier is sized in ENTRIES and one
+# entry tracks the cache dtype (58.50 KiB at fp8). Watch prefix_hit_rate after
+# this change: a drop is extra prefill, and prefill is exactly the traffic that
+# costs decode its cadence.
+FRANK_ARCHON_PREFIX_CACHE_MIB=8192
+# Frank's pin law: the PLE n-gram table serves from pinned host RAM the cards
+# read. `ram` REFUSES the start if memlock is too small — the unit's
+# LimitMEMLOCK=infinity is the other half of this law.
 FRANK_ARCHON_NGRAM_PLACEMENT=ram
-# TTM preflight: the pinned ledger (2 x 44 GiB pools + 47.7 GiB PLE + 8 GiB
-# prefix host = ~147 GiB) rides TTM's shared pages, capped by ttm's
-# pages_limit (the current kernel exposes NO pages_max; the cap function
-# takes the min over the knobs that exist). Below the plan the load dies
-# mid-pin with an allocator error instead of this named refusal.
-# THE CAP IS A RUNTIME KNOB — the kernel default is half of RAM, so every
-# raise dies with a reboot. Measured the hard way: the 2026-10-05 manual raise
-# to 52,428,800 pages (200 GiB) was gone after the 2026-10-12 23:17 POWER
-# FAILURE, the cap fell back to 125,988 MiB, and the preflight refused the
-# start 6,919 times in ~20 h while Restart=always re-ran it. The gate in
-# scripts/ttm-preflight.sh now RAISES the knob itself before refusing (the
-# root unit is privileged; the knob is a 0644 root module parameter), so the
-# recovery is a boot, not a visit to the machine.
-FRANK_ARCHON_TTM_MIN_MIB=150000
+# TTM preflight. The pinned ledger rides TTM's shared pages, and the kernel
+# RESETS pages_limit to half of RAM on every boot; below the plan the load dies
+# mid-pin with an anonymous allocator error. ttm-preflight.sh raises the knob
+# itself before refusing by name, so recovery is a boot, not a visit.
+# Floor = 2 x HOST_POOL + 48,824 (static weights + arena) + PREFIX_CACHE + MNBT + the
+# auto checkpoint store (MAX_SEQS x 10 slots of 111.98 MiB), per the bats ledger test;
+# at the 49,152 pool with 4 slots that computes to 161,848 MiB (196,664 at the
+# 66,560 pool the 2026-10-29 A/B rejected).
+FRANK_ARCHON_TTM_MIN_MIB=162000
 FRANK_ARCHON_TTM_TARGET_MIB=204800
-# Escape hatch for boot-ladder experiments. Unset -> no extra argv. Keep
-# multi-flag values as one string; the serve script expands it unquoted on
-# purpose.
+# Escape hatch for experiments. Unset -> no extra argv; the serve script expands
+# this unquoted on purpose, so keep multi-flag values as one string.
 FRANK_ARCHON_EXTRA_ARGS=
-# The .rad's minimum size in MiB (deployer gate; a truncated fetch is a
-# guaranteed long-timeout boot instead of a fast failure). The deployed
-# publisher container measures 116,322 MiB (121,969,901,568 B); the
-# in-house w4dn8 rollback measured 116,416 MiB. Floor sits just under both.
+# Deployer gate: a truncated fetch must be a fast refusal, not a long-timeout
+# boot. Floor sits under the live payload and under every rollback payload
+# (see the page's Eras table for what is on disk and what needs a re-download).
 FRANK_ARCHON_RAD_MIN_MIB=110000

scripts/frank-dragoon.env

@@ -1,209 +1,140 @@
 # frank-dragoon.env — tunables for the frank-dragoon endpoint on Dankclaw's
 # Strix Halo iGPU (gfx1151), host port 7770.
 #
-# RUNTIME (2026-10-17, Frank): halogen-flash-server 0.16.2 — the
-# closed-source C++/HIP engine (image ghcr.io/peonist-ai/halogen-flash-server,
-# digest-pinned below) serving Quat3rnion/halogen-qwen3.8-flash-next-v2-
-# abliterated — the NATIVE v2 conversion (2026-10-11) of orcarouter's pinned
-# BF16 Qwen3.8-Flash-Next, abliterated — as a podman container under a root
-# unit, driven by scripts/serve-frank-dragoon.sh. This CUTS OVER the
-# 2026-10-06 Swift1.5-finetune shape; rollback = revert the swap commit and
-# redeploy: the swift15 payload stays on disk at
-# /home/dank/models/halogen-flashnext-v2/, the 27B-era payload beside it.
+# ONE OWNER: payload identity, shape, eras, laws and every measured number live
+# in serverAppConfigs-docs/LLM/frank-dragoon.md. This file carries the knobs
+# plus only the reason a reader editing THIS line cannot recover from the page.
+# Where the two disagree, this file wins on values; the boot ledger arbitrates
+# the memory figures.
 #
-# MEASURED A/B, both boots on this box, identical battery traffic (engine
-# tools with the service stopped, behavior probes against it, 2026-10-17):
-# NIAH 18/18 vs 17/18, refusal census 0/5 on both, tool calls and the
-# red-PNG vision probe identical, needle hit at both depths of a 21.5k-token
-# haystack, decode 50.0 vs 46.8 t/s, cold prefill 1,604 vs 1,425 t/s,
-# wikitext-2 PPL 3.968 vs 3.877 (the finetune's edge is gone: +2.3 %) and
-# thinking tokens 1,476 vs 784 mean — the finetune compressed reasoning, the
-# base thinks ~1.9x longer per answer.
-#
-# The A/B ran at the card's ctx/pool of 262,144. Afterwards Frank lowered ctx to
-# 200,704 (RAM + the harness's compaction budget) and, on 2026-10-18, raised the
-# pool to 2 x ctx = 401,408 so BOTH slots can carry a full-length request at
-# once; the rates above are the ceiling-262,144 record and the production ledger
-# is the ctx-200,704 one.
-#
-# WHY THE RUNTIME MOVED WITH THE MODEL: the payload is .hgn — per its own
-# card it "will not load in transformers, vLLM, or llama.cpp" — and the v2
-# .hgn loads ONLY in the flash-server line (the old halogen engine is for
-# the 27B p1w4d file; the v2 quickstart pins flash-server). What the move
-# buys (card + live probe evidence): MTP spec decode (53-57 t/s greedy,
-# 3-10x serial), ~1,640 t/s prefill at 34k, ~72% fewer thinking tokens than
-# the censored 27B (905 mean), the ABLITERATED base (refusals should NOT
-# return), IMAGE INPUT (the v2 vision tower loads beside the checkpoint —
-# red-PNG probe verified on the card; the 27B engine ignored its encoder),
-# tool calls verified, and the native 262,144 context — served at 200,704 by
-# choice from 2026-10-17 (see SHAPE: the ceiling is a RAM decision, not a
-# capability limit).
-#
-# SHAPE — the knobs that are NOT free, with the arithmetic:
-#   HALOGEN_CTX=200704 is the PER-REQUEST ceiling and the per-conversation
-#   hard limit — a prompt over it is a 400 naming the limit, never a silent
-#   truncation. LOWERED from 262,144 on 2026-10-17 at Frank's direction (the
-#   checkpoint's native window is 262,144 and the engine serves anything
-#   below it): every position costs KV pool RAM and shrinks the prompt-cache
-#   entry (111 MiB at 262,144, ~85 MiB here). The 400-zone law is INVARIANT
-#   to the value — it says the harness window must sit ~45k UNDER the engine
-#   ceiling, because the harness clamps max_tokens to contextWindow − est −
-#   4096, its chars/4 est UNDER-counts the rendered prompt by ~3 % on
-#   code/agent traffic, and compaction only fires at 85 % (measured 2026-10-07
-#   at the old 200,192 ceiling: est 152,938, server 157,388, over by 354).
-#   Harness contract (models.yml): contextWindow 155648 (= ceiling − 45056,
-#   the gap that absorbs the estimate error), maxTokens 45056.
-#   Compaction fires at 85 % = 132,301 est ≈ 136k measured — the 180k target
-#   of the 262,144 era is unreachable at this ceiling: the window itself is
-#   155,648. Worst-case request: prompt + budget <= 155648 − 4096 + 3 %
-#   est-error ≈ 156.2k <= 200,704 — ~9.5× the 3 % error of margin (robust to a
-#   ~25 % under-count). Compaction rides on frank-zealot: input ≈ threshold −
-#   keepRecent(20k) ≈ 112k, inside its 200,704 slot.
-#   HALOGEN_KV_POOL_POSITIONS=401408 = SLOTS x CTX (2 x 200,704), the top of the
-#   legal band (the seam enforces ctx <= pool <= slots * ctx). Still ONE pool
-#   behind HALOGEN_KV_SLOTS=2 (ledger form: "2 slots over ONE <pool>-position KV
-#   pool, each request up to HALOGEN_CTX"), but sized so BOTH slots can hold a
-#   MAXIMUM-length request at the same time — the parity with frank-zealot's
-#   401,408-position pool that Frank ordered on 2026-10-18. The PER-REQUEST
-#   ceiling does NOT move with it: HALOGEN_CTX stays 200,704, because the
-#   checkpoint's native window is 262,144 and the engine refuses to go past it
-#   without HALOGEN_ROPE_YARN (its own 400 text says so). ~28 KiB/position
-#   (FLAGS.md) ⇒ the DEPLOYED 2026-10-18 boot RESERVES 11.0 GiB (`reserving the
-#   KV pool, 401408 positions (about 11.0 GiB)`), where pool == ctx (200,704)
-#   cost 5.5, 262,144 cost 7.2 and the 2026-10-06 shape's 524,288 measured 14.6.
-#   The preflight formula bills 11.3 (block scratch + ~115 MiB/slot of O(1)
-#   state) — quote the RESERVATION. FLAGS.md law: "RAM the pool
-#   takes is RAM that table loses" — the 47.68 GiB n-gram table surrenders this
-#   much page cache, so the warm prompt-cache ratio is what to re-measure.
-#   HALOGEN_KV_SLOTS=2 restores the w4b-era shape on this box: the MTP
-#   drafter is NOT slot-locked on this engine (batching and speculation
-#   coexist — DFlash-era slots>1 law does NOT carry over; the card's own
-#   quickstart ships 2 slots WITH the head), so the second slot is
-#   concurrent capacity, not a speculation tax.
-#   Memory ledger against the box's 121.2 GiB unified (the n-gram table is
-#   mmap'd page cache beside the checkpoint, not a pinned charge — the boot
-#   ledger agrees: "mapped 51.2 GB, NOT pinned (Pin::None)"):
-#     weights 62.9 GiB locked (0.8 of them the vision tower) + KV 11.0 +
-#     working 5.7 (HALOGEN_MAX_TOK=8192) = 79.6 GiB engine-side (the boot
-#     ledger of the 401408-position deploy, 2026-10-18), with the 1.3 GiB
-#     prompt cache (12 entries x 111 MiB at ctx 200704) booked separately by
-#     the ledger; HALOGEN_HOST_RESERVE_GIB=20 keeps the host honest. Against
-#     the 121.2 GiB box that leaves ~20 GiB over the reserve (MemAvailable
-#     read 22 GiB right after the raise) for baseline co-tenancy
-#     (frank-zealot, ComfyUI, Caddy/observ ~10-15). The boot ledger is the
-#     referee — read it after deploy and re-size if the engine prints a
-#     clamp.
-#   HALOGEN_SPEC_ADAPT=0 is MANDATORY (card): the default adaptive policy
-#   turns the MTP head off for whole requests after low acceptance and
-#   HALVES decode as a result. Frank accepted the head always-on trade
-#   (temp 1.0 traffic costs acceptance; the card still ships the recipe).
-#
-# CLIENT CONTRACT deltas vs the 27B shape (harness side lives in
-# ~/.omp/agent/models.yml):
-#   * SERVER-SIDE SAMPLER EXISTS NOW: HALOGEN_TEMPERATURE/TOP_P/TOP_K set
-#     the card's thinking recipe (1.0 / 0.95 / 20) as engine defaults, so
-#     an absent temperature is no longer greedy. models.yml extraBody
-#     carries the same values — consistent, not load-bearing.
-#   * HALOGEN_MAX_TOK=8192 is only the DEFAULT for requests that omit
-#     max_tokens; the harness sends at most its maxTokens 45056. Requests
-#     over HALOGEN_MAX_TOKENS_CAP (65536) are a 400 naming the cap. Cap and
-#     queue timeout are COUPLED (FLAGS.md table): 65536 at ~10 t/s serial
-#     worst case needs > 6550 s; 7200 honors it. Do not move one alone.
-#   * max_tokens INCLUDES thinking; a reasoning reply over a small budget
-#     returns empty content with finish_reason "length".
-#   * Prompt cache is ON by default (PROMPT_CACHE=2): a follow-up turn at
-#     100k context re-prefills ~2 s instead of ~88 s cold. No pin needed —
-#     unlike the 27B engine, auto-sizing fits this box's ledger.
-#   * Thinking lands in reasoning_content (same as the 27B engine); the
-#     v2 chat template rejects reasoning_effort outside xhigh|medium|low,
-#     so the models.yml fold (high->xhigh) still applies.
-#   * Vision: OpenAI image_url content parts only (the fleet's standard
-#     contract). >=256 px probe images or the model answers "video".
-#   * Concurrency at depth is tight now: the KV pool is ONE 200,704-position
-#     budget behind 2 slots — exactly one maximum-length request fills it. Two
-#     conversations coexist only while they stay under ~100k each. The
-#     PER-REQUEST ceiling moved WITH the pool (262144 -> 200704), so the harness
-#     contract moves too: contextWindow 155648 = ceiling − 45k, never higher.
-#   * response_format json_object/json_schema unsupported in this line
-#     (no constrained decoding) — unchanged from the 27B engine.
-#   * The engine's token protocol has no auth and binds loopback INSIDE
-#     the container; only the frontend port (8731) is published. Same as
-#     before. Kernel needs CONFIG_HSA_AMD_SVM (CachyOS ships it) and
-#     amdgpu.noretry=0 (check at deploy).
-
-# Image: digest-pinned (fetch law: tag pulls silently re-point; the digest
-# is what was qualified). 0.16.2 = the version the model card names as its
-# quickstart pin; RepoDigest captured at pull on 2026-10-06.
-FRANK_DRAGOON_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a
-# Container identity: kept from the 27B era (unit ExecStop names it).
+# The .hgn format is the lock: it loads ONLY in the flash-server line, so the
+# engine moves with the payload, never ahead of it.
+# Image digest-pinned (a tag pull silently re-points; the digest is what was
+# qualified). 0.18.20 since 2026-10-30 (was 0.16.2, the model card's quickstart
+# pin) — the vendor has moved two minor versions and the payload lock is what
+# the deploy's health gate plus the boot ledger decide; rollback is the
+# previous digest, 0c61bf84…, on this same line.
+FRANK_DRAGOON_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:e35133b206da3094cc4d70c78800b26235bd9124b9aeb0e029d4a7c153b9cff0
+# Container identity kept from the 27B era: the unit's ExecStop names it.
 FRANK_DRAGOON_CONTAINER=frank-dragoon-halogen
-# Old quadlet-era container identity; the serve script clears stale names.
+# Predecessor identity the serve script clears if it is still holding the GPUs.
 FRANK_DRAGOON_LEGACY_CONTAINERS=frank-dragoon
-# host:container — the frontend listens on 8731 inside (HALOGEN_API_PORT
-# default); the engine port (8730) is never published.
+# host:container — the frontend listens on 8731 inside; the engine's own port
+# (8730) has no auth and is NEVER published.
 FRANK_DRAGOON_PUBLISH=7770:8731
 
 FRANK_DRAGOON_MODEL_DIR=/home/dank/models/halogen-v2-abliterated
-# The checkpoint: Quat3rnion/halogen-qwen3.8-flash-next-v2-abliterated
-# qwen38-flash-next-v2-abliterated.hgn — the NATIVE 2026-10-11 v2 conversion
-# of orcarouter's pinned BF16 Qwen3.8-Flash-Next, abliterated. NOT the writer
-# transplant this filename used to carry: 66,687,678,432 B where the swift15
-# r2 measured 66,687,678,464 B — same filename family, different artifact.
-# Hub LFS sha256 verified at fetch by scripts/download-frank-dragoon-v2abl.sh.
+# Same filename family, DIFFERENT artifact: this is the native v2 conversion,
+# 32 bytes smaller than the writer transplant that used to carry the name.
+# Never infer the payload from the filename — this size and sha256 are the gate.
 FRANK_DRAGOON_CHECKPOINT=qwen38-flash-next-v2-abliterated.hgn
 FRANK_DRAGOON_CHECKPOINT_BYTES=66687678432
 FRANK_DRAGOON_CHECKPOINT_SHA256=b5bd925fa2b44343be4a2cd328cae6b57ce7ffa2fe2cc56281717377e5a1ce8a
-# The n-gram lookup table: REQUIRED (the engine reads it beside the
-# checkpoint; the card's quickstart names it explicitly). From
-# peonist-ai/halogen-qwen3.8-flash-next @ 5cc17cea.
-# The card does NOT establish this stock table as matched to the NATIVE
-# conversion and no source-matched table is published, so the engine's own
-# tools are the referee: NIAH 18/18 and PPL 3.968 on it (2026-10-17). It is
-# HARDLINKED from the swift15 payload dir (a hardlink is a regular file; the
-# serve gate refuses symlinks), i.e. byte-identical to the file that era
-# served.
+# The n-gram table is REQUIRED (the engine reads it beside the checkpoint) and
+# disk-backed: mmap'd page cache, never pinned. It is a stock peonist companion
+# hardlinked from the previous payload dir — a hardlink is a regular file, and
+# the serve gate refuses symlinks — with the engine's own NIAH/PPL verdicts on
+# the page as the referee.
 FRANK_DRAGOON_NGRAM_TABLE=qwen38-flash-next-ngram.hgn
 FRANK_DRAGOON_NGRAM_BYTES=51200246144
 FRANK_DRAGOON_NGRAM_SHA256=3450acada94e19aabad88bd49b45eb70304820949c2240fc178067b20ae5dffe
-# The vision tower: OPTIONAL on the engine, ENABLED here (image input).
-# From peonist-ai/halogen-qwen3.8-flash-next @ e053f488.
+# Vision tower: optional on the engine, ENABLED here (image input).
 FRANK_DRAGOON_VISION_TOWER=qwen38-flash-next-vision.hgn
 FRANK_DRAGOON_VISION_BYTES=897916416
 FRANK_DRAGOON_VISION_SHA256=d62e0ae553fe88afd3833733d4a4c669f34d20fd8dfce4b9610525bed2134b10
-# The tokenizer is a FLAT DIRECTORY, $FRANK_DRAGOON_MODEL_DIR/tokenizer,
-# mounted whole at /tokenizer. The two repos ship the same one; fetched
-# fresh from peonist @ e053f488 (tokenizer.json is LFS, byte-identical to
-# the 27B-era copy; the rest are plain git blobs — size-checked only). The
-# serve gate and the deployer refuse a missing file or ANY symlink: an HF
-# cache snapshot symlinks each file into blobs/, and those links dangle
+# The tokenizer is a FLAT DIRECTORY ($FRANK_DRAGOON_MODEL_DIR/tokenizer) mounted
+# whole. The serve gate and the deployer refuse a missing file or ANY symlink:
+# an HF cache snapshot symlinks each file into blobs/ and those links dangle
 # inside the container mount.
 FRANK_DRAGOON_TOKENIZER_FILES="tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json"
 
 # --- engine flags (docs/FLAGS.md of the flash-server repo; anything unset
 # stays at the image default, which is what was qualified) ------------------
-# Served model id — Protoss name (wire rename 2026-10-09); the service, unit,
-# scripts and this file keep the frank-dragoon name. OpenWebUI api_configs
-# model_ids for the :7770 backend MUST name this value.
+# Served id. OpenWebUI api_configs model_ids for the :7770 backend MUST name it.
 HALOGEN_MODEL_ID=frank-dragoon
-HALOGEN_CTX=200704
+# PER-REQUEST ceiling, set to the checkpoint's NATIVE window (262,144 — a
+# longer prompt is a 400 naming the limit, never a truncation; stretching
+# past native would need HALOGEN_ROPE_YARN and is not on plan). The harness
+# `contextWindow` must sit ~45k under THIS number (the 400-zone law on the
+# page): with the window at 200,000 the gap is 62,144, which absorbs the
+# chars/4 under-count. A window at or near the ceiling makes every long
+# request a guaranteed 400.
+HALOGEN_CTX=262144
+# ONE pool behind both slots. The harness window is 200,000, so 401,408
+# still carries TWO maximum harness-length conversations at once
+# (2 x 200,000 + one 45,056 budget <= pool + cap headroom); two literal
+# ceiling-length requests would queue instead — buying those means raising
+# the pool, and RAM the pool takes is RAM the n-gram table's page cache
+# loses. Pool positions buy concurrency; the per-request window never moves
+# with them. Legal band: ctx <= pool <= slots * ctx (the seam enforces it).
 HALOGEN_KV_POOL_POSITIONS=401408
+# Two concurrent conversations: the MTP drafter is not slot-locked on this
+# engine, so the second slot is capacity, not a speculation tax.
 HALOGEN_KV_SLOTS=2
 HALOGEN_HOST_RESERVE_GIB=20
-# Spec decode: MTP head is INSIDE the checkpoint (do NOT set HALOGEN_MTP_HEAD);
-# depth 2 is the documented "best all-round" default. HALOGEN_DRAFTER /
-# HALOGEN_DFLASH_* from the 27B engine DO NOT EXIST here.
+# The MTP head is INSIDE the checkpoint — do NOT set HALOGEN_MTP_HEAD.
+# 0 is mandatory: the adaptive default disarms the head for whole requests after
+# low acceptance and halves decode. Depth 2 is the card's best-all-round default.
 HALOGEN_SPEC_ADAPT=0
-# Coupled pair (FLAGS.md table): the cap bounds how long ONE request holds
-# the GPU, the timeout bounds how long the NEXT client waits. Do not move
-# one alone (see the client-contract note above).
+# Coupled trio: MAX_TOK is only the default for requests that omit max_tokens;
+# the cap bounds how long ONE request holds the GPU and the queue timeout bounds
+# how long the NEXT client waits. Move all three together or none.
 HALOGEN_MAX_TOK=8192
 HALOGEN_MAX_TOKENS_CAP=65536
 HALOGEN_QUEUE_TIMEOUT=7200
-# Server-side sampler = the card's thinking recipe (the engine now HAS one;
-# requests may still override per-call). Matches models.yml extraBody.
+# The engine has a server-side sampler (an absent temperature is NOT greedy
+# here); models.yml extraBody carries the same recipe.
 HALOGEN_TEMPERATURE=1.0
 HALOGEN_TOP_P=0.95
 HALOGEN_TOP_K=20
-# Card's belt-and-braces: the engine pins weights by default
-# (HALOGEN_FLASH_PIN_TRUNK=1; only the retired w4b build honored 0).
+# Weights stay pinned: the engine pins by default and only the retired w4b build
+# honored HALOGEN_FLASH_PIN_TRUNK=0, which this payload rejects.
 HALOGEN_WEIGHTS_LOCK=1
+
+# --- NPU lane (XDNA2) -------------------------------------------------------
+# HALOGEN_NPU_MODELS is the whole switch: EMPTY or unset = the endpoint exactly
+# as it ran before this lane existed (no accel device, no XRT mount, no
+# fabric-clock dependency, no /v1/embeddings). Rolling the lane back is blanking
+# this one line and re-deploying — nothing else moves.
+# Names are the vendor catalog's folder names (`/opt/halogen/npu/models.txt`
+# inside the image) and the engine reads a model ONLY at /models/npu/<name>/ —
+# HALOGEN_NPU_DIR is gone from the image and setting it is a boot refusal.
+# The engine cannot fetch anything here: /models is read-only and
+# HALOGEN_DOWNLOAD stays unset so the container opens no outbound connection at
+# boot, so scripts/download-frank-dragoon-npu.sh must stage the files first
+# (the serve gate and the deployer both refuse by name when a named model is
+# missing). FRANK_DRAGOON_NPU_DIR therefore lives INSIDE the payload dir: the
+# payload mount is the container's only handle on /models, and runc cannot
+# mkdir a nested /models/npu mountpoint through a read-only bind — the lane's
+# first boot died on precisely that (2026-10-19). Swap the payload dir and this
+# path moves with it.
+# Cost: ~1.5 GiB extra pinned host RAM per named model, and a hard dependency on
+# the host holding the GPU fabric clock — on Strix Halo the iGPU and the NPU
+# together hang the machine (and corrupt NPU results) while FCLK changes speed,
+# so halogen refuses to start with this variable set unless the clock is held
+# (dankclaw/host/halogen-fabric-clock; the deployer installs and gates it).
+HALOGEN_NPU_MODELS=qwen3-embedding-0.6b
+FRANK_DRAGOON_NPU_DIR=/home/dank/models/halogen-v2-abliterated/npu
+# Node paths and the XRT directory are variables so the bats seam can drive
+# these gates on a box with no NPU. Ubuntu's distro XRT (libxrt2 + libxrt-npu2)
+# is mounted FILE BY FILE and THREE TIMES EACH — where the image looks for XRT
+# (/opt/xilinx/xrt/lib), at its own path (those libraries find each other at
+# their install path), and at /opt/xilinx/lib/<multiarch>/ (the NPU child
+# searches that third prefix; without it the lane dies naming it and the
+# supervisor takes the container down — 2026-10-21 boot loop). The basename of
+# this directory IS the multiarch name. The vendor forbids mounting an
+# /opt/xilinx/xrt of symlinks into the system dir, which dangle in the container.
+FRANK_DRAGOON_DEV_ACCEL=/dev/accel/accel0
+FRANK_DRAGOON_XRT_DIR=/usr/lib/x86_64-linux-gnu
+# sysfs root the fabric-clock preflight reads (same knob the vendor's
+# halogen-fabric-clock status reads; the seam points it at a fixture).
+FRANK_DRAGOON_SYSFS=/sys
+# Container network. Empty = podman's rootless default (`pasta`): a private
+# netns that nothing else can join — podman refuses a second network while the
+# primary is pasta, the published port is refused from dankclaw_default
+# containers, and host.containers.internal resolves link-local, which
+# open-notebook's URL guard refuses to store. Named = the app bridge, so
+# open-notebook can dial http://frank-dragoon-halogen:8731 for the NPU
+# endpoints. The engine's unauthenticated 8730 stays private either way (it
+# binds the container's own loopback); only the API port becomes name-resolvable.
+FRANK_DRAGOON_NETWORK=dankclaw_default

scripts/frank-dragoon.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=halogen-flash-server container (Quat3rnion NATIVE v2 abliterated Qwen3.8-Flash-Next, MTP spec decode, 2 slots over ONE 401408-position pool at ctx 200704, vision ON), root systemd unit (dankclaw, frank-dragoon, Strix Halo iGPU)
+Description=halogen-flash-server container (Quat3rnion NATIVE v2 abliterated Qwen3.8-Flash-Next, MTP spec decode, 2 slots over ONE 401408-position pool at ctx 262144, vision ON), root systemd unit (dankclaw, frank-dragoon, Strix Halo iGPU)
 After=network-online.target
 Wants=network-online.target
 

scripts/frank-tts-engine.env

@@ -27,6 +27,16 @@
 # below stop describing the process you are running. Nothing in this fleet
 # pins this container; that is a decision, not an oversight.
 #
+# THE DEVICE LAW (since the 2026-10-19 Vulkan cutover). Synthesis computes on
+# the gfx1151 iGPU — the SAME card frank-dragoon serves. It is a shared device,
+# never a dedicated one, so two numbers matter and neither is what `rocm-smi`
+# prints first: the memory the container may use is the RADV heap inside the
+# GTT-backed aperture (the ~2 GiB DRM carve-out is not the working set), and the
+# timing that matters is dragoon's, measured WITH a synthesis in flight. The
+# rollback is one line: set QWENTTS_DEVICE empty, and the entrypoint passes no
+# --device at all — the pre-iGPU CPU shape, no rebuild, no restart of anything
+# else on the box.
+#
 # THE REGISTRY LAW (why the unit has an ExecStartPost at all). Registrations live
 # in memory only, so a restart loses them, and re-registration alone does NOT warm
 # a voice — the warm pass has to synthesize something. Without it the façade
@@ -103,6 +113,17 @@ QWENTTS_LANG=auto
 # round-trip outside it) is what absorbs the queue.
 QWENTTS_MAX_BATCH=1
 
+# The compute device, by ggml device NAME. `Vulkan0` is the gfx1151 iGPU: the
+# image carries the Mesa (RADV) ICD and no NVIDIA ICD, so this container's
+# Vulkan enumeration has exactly one entry and it is the iGPU. The unit hands
+# the container /dev/dri and keeps the render group; on this hybrid host the
+# iGPU's render node is renderD129 and renderD128 is the NVIDIA card, which is
+# why the unit passes the directory and lets the driver resolve the node.
+#
+# ROLLBACK: an empty value means no --device flag at all — the CPU shape this
+# endpoint shipped with until 2026-10-19. One line, no rebuild.
+QWENTTS_DEVICE=Vulkan0
+
 # --- the boot warmer (dankclaw/voiceservice/tts-engine-warm.sh) --------------
 # These three are read by the warm hook, not by tts-server. They live here
 # because the hook is exec-ed into this container and therefore inherits this

scripts/frank-tts.env

@@ -7,7 +7,7 @@
 # whichever TTS engine wins the engine decision (dankclaw/voiceservice/app.py
 # is a single stdlib-only file in a locally built image). Consumers — omp's
 # TTS device, OpenWebUI, open-notebook — address a VOICE BY NAME
-# (voice=narrator1, or the harness' voice_id=narrator1) and never see a path:
+# (voice=wilwheaton, or the harness' voice_id=wilwheaton) and never see a path:
 # a name is real exactly when /voices/<name>/reference.wav exists, and
 # anything else is a 400 that lists the names that are.
 #
@@ -65,7 +65,7 @@ TTS_UPSTREAM_MODEL=voxcpm2
 # name still 400s with the list that exists — this covers the absent case only —
 # and the journal marks such a line `(default)` so an unrequested voice never
 # reads as a requested one. Any pack directory in /home/dank/voices is legal.
-DEFAULT_VOICE=narrator1
+DEFAULT_VOICE=wilwheaton
 
 # CLI-backend knobs, inert while TTS_BACKEND=omni-http. Empty means "not
 # configured": the endpoint answers a 500 that says so instead of running

scripts/frank-zealot.env

@@ -1,141 +1,85 @@
-# frank-zealot — llama.cpp (PrismML fork build 10777, CUDA) serving
-# prism-ml/Ternary-Bonsai-2-9B-gguf at PQ2_0 **with the MTP head in the file**
-# on the RTX 3090, :7771.
+# frank-zealot — llama.cpp (PrismML fork build 12118 = 0.7.0-dev, commit
+# 024474448, CUDA) on the RTX 3090, :7771. Payload: Ternary-Bonsai-2-9B PQ2_0 with the MTP head in the file, plus
+# the refusal-direction adapter applied at load.
 #
-# 2026-10-18 (Frank): swap from the Bonsai-2-**27B** PQ2_0 to the 9B MTP export
-# at THREE 262,144-token conversations (Frank: "if you can possibly fit a third
-# context, do it"). The 9B bills HALF the KV of the 27B per token, which is what
-# funds 3 x the native window on one 24,123 MiB card. Speculative decoding ships
-# OFF — measured, not assumed; see the MTP law below, which is the whole story
-# and the reason the publisher's "depth 2 works best" line does not transfer.
-#
-# THE RUNTIME IS A LOCK, NOT A PREFERENCE. The file is general.file_type 141
-# carrying custom GGML type 142 tensors plus prism.hadamard.* metadata
-# (normalized-Sylvester-Walsh transform); stock llama.cpp refuses type 142 at
-# load and a loader that ignored the Hadamard metadata would dequant WRONG. On
-# top of that lock, MTP beside a quantized V cache needs fork build >= b10776,
-# so the vendor prebuilt moved from 10754 (commit 2459f68b5, the 27B era) to
-# 10777 (commit e3de71522). The 10754 tree stays on disk as the rollback
-# runtime for the 27B payload.
-LLAMA_BIN=/home/dank/GitPackages/llama-prism-b10777/llama-server
-# 2,675,410,720 B, Hub LFS sha256 c79fcdfaf6b7… (verified at fetch by
-# scripts/download-frank-zealot-bonsai9.sh). This IS the MTP build: the draft
-# head ships INSIDE the checkpoint, so MTP availability is a property of the
-# FILE, never of a flag. The plain PQ2_0 sibling could not run it at any flag
-# value; loading this file instead of that one is the only change that matters.
+# ONE OWNER: shape, payload identity, eras, laws and every measured number
+# live in serverAppConfigs-docs/LLM/frank-zealot.md. This file carries the
+# knobs and only the reason a reader editing THIS line cannot recover
+# elsewhere. Where the two disagree, this file wins on values.
+LLAMA_BIN=/home/dank/GitPackages/llama-prism-0244744/bin/llama-server
+# The fork is a LOCK, not a preference: custom tensor type 142 + prism.hadamard.*
+# metadata. Stock llama.cpp refuses the file; a loader ignoring the Hadamard
+# metadata dequantizes wrong, silently.
 MODEL_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.gguf
-# 624,229,664 B, Hub LFS sha256 e0cb6f69… — Q8_0 IS MANDATORY, not a preference:
-# the publisher's card says the BF16 mmproj "can crash llama-server when several
-# image requests run at once; Q8_0 is not affected", and this endpoint runs
-# --parallel 3, i.e. exactly that shape. The projector's worst case lands AFTER
-# the engine's own fit check, so it is never inside a projection number.
+# Q8_0 IS MANDATORY: the publisher states the BF16 tower can crash llama-server
+# under concurrent image requests, which is exactly --parallel 3.
 MMPROJ_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf
-# Base alignment: Bonsai-2 is the censored BASE lineage. The 27B CRACK re-export
-# (/home/dank/models/Bonsai2-CRACK/) is byte-identical geometry to the 27B it
-# matched, NOT to this 9B file — if a refusal ever surfaces there is no
-# one-line swap here; the 9B dealigned export would have to be fetched first.
-#
-# Served model id — the harness/family name. The SERVICE, unit, scripts and
-# this env file keep the frank-zealot name; only the alias on the wire is the
-# model id. Consumers that key on the served id: OpenWebUI (model row,
-# access_grant, api_configs model_ids) and the harness models.yml.
+# Refusal-direction adapter (a rank-1 LoRA, 3.2 MB) applied at load; it edits
+# the compute graph, it is not a second model. Scale 1 is the card's "held-out
+# refusals 0 %" point. LORA_FILE="" is the documented opt-out — the plain pack
+# again, no rebuild — and LORA_SCALE=0 is the same result through the flag.
+# PROVENANCE ASYMMETRY: the adapter's own prism.abliterate.base_gguf names
+# Ternary-Bonsai-2-9B-PTQ1_0.gguf as the pack its direction was fitted against,
+# which is NOT this packing and is not on this host. The tensor names match, so
+# it applies; the strength was calibrated on a sibling quantization, so the
+# effect is proven by the endpoint's behaviour, never by the metadata.
+LORA_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf
+LORA_SCALE=1
+# Served id — the harness/family name, not a fact about the payload.
+# Id-keyed consumers: OpenWebUI (model row, access_grant, api_configs), models.yml.
 ALIAS=frank-zealot
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7771
-# CTX_SIZE is the POOL llama.cpp divides across slots: 786,432 / 3 = 262,144 per
-# conversation — each slot at the model's TRAINED window, which is the point of
-# the swap (the 27B era had to cut each conversation to 200,704 to fit two).
-# MUST stay divisible by PARALLEL (the seam enforces it) and the harness
-# contextWindow quotes the PER-SLOT number — a longer prompt is answered with a
-# 400, never truncated. The ceiling cannot go past 3 x 262,144: that is the
-# native wall (no YaRN in this path) and the arithmetic agrees — a 4th slot
-# would push each below the compaction contract.
-CTX_SIZE=786432
+# CTX_SIZE is the POOL llama.cpp divides across slots: 602,112 / 3 = 200,704 a
+# conversation. MUST stay divisible by PARALLEL (the seam enforces it) and
+# models.yml contextWindow quotes the PER-SLOT number — a longer prompt is a
+# 400, never a truncation. The pool is bought, not granted: K+V at 25,088
+# B/token is 14,406.00 MiB here and the MTP draft context bills its own f16 KV
+# over the WHOLE pool (4,096 B/position ⇒ 2,352.00 MiB), so the old 786,432
+# pool cannot carry a drafter at any KV dtype — 18,816 + 3,072 of cache alone
+# against the 24,123 MiB the driver leaves addressable.
+CTX_SIZE=602112
 PARALLEL=3
-BATCH_SIZE=4096
-# Ubatch is a prefill knob (decode never moves) with NO interior optimum on this
-# CUDA card up to 2048 — 2048 is the measured optimum (+4.6 % prefill). It has a
-# second job here: it sets the compute-buffer size, and the compute buffers are
-# what decide whether MTP fits (see the MTP law). 1024 is the documented MTP arm.
-UBATCH_SIZE=2048
-# KV law (header + boot ledger, both confirmed): qwen35 hybrid, 32 layers with
-# qwen35.full_attention_interval = 4 ⇒ 8 full-attention + 24 Gated DeltaNet;
-# 4 KV heads × 256 dim = 8,192 elements per token per cache. q8_0 bills 8,704 B
-# per cache ⇒ 17,408 B/token = 17 KiB, so the 786,432 pool is 13,056.00 MiB —
-# the ledger said exactly that. HALF the 27B's 34,816 B/token, which is the
-# whole reason three native windows fit. f16 would bill 26,112 MiB at this pool
-# (against a 24,123 MiB card) and dies at cudaMalloc; the 24 DeltaNet layers
-# bill a fixed recurrent state per SLOT instead of per token — the PRODUCTION
-# ledger prints "RS buffer size = 150.75 MiB (3 cells, 32 layers, 3 seqs)"
-# (~50 MiB a cell), while the SPEC-ARMED arms print 452.25 MiB, because MTP
-# depth 2 bills three seqs per slot (9 cells) — which is why pool arithmetic
-# alone under-prices the shape.
-CACHE_TYPE_K=q8_0
+# The logical batch is VRAM- and throughput-neutral at a fixed ubatch (the law
+# on [[frank-zealot]]); 1024×6 keeps it a clean multiple of the ubatch.
+BATCH_SIZE=6144
+# Ubatch is a prefill knob (decode never moves with it) and it sets the
+# compute-buffer size, which decides whether a drafter fits. 1024 is the rung
+# every measured MTP arm here ran on (213.5 t/s against 109.2 at 2048). Its
+# VRAM credit is SHAPE-specific: the 786,432-pool arm census showed ~1,306 MiB
+# back, this boot measured 2,089.88 MiB of target compute against 2,496.13 at
+# ubatch 2048 — 406 MiB back, with the split moving (CUDA0 +257, CUDA_Host
+# −663). The drafter is funded by CACHE_TYPE_V below, not by this knob.
+UBATCH_SIZE=1024
+# K exact, V in the 8-bit class: 16,384 + 8,704 = 25,088 B/token (8,192
+# elements per token per cache × 2 B for f16, × 1.0625 for q8_0 — there is no
+# true fp8 KV dtype in this build). That 7,680 B/token saved against exact V is
+# what funds BOTH the third conversation and the drafter. Quantized V is
+# illegal without --flash-attn, which the serve script pins. Rate law and the
+# arithmetic: [[frank-zealot]] → Laws.
+CACHE_TYPE_K=f16
 CACHE_TYPE_V=q8_0
-# SPECULATIVE DECODING: OFF, MEASURED. The head is in the file and the drafter
-# is GOOD (acceptance 0.798/0.617 per position, mean accepted length 2.41) — it
-# still loses, because the draft context bills its OWN f16 KV over the WHOLE
-# pool: 4,096 B/position × 786,432 = 3,072.00 MiB, plus its compute buffers,
-# 4,830 MiB all told. Booted arms, same card, same traffic (2026-10-18):
-#   3 × 262,144  spec OFF  ub 2048 → 17,876 used / 6,249 free, decode 167.2 t/s
-#   3 × 262,144  spec ON   ub 2048 → 22,706 used / 1,419 free, decode 109.2 t/s
-#   3 × 262,144  spec ON   ub 1024 → 21,400 used / 2,725 free, decode 213.5 t/s
-#   2 × 262,144  spec ON   ub 2048 → 17,178 used / 6,947 free, decode 213.2 t/s
-#   1 × 262,144  spec ON   ub 2048 → 11,652 used / 12,473 free, decode 214.2 t/s
-# i.e. MTP is worth +28 % decode whenever the card has room, and at the 3-slot
-# pool it costs 1.3 GiB of compute buffers that ubatch 2048 refuses to give up —
-# the squeeze, not the drafter, is what halves decode. The arm that WOULD run
-# MTP at three windows is SPEC_TYPE=draft-mtp + UBATCH_SIZE=1024 (decode 213.5,
-# 3-stream 250.8 aggregate) and pays prefill −17 % and 2.7 GiB of headroom on a
-# card whose projector worst-case lands after the fit check. This endpoint's
-# load is the fleet's COMPACTION HUB (prefill-bound, ~170k in) plus 2-3
-# concurrent subagents, so the deployed arm takes the +17 % prefill
-# (2,877 vs 2,395 t/s at 106,418 tokens) and the +12 % three-stream aggregate
-# (280.2 vs 250.8) over single-stream speed, and keeps 6.2 GiB free.
-# SPEC_TYPE=none strips the WHOLE spec set (--spec-type, every --spec-draft-*):
-# flags move as a set or the arm measures a crash instead of an absence.
-SPEC_TYPE=none
+# Speculation ON (Frank, 2026-10-19): the head is in the file
+# (qwen35.nextn_predict_layers = 1, blk.32.nextn.*) and rides the MAIN model's
+# weights, so there is no --spec-draft-model. Its bill is pool × 4,096 B of
+# draft KV plus ~0.45 GiB of buffers and a tripled recurrent state — paid for
+# by CACHE_TYPE_V=q8_0 and UBATCH_SIZE=1024 above. Build gate: quantized V
+# beside MTP needs fork build >= b10776; LLAMA_BIN is 12118 (b10777 is kept on
+# disk as the rollback path).
+# SPEC_TYPE=none is the rollback and strips the WHOLE set (--spec-type and
+# every --spec-draft-*); half a set has segfaulted at load.
+SPEC_TYPE=draft-mtp
 SPEC_DRAFT_N_MAX=2
 SPEC_DRAFT_N_MIN=0
 SPEC_DRAFT_P_MIN=0
-# The card's sampler recipe lives in the checkpoint (general.sampling.* = temp
-# 1.0 / top_p 0.95 / top_k 20 / min_p 0 / repeat_penalty 1.0 — confirmed in the
-# engine's own /props defaults) and llama.cpp reads it itself. Presence penalty
-# is the ONE recipe value with no GGUF metadata key, so it must reach the argv.
-# PRESENCE_PENALTY="" is the documented opt-out.
+# The sampler recipe lives in the checkpoint (general.sampling.*) and
+# llama.cpp reads it itself. Presence penalty is the one recipe value with no
+# GGUF metadata key, so it must reach argv. "" is the documented opt-out.
 PRESENCE_PENALTY=1.0
 NGL=99
-# Boot ledger (2026-10-18 PRODUCTION boot under the unit, verbosity 4):
-#   17,922 MiB used / 6,203 MiB free of 24,123 (nvidia-smi settled)
-#   CUDA0 model 2,036.71 MiB + CPU_Mapped 257.66 MiB
-#   CUDA0 KV 13,056.00 MiB (q8_0 K + q8_0 V, 786,432 positions)
-#   CUDA0 RS 150.75 MiB (3 cells, 32 layers, 3 seqs)
-#   compute 1,409.02 MiB CUDA + 1,088.87 MiB CUDA_Host (+2.84 MiB output buf)
-#   mmproj worst-case estimate 868.32 MiB (248.10 CUDA + 24.93 CPU compute
-#   meta) — charged AFTER the engine's own fit check, as always
-#   no clamp line, no cudaMalloc error
-# Measured on this boot (/completion timings, greedy, ignore_eos, 512 tokens):
-# 168.7 t/s decode (three runs 168.4-168.7) and ~300 t/s aggregate across
-# THREE concurrent streams (102.8 each). The trial arm's record (167.2 decode,
-# 280.2 aggregate, 2,877 t/s prefill at 106,418) stands; the 27B it replaces
-# gave 76.6 / 109.4 / 691.7 at 166k. The FIRST request after a boot is a
-# warmup artifact (37.7 t/s class) and a natural generation that stops early
-# makes a wall-clock aggregate read 4x low - measure with ignore_eos and read
-# the engine's own timings.
-# Fallback rung if the margin is ever eaten (desktop, co-tenant): q4_0 KV on the
-# 27B era measured 16,392 used / 7,733 free at 2 × 200,704; the same drop here
-# frees ~6.5 GiB at three windows. UBATCH_SIZE=1024 is the cheaper lever
-# (−1,306 MiB measured on this shape).
-# Log verbosity threshold (llama.cpp -lv). 4 = debug: KV layout, buffer sizes
-# and graph capture land in the journal, which is what the VRAM-fit forensics
-# read. Levels above 4 flood the journal.
+# 4 = debug: KV layout, buffer sizes and graph capture land in the journal, and
+# that is what VRAM forensics read. Above 4 floods the journal.
 VERBOSITY=4
-
-# Serve /metrics. llama.cpp answers 501 without --metrics, and Dankclaw's
-# Prometheus scrapes /metrics on every fleet endpoint — a 501 reads as DOWN, so
-# this service has held the critical LlmEndpointDown alert for as long as the
-# metric has history (verified 2026-10-18: the oldest retained sample of
-# up{job="frank-zealot"}, 2026-10-11, is already 0; scrapeUrl
-# http://host.containers.internal:7771/metrics, lastError "server returned HTTP
-# status 501 Not Implemented"). 0 opts out and re-breaks the scrape.
+# Without --metrics llama.cpp answers 501 on /metrics, which Prometheus reads
+# as DOWN while /health stays 200. 0 opts out and re-breaks the scrape.
 METRICS=1

scripts/frank-zealot.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=llama.cpp (PrismML build 10777) Ternary-Bonsai-2-9B PQ2_0-MTP GGUF (qwen35 hybrid, 3x262144 slots over one 786432 pool, q8_0 K+V, MTP off, mmproj Q8_0 vision) on the RTX 3090 (dankclaw, frank-zealot)
+Description=llama.cpp (PrismML build 10777) Ternary-Bonsai-2-9B PQ2_0-MTP GGUF + refusal adapter (qwen35 hybrid, 3x200704 slots over one 602112 pool, f16 K + q8_0 V, MTP armed, mmproj Q8_0 vision) on the RTX 3090 (dankclaw, frank-zealot)
 After=network-online.target
 Wants=network-online.target
 

scripts/gen-vault-index.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+"""Regenerate serverAppConfigs-docs/index.md from page frontmatter.
+
+The root index is a locator (AGENTS.md -> *Documentation Policy*): one row per
+page, one clause, no numbers. Hand-maintaining it is how rows go missing, and
+the docs seam fails when a page is not linked from it, so it is generated:
+
+  * preamble     everything above the first `## ` in the existing file is kept
+                 verbatim (that is the curated statement of the vault's law)
+  * section      from the page's directory, overridden by its frontmatter tags
+                 (`endpoint` / `retired-endpoint` split the LLM pages)
+  * row text     the page's frontmatter `description`, verbatim
+
+A page with no description is a hard error rather than an empty cell, so the
+generator cannot silently produce a silent index. Sections with no pages are
+skipped. Deterministic: the same vault always yields the same bytes.
+
+Sub-directory indices are hand-grouped (they carry curated prose), so they are
+synced rather than regenerated: dead rows dropped, row text refreshed from the
+page's description, uncovered siblings appended.
+"""
+import pathlib
+import re
+import sys
+
+RESERVED = {"index.md", "log.md"}
+SECTIONS = [
+    ("Inference endpoints", "endpoint"),
+    ("Retired endpoints", "retired-endpoint"),
+]
+DIRECTORY_SECTIONS = [
+    ("Dankclaw", "Dankclaw services"),
+    ("Dankserver", "Dankserver services"),
+    ("FrankBook-Pro", "FrankBook-Pro"),
+    ("Desktop", "Workstations"),
+    ("Infrastructure", "Cross-host infrastructure"),
+    ("Architecture", "Repo architecture and policy"),
+    ("Operations", "Operations and incidents"),
+    ("Agent", "Agent / harness"),
+    ("Research", "Research and methodology"),
+]
+PIPELINE_SECTION = "Pipeline Parallelism (series)"
+LLM_FALLBACK = "LLM technique and hardware references"
+CHANGE_SECTION = "Change history"
+ORDER = [t for t, _ in SECTIONS] + [t for _, t in DIRECTORY_SECTIONS] + \
+        [PIPELINE_SECTION, LLM_FALLBACK, CHANGE_SECTION]
+
+
+def frontmatter_fields(text):
+    if not text.startswith("---"):
+        return {}
+    end = text.find("\n---", 3)
+    if end == -1:
+        return {}
+    raw = text[3:end]
+    fields = {}
+    for key in ("description", "type"):
+        m = re.search(rf"^{key}:\s*(.*)$", raw, re.M)
+        if m:
+            fields[key] = m.group(1).strip().strip("\"'")
+    tags = []
+    m = re.search(r"^tags:\s*\[(.*?)\]$", raw, re.M)
+    if m:
+        tags = [t.strip().strip("\"'") for t in m.group(1).split(",") if t.strip()]
+    elif re.search(r"^tags:\s*$", raw, re.M):
+        block = re.search(r"^tags:\s*\n((?:\s+-\s+.*\n?)+)", raw, re.M)
+        if block:
+            tags = [l.strip().lstrip("-").strip().strip("\"'")
+                    for l in block.group(1).splitlines() if l.strip()]
+    fields["tags"] = tags
+    return fields
+
+
+def section_for(page, vault, fields):
+    rel = page.relative_to(vault)
+    if len(rel.parts) == 1:
+        return CHANGE_SECTION if page.name == "log.md" else "Uncategorised"
+    top = rel.parts[0]
+    if top == "LLM":
+        if PIPELINE_SECTION_DIR in rel.parts:
+            return PIPELINE_SECTION
+        for title, tag in SECTIONS:
+            if tag in fields.get("tags", []):
+                return title
+        return LLM_FALLBACK
+    for prefix, title in DIRECTORY_SECTIONS:
+        if top == prefix:
+            return title
+    return "Uncategorised"
+
+
+ROW = re.compile(r"^(\| )\[\[([^\]|]+)\]\]( \|)( .*?)( \|$)")
+BULLET = re.compile(r"^(- )\[\[([^\]]+)\]\]( — )(.*?)$")
+
+
+def sync_subindex(index: pathlib.Path, vault: pathlib.Path):
+    """Keep a hand-grouped sub-index honest without erasing its grouping.
+
+    Rows are locators, so their text is the linked page's own description; a
+    row whose page is gone is a dead promise and is dropped; a direct sibling
+    with no row is appended under `## Pages` (that is what the link-integrity
+    seam checks). Curated headings and prose are untouched.
+    """
+    here = index.parent
+    known = {p.stem for p in here.rglob("*.md")
+             if ".AGENTTRASH" not in str(p) and p.name != "index.md"}
+    lines = index.read_text().split("\n")
+    out, dropped, refreshed = [], 0, 0
+    for line in lines:
+        # A row whose first cell is empty is a description with no page: the link
+        # was deleted when the page was destroyed, so the row is a dead promise.
+        if re.match(r"^\|\s*\|", line.strip()):
+            dropped += 1
+            continue
+        for pat, wrap in ((ROW, "| [[{}]] |{} {} |"), (BULLET, "- [[{}]] — {}")):
+            m = pat.match(line)
+            if not m:
+                continue
+            stem = m.group(2).strip()
+            if stem not in known:
+                dropped += 1
+                line = None
+                break
+            cand = next((p for p in here.rglob(f"{stem}.md")
+                         if ".AGENTTRASH" not in str(p)), None)
+            desc = frontmatter_fields(cand.read_text(errors="replace")).get("description") \
+                if cand else None
+            if desc:
+                current = m.group(4).strip()
+                if current != desc:
+                    line = wrap.format(stem, " " if pat is ROW else "", desc) \
+                        if pat is ROW else wrap.format(stem, desc)
+                    refreshed += 1
+            break
+        if line is not None:
+            out.append(line)
+    text = "\n".join(out)
+    # Presence must be the wikilink itself: a bare stem is a substring of other
+    # page names (`frank-medium` lives inside `frank-medium-rocmfpx`), and a
+    # sibling matched that way silently never gets a row.
+    missing = sorted(
+        (p.stem, frontmatter_fields(p.read_text(errors="replace")).get("description", ""))
+        for p in here.glob("*.md")
+        if p.name != "index.md" and f"[[{p.stem}]]" not in text
+    )
+    if missing:
+        block = ["", "## Pages", "", "| Page | Owns |", "|---|---|"]
+        block += [f"| [[{stem}]] | {desc} |" for stem, desc in missing]
+        text = text.rstrip("\n") + "\n" + "\n".join(block) + "\n"
+    if text != index.read_text():
+        index.write_text(text)
+    return dropped, refreshed, len(missing)
+
+
+PIPELINE_SECTION_DIR = "Pipeline Parallelism"
+
+
+def main(argv):
+    vault = pathlib.Path(argv[1] if len(argv) > 1 else "serverAppConfigs-docs")
+    index = vault / "index.md"
+    if not index.is_file():
+        print(f"missing {index}")
+        return 2
+    existing = index.read_text()
+    preamble = existing.split("\n## ", 1)[0].rstrip("\n")
+
+    pages = [p for p in vault.rglob("*.md")
+             if ".AGENTTRASH" not in str(p) and ".obsidian" not in str(p)
+             and p.name != "index.md" and p != index]
+    buckets, errors = {}, []
+    for page in pages:
+        fields = frontmatter_fields(page.read_text(errors="replace"))
+        if page.name != "log.md":
+            desc = fields.get("description")
+            if not desc:
+                errors.append(str(page.relative_to(vault)))
+                continue
+        else:
+            desc = "Chronological incident and law log for the whole bundle."
+        buckets.setdefault(section_for(page, vault, fields), []).append((page.stem, desc))
+    if errors:
+        print("pages with no description (fix the frontmatter, not the index):")
+        for e in sorted(errors):
+            print("  " + e)
+        return 1
+
+    out = [preamble, ""]
+    for title in ORDER + sorted(set(buckets) - set(ORDER)):
+        rows = sorted(buckets.get(title, []), key=lambda r: r[0].lower())
+        if not rows:
+            continue
+        out += [f"## {title}", "", "| Page | Owns |", "|---|---|"]
+        out += [f"| [[{stem}]] | {desc} |" for stem, desc in rows]
+        out.append("")
+    out.append("<!-- generated by scripts/gen-vault-index.py; edit page frontmatter instead -->")
+    index.write_text("\n".join(out) + "\n")
+    touched = 0
+    for sub in sorted(vault.rglob("index.md")):
+        if sub == index or ".AGENTTRASH" in str(sub):
+            continue
+        dropped, refreshed, added = sync_subindex(sub, vault)
+        if dropped or refreshed or added:
+            touched += 1
+            print(f"  {sub}: -{dropped} dead rows, {refreshed} refreshed, +{added} added")
+    print(f"{index}: {sum(len(v) for v in buckets.values())} rows in "
+          f"{len([t for t in ORDER if buckets.get(t)])} sections, {touched} sub-indices synced")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

scripts/gpu-exporter.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""gpu-exporter: unified per-GPU metrics for a mixed AMD/NVIDIA host.
+
+Emits the `gpu_*` Prometheus families the host-system dashboard graphs:
+
+    gpu_utilization_percent  gpu_memory_used_bytes  gpu_memory_total_bytes
+    gpu_temperature_celsius  gpu_power_watts
+
+each labelled {gpu, vendor, model}. The scrape job adds `host`.
+
+Two sources, both host-native — no container, no GPU passthrough:
+
+  * AMD (R9700s, the 8060S iGPU, the Raphael display GPU): the amdgpu sysfs tree.
+    Utilization comes ONLY from gpu_busy_percent — amd-smi reports gfx_busy_inst
+    as N/A on gfx1201 (verified 2026-10-20). VRAM is already bytes, temperature
+    is milli-degrees, power is micro-watts. Crucially power1_average is EMPTY on
+    the R9700s, so those cards legitimately have no gpu_power_watts line; a
+    missing file becomes an ABSENT metric, never a fabricated 0 that a dashboard
+    would plot as a real reading.
+
+  * NVIDIA (the RTX 3090 on Dankclaw): one `nvidia-smi --query-gpu` CSV. On
+    Dankserver there is an nvidia-smi binary but no NVIDIA GPU, so the command
+    fails; that is tolerated and simply yields no NVIDIA series.
+
+Default mode prints one scrape to stdout and exits (the test harness, and any
+`service ... --oneshot` one-off). `--serve` binds the HTTP endpoint for
+Prometheus; it binds EXPLICITLY (loopback + the host LAN address), never a
+wildcard, matching node-exporter's posture on these boxes.
+"""
+import argparse
+import glob
+import os
+import subprocess
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+FAMILIES = [
+    ("gpu_utilization_percent", "gauge", "GPU busy time as a percentage of wall time."),
+    ("gpu_memory_used_bytes", "gauge", "Device memory in use, in bytes."),
+    ("gpu_memory_total_bytes", "gauge", "Device memory capacity, in bytes."),
+    ("gpu_temperature_celsius", "gauge", "Package temperature in degrees Celsius."),
+    ("gpu_power_watts", "gauge", "Board power draw in watts."),
+]
+
+DEFAULT_NVIDIA_CMD = (
+    "nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,"
+    "memory.total,temperature.gpu,power.draw --format=csv,noheader,nounits"
+)
+
+
+def _read(path):
+    try:
+        with open(path) as fh:
+            return fh.read().strip()
+    except OSError:
+        return ""
+
+
+def _pci_slot(device_dir):
+    uevent = _read(os.path.join(device_dir, "uevent"))
+    for line in uevent.splitlines():
+        if line.startswith("PCI_SLOT_NAME="):
+            return line.split("=", 1)[1]
+    return ""
+
+
+def _pci_id(device_dir):
+    uevent = _read(os.path.join(device_dir, "uevent"))
+    for line in uevent.splitlines():
+        if line.startswith("PCI_ID="):
+            return "amd:" + line.split("=", 1)[1].lower()
+    return "amd"
+
+
+def collect_amd(sys_dir):
+    for card in sorted(glob.glob(os.path.join(sys_dir, "card[0-9]*"))):
+        device = os.path.join(card, "device")
+        busy = _read(os.path.join(device, "gpu_busy_percent"))
+        if busy == "":
+            continue  # not an amdgpu compute node (e.g. a render node w/o metrics)
+        labels = {"gpu": _pci_slot(device) or os.path.basename(card),
+                  "vendor": "amd", "model": _pci_id(device)}
+        yield "gpu_utilization_percent", labels, busy
+        for name, leaf in (("gpu_memory_used_bytes", "mem_info_vram_used"),
+                           ("gpu_memory_total_bytes", "mem_info_vram_total")):
+            v = _read(os.path.join(device, leaf))
+            if v != "":
+                yield name, labels, v
+        temp = ""
+        for hwmon in sorted(glob.glob(os.path.join(device, "hwmon", "hwmon*", "temp1_input"))):
+            temp = _read(hwmon)
+            if temp != "":
+                break
+        if temp != "":
+            yield "gpu_temperature_celsius", labels, str(int(temp) / 1000.0)
+        power = _read(os.path.join(device, "power1_average"))
+        if power != "":  # empty on the R9700s -> metric absent, not zero
+            yield "gpu_power_watts", labels, str(int(power) / 1e6)
+
+
+def collect_nvidia(cmd):
+    if not cmd:
+        return
+    try:
+        out = subprocess.run(cmd, shell=True, capture_output=True, text=True,
+                             timeout=10, check=True).stdout
+    except (subprocess.SubprocessError, OSError):
+        return  # no NVIDIA GPU / no driver: tolerate, emit nothing
+    for line in out.splitlines():
+        parts = [p.strip() for p in line.split(",")]
+        if len(parts) < 7:
+            continue
+        idx, name, util, mused, mtotal, temp, power = parts[:7]
+        labels = {"gpu": idx, "vendor": "nvidia", "model": name}
+        yield "gpu_utilization_percent", labels, util
+        yield "gpu_memory_used_bytes", labels, str(int(float(mused)) * 1048576)
+        yield "gpu_memory_total_bytes", labels, str(int(float(mtotal)) * 1048576)
+        if temp not in ("", "[N/A]"):
+            yield "gpu_temperature_celsius", labels, temp
+        if power not in ("", "[N/A]"):
+            yield "gpu_power_watts", labels, power
+
+
+def collect(sys_dir, nvidia_cmd):
+    samples = {}
+    for source in (collect_amd(sys_dir), collect_nvidia(nvidia_cmd)):
+        for name, labels, value in source:
+            try:
+                value = str(float(value))  # normalise; drop trailing junk
+            except ValueError:
+                continue
+            samples.setdefault(name, []).append((labels, value))
+    return samples
+
+
+def _fmt(value):
+    f = float(value)
+    if f == int(f) and abs(f) < 1e15:
+        return str(int(f))
+    return repr(f)
+
+
+def render(samples):
+    out = []
+    for name, kind, help_text in FAMILIES:
+        out.append("# HELP %s %s" % (name, help_text))
+        out.append("# TYPE %s %s" % (name, kind))
+        for labels, value in sorted(samples.get(name, []), key=lambda x: sorted(x[0].items())):
+            lset = ",".join('%s="%s"' % (k, v) for k, v in sorted(labels.items()))
+            out.append("%s{%s} %s" % (name, lset, _fmt(value)))
+    return "\n".join(out) + "\n"
+
+
+class Handler(BaseHTTPRequestHandler):
+    sys_dir = "/sys/class/drm"
+    nvidia_cmd = DEFAULT_NVIDIA_CMD
+
+    def do_GET(self):  # noqa: N802 (http.server API)
+        if self.path.split("?")[0] not in ("/metrics", "/"):
+            self.send_error(404)
+            return
+        body = render(collect(self.sys_dir, self.nvidia_cmd)).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, *args):
+        pass
+
+
+def main(argv):
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--amdgpu-sysfs", default="/sys/class/drm")
+    ap.add_argument("--nvidia-cmd", default=DEFAULT_NVIDIA_CMD)
+    ap.add_argument("--serve", action="store_true", help="serve /metrics instead of one-shot")
+    ap.add_argument("--port", type=int, default=9401)
+    ap.add_argument("--bind", action="append", default=None,
+                    help="explicit bind address (repeatable); env GPU_EXPORTER_BINDS "
+                         "comma-list if unset. NEVER a wildcard.")
+    args = ap.parse_args(argv)
+
+    if not args.serve:
+        sys.stdout.write(render(collect(args.amdgpu_sysfs, args.nvidia_cmd)))
+        return 0
+
+    binds = args.bind
+    if not binds:
+        env = os.environ.get("GPU_EXPORTER_BINDS", "")
+        binds = [b.strip() for b in env.split(",") if b.strip()] or ["127.0.0.1"]
+    Handler.sys_dir = args.amdgpu_sysfs
+    Handler.nvidia_cmd = args.nvidia_cmd
+    servers = []
+    for b in binds:
+        if b in ("0.0.0.0", "::", "*"):
+            sys.stderr.write("refusing wildcard bind %s\n" % b)
+            return 2
+        servers.append(ThreadingHTTPServer((b, args.port), Handler))
+    sys.stderr.write("gpu-exporter serving :%d on %s\n" % (args.port, ",".join(binds)))
+    # EVERY socket must be served: a listen()ed-but-never-accept()ed bind looks
+    # healthy in `ss` yet times out every scrape once its SYN backlog fills —
+    # and the loopback deploy gate cannot see it. One thread per extra bind.
+    for s in servers[1:]:
+        threading.Thread(target=s.serve_forever, daemon=True).start()
+    servers[0].serve_forever()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

scripts/gpu-exporter.service

@@ -0,0 +1,35 @@
+# Systemd USER unit: gpu-exporter (both Dankclaw and Dankserver).
+#
+# Install path: ~/.config/systemd/user/gpu-exporter.service
+# Installed and enabled by deploy_gpu_exporter in scripts/deploy-remote.sh.
+#
+# Per-GPU metrics (utilization / VRAM / temperature / power) for the mixed
+# AMD + NVIDIA fleet, scraped by Prometheus. Like signal-relay it is a plain
+# user unit, not a quadlet: it runs python3 straight from the checkout (a pull
+# updates it with no copy step), a hand-written user unit can be enabled where
+# the podman v1 generator's units cannot, and with Linger=yes for dank it starts
+# at boot. It reads the host's own cgroup/drm sysfs and shells to nvidia-smi, so
+# it must run on the host — a container would need GPU/DRM passthrough for
+# nothing gained.
+#
+# BINDING. The base unit serves loopback only. deploy_gpu_exporter writes a
+# drop-in adding THIS host's LAN address (the same two-address posture
+# node-exporter and cadvisor use), so the endpoint is never a wildcard. It is
+# unauthenticated metrics on the LAN, identical to node-exporter :9100 and
+# cadvisor :8080 — Prometheus reaches same-host via host.containers.internal and
+# the Dankserver instance over the LAN from Dankclaw.
+
+[Unit]
+Description=GPU metrics exporter (amdgpu sysfs + nvidia-smi)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+# Loopback by default; the deploy drop-in appends the host LAN address.
+Environment=GPU_EXPORTER_BINDS=127.0.0.1
+ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/gpu-exporter.py --serve
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=default.target

scripts/grams-to-imatrix.py

@@ -0,0 +1,418 @@
+#!/usr/bin/env python3
+"""grams-to-imatrix.py — the engine's own calibration recording, re-voiced as a
+llama.cpp importance matrix for `rad-convert --imatrix F`. RUNS ON DANKSERVER
+beside the recording. Python 3 stdlib only: numpy must NOT be required, because
+this runs in the plain container next to a 335 GiB checkpoint, not in a venv.
+
+WHY. `rad-convert` takes its per-column activation second moments and its
+expert-popularity profile from ONE input, an imatrix (core/format/imatrix.h).
+Only the GGUF form carries a per-expert dimension — with the legacy flat form,
+or with none, every expert in a layer is weighted by the layer's aggregate,
+which is exactly the wrong thing on a 512-expert MoE. The calibration tap has
+already measured everything; this tool only re-voices it, so the imatrix and the
+protected-expert census (derive-protected-experts.py) are the SAME recording:
+one measurement, two consumers, neither able to disagree about which experts are
+hot.
+
+INPUT. RADGRAM2 files in a RADIANCE_CALIB_DIR output directory
+(arch/common/rad_arch.h, calib_write). Header law, same parse as
+scripts/derive-protected-experts.py — 32 bytes, `<8sIIQI`: magic `RADGRAM2`,
+u32 k, u32 rank, u64 rows, u32 domain, a pad word, then k*k f32 row-major. Two
+kinds of name, both GGUF-style, both with the checkpoint's GLOBAL expert
+numbering (rad_block_moe_fp8.h: ge = expert_base + e):
+
+    gram.r<rank>.blk.L.ffn_down_exps.bin        the layer's pooled gram
+    gram.r<rank>.blk.L.ffn_down_exps.<e>.bin    expert e's own gram
+
+`rows` is that file's routing/token count and the matrix is Σ x xᵀ over those
+rows, so **the diagonal is exactly the per-column activation second moment an
+imatrix wants**, and `rows` is the count the reader hands back from `count()`.
+Every off-diagonal term is dropped on purpose: the GGUF format has nowhere to
+put a full matrix, and the quantiser asks for columns.
+
+OUTPUT, per layer L (core/format/imatrix.h: names WITHOUT `.weight`, a MoE
+layer's experts folded into ONE entry, so expert 9 is slice 9 of
+`blk.L.ffn_down_exps`):
+
+  blk.L.ffn_down_exps    in_sum2 [k, n_expert] expert-major, counts [1, n_expert]
+  blk.L.ffn_gate_up_exps the SAME matrices at the SAME offsets. A routed expert's
+                         gate_up input is the same hidden-state rows as its down
+                         input, so the second moment over them is identical and
+                         so is the routing count. Two tensor infos over one
+                         payload: the reader maps each half independently
+                         (imatrix.cpp joins `<base>.in_sum2` with `<base>.counts`
+                         by base name), and copying ~200 MB to state one fact
+                         twice buys no correctness.
+  blk.L.ssm_inz          [k] and [1], from the pooled gram: that row's input is
+                         the same hidden state, so its columns take the pooled
+                         diagonal and its count is the layer's pooled rows.
+
+  Then STOP. Nothing whose input width is not the gram's k — no ssm_out, no
+  attn_output, no mtp.*, no ple_ngram. importance_for() answers false when the
+  entry's cols do not equal the weight's width and the convert proceeds without
+  it: an entry the reader silently ignores is a lie in the artifact, worse than
+  an absent one. The corollary to check before trusting the two hidden-width
+  entries: the tap writes the gate_up gram at g.n_embd and the down gram at the
+  expert width, so the gram's k must equal the checkpoint's hidden width (its
+  config.json). Where they differ, gate_up/ssm_inz are inert, not wrong.
+
+INHERITANCE. An expert with no own gram carries the layer's POOLED diagonal, and
+its count stays its own routing rows — the rule the engine itself applies when it
+picks a Hessian (libquant/lq_hess.h, calib_resolve: an expert's own gram only
+once its rank files hold kOwnRowsPerK * K rows; below that a fit to so few rows
+is worse than round-to-nearest, and those experts carry a fraction of a percent
+of the routing). The count never inherits: popularity is `count`'s job. The count
+of inherited experts is reported on stderr.
+
+RANK FOLDING. TP=2 writes gram.r0.* and gram.r1.* with GLOBAL expert numbering,
+so the ranks describe DISJOINT experts and the fold is a UNION into the one
+[k, n_expert] entry. A pooled gram is the opposite case: every rank summed the
+same stem over its own rows, so the fold is a SUM, rows included (calib_sum,
+calib_rows). The same global expert in two per-expert files is legal only when
+the numbers agree exactly; otherwise the recording is corrupt and the entry would
+be whichever file sorted last, so refuse.
+
+GGUF v3 (the only version imatrix.cpp accepts — version != 3 is a refusal):
+`general.alignment` u32, `imatrix.chunk_size` u32, `imatrix.chunk_count` u32,
+`imatrix.datasets` array of strings; tensors f32 (ggml type 0) with offsets
+relative to the aligned data section. Every tensor offset is padded up to
+`general.alignment` and the data section itself starts aligned. That is the
+classic hand-rolled-writer bug: an unaligned offset makes the reader's extent
+arithmetic point into the middle of the previous tensor's floats, and it reads
+them without complaint.
+
+chunk_count DERIVATION. The corpus row count is the LARGEST single-layer pooled
+row total, not a sum over layers: every layer's tap accumulated over the whole
+corpus, so adding the layers counts each token once per layer and inflates the
+figure by the layer count. chunk_count = that row total // chunk_size, the
+"chunks of chunk_size" line imatrix.cpp prints for itself.
+
+REFUSALS, each exit 2 with a message naming what is missing:
+  NO_GRAMS         the directory holds no RADGRAM2 down-projection gram
+  NOT_RADGRAM2     a gram whose magic is not RADGRAM2
+  SHORT            a gram whose length is not 32 + k*k*4
+  NO_POOLED        a layer with expert grams and no pooled gram to inherit from
+  K_MISMATCH       a layer whose files disagree about k (rank pooled grams, or an
+                   expert gram against its layer's pooled gram)
+  EXPERT_CONFLICT  a global expert recorded twice with different numbers
+  EXISTS           the -o path already holds a file (the trash law: move it)
+  CHUNK_COUNT      the derived chunk count does not fit the u32 metadata field
+
+USAGE. grams-to-imatrix.py <calib-dir> -o <out.gguf>
+       [--datasets NAME,...] [--chunk-size N] [--min-rows N] [-v]
+"""
+import argparse
+import os
+import re
+import struct
+import sys
+
+MAGIC = b"RADGRAM2"
+HDR = 32                       # 8s I I Q I + pad, rad_arch.h
+KOWN_ROWS_PER_K = 4            # libquant/lq_hess.h kOwnRowsPerK
+ALIGNMENT = 32                 # general.alignment, and the pad unit
+
+POOLED_RE = re.compile(r"^gram\.r(\d+)\.blk\.(\d+)\.ffn_down_exps\.bin$")
+EXPERT_RE = re.compile(r"^gram\.r(\d+)\.blk\.(\d+)\.ffn_down_exps\.(\d+)\.bin$")
+
+GT_U32, GT_STR, GT_ARR = 4, 8, 9
+GGML_F32 = 0
+
+
+def refuse(msg):
+    sys.stderr.write("REFUSED %s\n" % msg)
+    sys.exit(2)
+
+
+def read_gram(path):
+    """(k, rank, rows, domain, diagonal) from one RADGRAM2 file."""
+    try:
+        with open(path, "rb") as f:
+            raw = f.read()
+    except OSError as e:
+        refuse("UNREADABLE %s: %s" % (path, e))
+    if len(raw) < HDR:
+        refuse("SHORT %s: %d bytes, under the %d-byte RADGRAM2 header"
+               % (path, len(raw), HDR))
+    magic, k, rank, rows, domain = struct.unpack_from("<8sIIQI", raw, 0)
+    if magic != MAGIC:
+        refuse("NOT_RADGRAM2 %s: magic %r, expected b'RADGRAM2'" % (path, magic))
+    if k <= 0:
+        refuse("SHORT %s: header claims k=%d, which describes no matrix" % (path, k))
+    need = HDR + k * k * 4
+    if len(raw) != need:
+        refuse("SHORT %s: %d bytes, expected %d (32 + k*k*4 for k=%d)"
+               % (path, len(raw), need, k))
+    # The diagonal only, read by stride out of the row-major matrix.
+    diag = [struct.unpack_from("<f", raw, HDR + (i * k + i) * 4)[0]
+            for i in range(k)]
+    return k, rank, rows, domain, diag
+
+
+def load_dir(d, verbose):
+    """{layer: {"pooled": gram|None, "experts": {global e: gram}}} + rotated count.
+
+    Files outside the ffn_down_exps stem are not ours to fold: the tap names a
+    pooled gram after every projection it taps, and those carry other widths."""
+    if not os.path.isdir(d):
+        refuse("NO_GRAMS %s: not a directory" % d)
+    layers, rotated = {}, 0
+    for name in sorted(os.listdir(d)):
+        m = POOLED_RE.match(name) or EXPERT_RE.match(name)
+        if not m:
+            continue
+        path = os.path.join(d, name)
+        if not os.path.isfile(path):
+            continue
+        pooled = bool(POOLED_RE.match(name))
+        layer = int(m.group(2))
+        k, rank, rows, domain, diag = read_gram(path)
+        if domain != 0:
+            rotated += 1
+        L = layers.setdefault(layer, {"pooled": None, "experts": {}})
+        if pooled:
+            P = L["pooled"]
+            if P is None:
+                L["pooled"] = {"k": k, "rows": rows, "diag": diag, "paths": [path]}
+            elif P["k"] != k:
+                refuse("K_MISMATCH layer %d: pooled grams %s (k=%d) and %s (k=%d)"
+                       " disagree about the width"
+                       % (layer, P["paths"][0], P["k"], path, k))
+            else:
+                # calib_sum: each rank summed the same stem over its own rows.
+                P["rows"] += rows
+                P["paths"].append(path)
+                for i in range(k):
+                    P["diag"][i] += diag[i]
+            if verbose:
+                sys.stderr.write("pooled layer %d: %s k=%d rows=%d\n"
+                                 % (layer, name, k, rows))
+            continue
+        e = int(m.group(3))
+        own = L["experts"].get(e)
+        if own is None:
+            L["experts"][e] = {"k": k, "rows": rows, "diag": diag,
+                               "path": path, "ranks": [rank]}
+            if verbose:
+                sys.stderr.write("expert layer %d r%d e%d: rows=%d\n"
+                                 % (layer, rank, e, rows))
+            continue
+        if own["rows"] != rows or own["diag"] != diag:
+            refuse("EXPERT_CONFLICT layer %d expert %d: %s (rows=%d) and %s"
+                   " (rows=%d) record the same global expert with different"
+                   " numbers" % (layer, e, own["path"], own["rows"], path, rows))
+        own["ranks"].append(rank)
+        sys.stderr.write("note: layer %d expert %d recorded identically by rank %s"
+                         " and %s; folded once\n"
+                         % (layer, e, own["path"], name))
+    if not layers:
+        refuse("NO_GRAMS %s: no RADGRAM2 gram.r<rank>.blk.<L>.ffn_down_exps"
+               "[.<expert>].bin files in the directory" % d)
+    return layers, rotated
+
+
+def build_entries(layers, min_rows, verbose):
+    """([ (name, dims, payload|None, key) ], census, layers emitted)."""
+    entries = []
+    census = {"inherited": 0, "experts": 0, "routing": 0, "rows": 0, "layers": 0}
+    for layer in sorted(layers):
+        P, E = layers[layer]["pooled"], layers[layer]["experts"]
+        if E and P is None:
+            refuse("NO_POOLED layer %d: %d expert gram(s) but no pooled gram (an"
+                   " expected gram.r<rank>.blk.%d.ffn_down_exps.bin), so an expert"
+                   " under the row threshold would have nothing to inherit"
+                   % (layer, len(E), layer))
+        if P is None:
+            continue
+        k = P["k"]
+        census["layers"] += 1
+        census["rows"] = max(census["rows"], P["rows"])
+        dn = "blk.%d.ffn_down_exps" % layer
+        # A staging key names ONE payload. A gate_up entry reuses the down
+        # layer's bytes by carrying the same key with no payload of its own;
+        # in_sum2 and counts are different bytes and must never share a key.
+        ksum, kcnt = (layer, "sum2"), (layer, "counts")
+        kpgm, kpcnt = (layer, "pooled_sum2"), (layer, "pooled_counts")
+        # One entry holds one width: an expert gram recorded at another k would
+        # inherit (or be inherited by) a diagonal of the wrong length, and the
+        # reader would then answer `cols` from whichever file set dims[0].
+        for e in sorted(E):
+            if E[e]["k"] != k:
+                refuse("K_MISMATCH layer %d expert %d: %s holds k=%d and this"
+                       " layer's pooled gram is k=%d" % (layer, e, E[e]["path"],
+                                                          E[e]["k"], k))
+
+        if not E:
+            # No per-expert recording for this layer: emit the dense form
+            # ([k], [1]) rather than 512 identical slices, which would claim a
+            # per-expert measurement that was never taken.
+            entries.append((dn + ".in_sum2", [k],
+                            struct.pack("<%df" % k, *P["diag"]), ksum))
+            entries.append((dn + ".counts", [1],
+                            struct.pack("<f", float(P["rows"])), kcnt))
+            sys.stderr.write("note: layer %d has no per-expert grams; %s is"
+                             " dense over the pooled gram\n" % (layer, dn))
+        else:
+            floor = min_rows if min_rows > 0 else KOWN_ROWS_PER_K * k
+            n_expert = max(E) + 1
+            sum2, counts = bytearray(), bytearray()
+            for e in range(n_expert):
+                own = E.get(e)
+                if own is not None and own["rows"] >= floor:
+                    src = own["diag"]
+                    rows = own["rows"]
+                else:
+                    src = P["diag"]
+                    rows = own["rows"] if own is not None else 0
+                    census["inherited"] += 1
+                sum2 += struct.pack("<%df" % k, *src)
+                counts += struct.pack("<f", float(rows))
+                census["experts"] += 1
+                census["routing"] += rows
+            if min(E) != 0:
+                sys.stderr.write("note: layer %d has no gram for experts 0..%d;"
+                                 " those slices carry the pooled gram at count 0\n"
+                                 % (layer, min(E) - 1))
+            entries.append((dn + ".in_sum2", [k, n_expert], bytes(sum2), ksum))
+            entries.append((dn + ".counts", [1, n_expert], bytes(counts), kcnt))
+            gu = "blk.%d.ffn_gate_up_exps" % layer
+            entries.append((gu + ".in_sum2", [k, n_expert], None, ksum))
+            entries.append((gu + ".counts", [1, n_expert], None, kcnt))
+            if verbose:
+                sys.stderr.write("layer %d: k=%d n_expert=%d pooled_rows=%d"
+                                 " own-gram floor=%d\n"
+                                 % (layer, k, n_expert, P["rows"], floor))
+
+        inz = "blk.%d.ssm_inz" % layer
+        entries.append((inz + ".in_sum2", [k],
+                        struct.pack("<%df" % k, *P["diag"]), kpgm))
+        entries.append((inz + ".counts", [1],
+                        struct.pack("<f", float(P["rows"])), kpcnt))
+    return entries, census
+
+
+def enc_str(v):
+    b = v.encode("utf-8")
+    return struct.pack("<Q", len(b)) + b
+
+
+def write_gguf(path, entries, datasets, chunk_size, chunk_count):
+    """Write GGUF v3 and return its size in bytes.
+
+    One payload per key: entries that share a key share the bytes and the
+    offset, so a second tensor info states the same fact without copying it."""
+    kv = bytearray()
+    kv += enc_str("general.alignment") + struct.pack("<I", GT_U32) \
+        + struct.pack("<I", ALIGNMENT)
+    kv += enc_str("imatrix.chunk_size") + struct.pack("<I", GT_U32) \
+        + struct.pack("<I", chunk_size)
+    kv += enc_str("imatrix.chunk_count") + struct.pack("<I", GT_U32) \
+        + struct.pack("<I", chunk_count)
+    kv += enc_str("imatrix.datasets") + struct.pack("<I", GT_ARR) \
+        + struct.pack("<I", GT_STR) + struct.pack("<Q", len(datasets))
+    for d in datasets:
+        kv += enc_str(d)
+
+    head = bytearray()
+    head += b"GGUF" + struct.pack("<I", 3)
+    head += struct.pack("<Q", len(entries)) + struct.pack("<Q", 4)
+    head += kv
+
+    payloads = {}
+    for name, _dims, payload, key in entries:
+        if payload is None:
+            continue
+        prev = payloads.get(key)
+        if prev is not None and prev != payload:
+            # Two different payloads on one key would ship whichever was
+            # appended last while BOTH tensor infos pointed at it: the counts
+            # payload landing over the in_sum2 payload is exactly how a
+            # hand-rolled GGUF writer loses half its data without an error.
+            refuse("INTERNAL %s: two different payloads share one staging key"
+                   % name)
+        payloads.setdefault(key, payload)
+    infos = bytearray()
+    offsets, order, size = {}, [], 0
+    for name, dims, payload, key in entries:
+        if key not in payloads:
+            refuse("INTERNAL %s: no payload staged for its key" % name)
+        if key not in offsets:
+            offsets[key] = (size + ALIGNMENT - 1) // ALIGNMENT * ALIGNMENT
+            size = offsets[key] + len(payloads[key])
+            order.append(key)
+        infos += enc_str(name) + struct.pack("<I", len(dims))
+        for d in dims:
+            infos += struct.pack("<Q", d)
+        infos += struct.pack("<I", GGML_F32) + struct.pack("<Q", offsets[key])
+
+    body = bytes(head) + bytes(infos)
+    data_start = (len(body) + ALIGNMENT - 1) // ALIGNMENT * ALIGNMENT
+    with open(path, "wb") as f:
+        f.write(body)
+        f.write(b"\0" * (data_start - len(body)))
+        # The offsets in the tensor infos are the file's law, not a hint: each
+        # payload must land at its own padded offset. Writing them back to back
+        # instead leaves every reader that trusts the info — imatrix.cpp does —
+        # reading its neighbour's floats: plausible values, no error, wrong
+        # quantisation.
+        here = data_start
+        for key in order:
+            f.write(b"\0" * (data_start + offsets[key] - here))
+            f.write(payloads[key])
+            here = data_start + offsets[key] + len(payloads[key])
+        return f.tell()
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Fold a radiance calibration recording into a GGUF v3 imatrix.")
+    ap.add_argument("calib_dir")
+    ap.add_argument("-o", "--out", required=True)
+    ap.add_argument("--datasets", default="",
+                    help="comma-separated names for imatrix.datasets")
+    ap.add_argument("--chunk-size", type=int, default=4096,
+                    help="rows per chunk for imatrix.chunk_size (default 4096)")
+    ap.add_argument("--min-rows", type=int, default=0,
+                    help="own-gram row threshold (default 4*k, the engine's"
+                         " kOwnRowsPerK law)")
+    ap.add_argument("-v", "--verbose", action="store_true")
+    a = ap.parse_args()
+
+    if os.path.exists(a.out):
+        # The repo's deletion law: an artifact that exists is moved to trash by
+        # a human, never silently overwritten by a re-run of the tool that made
+        # it, so a second convert cannot destroy the imatrix the first one fed.
+        refuse("EXISTS %s: an imatrix is already there; move it to"
+               " .AGENTTRASH/ or name another -o" % a.out)
+    if a.chunk_size <= 0:
+        refuse("BAD_CHUNK_SIZE --chunk-size %d is not positive" % a.chunk_size)
+    layers, rotated = load_dir(a.calib_dir, a.verbose)
+    entries, census = build_entries(layers, a.min_rows, a.verbose)
+    if not entries:
+        refuse("NO_GRAMS %s: no layer holds a pooled gram to fold" % a.calib_dir)
+    if rotated:
+        sys.stderr.write("note: %d gram(s) recorded in a rotated domain: their"
+                         " diagonal is the rotated activation's second moment,"
+                         " not the model's own\n" % rotated)
+
+    chunk_count = census["rows"] // a.chunk_size
+    if chunk_count > 0xFFFFFFFF:
+        # imatrix.chunk_count is a u32 in the reader's grammar, so there is no
+        # legal way to record a bigger one. Reaching this means either a chunk
+        # size picked for a far larger run or a row total read out of the wrong
+        # header offset — rows is a u64 and a u32 parse is off by 2**32 — and
+        # the reader would trust whatever number landed in the field.
+        refuse("CHUNK_COUNT %d = %d rows // %d does not fit the u32"
+               " imatrix.chunk_count field; raise --chunk-size"
+               % (chunk_count, census["rows"], a.chunk_size))
+    nbytes = write_gguf(a.out, entries, [d for d in a.datasets.split(",") if d],
+                        a.chunk_size, chunk_count)
+    sys.stderr.write("IMATRIX_OK entries=%d layers=%d experts=%d inherited=%d"
+                     " rows=%d routing=%d chunks=%d/%d bytes=%d out=%s\n"
+                     % (len(entries), census["layers"], census["experts"],
+                        census["inherited"], census["rows"], census["routing"],
+                        chunk_count, a.chunk_size, nbytes, a.out))
+
+
+if __name__ == "__main__":
+    main()

scripts/jellyfin-providers.sh

@@ -0,0 +1,391 @@
+#!/usr/bin/env bash
+# Metadata-provider keys for Jellyfin on Dankserver, placed from the service's
+# host env file with the plugin artifact pinned by content hash.
+#
+# Jellyfin 12 ships TMDb, OMDb, StudioImages and MusicBrainz as CORE plugins:
+# they have no DLL, only a configuration file under the data root, and that
+# file is created by the running container. So the fleet's provider state lives
+# in two places the repo does not track —
+#
+#   plugins/configurations/Jellyfin.Plugin.Tmdb.xml   <TmdbApiKey /> (empty = keyless)
+#   plugins/configurations/Jellyfin.Plugin.Tvdb.xml   <ApiKey />     (needs the plugin)
+#
+# and both are owned by the container user. This tool is the only thing that
+# writes them: it is declarative, idempotent, and it never prints a key, so it
+# is safe to run while someone watches.
+#
+# Keys never enter git and never enter argv (a value in argv is world-readable
+# through /proc/<pid>/cmdline). They live in the per-service host env file that
+# is the fleet's convention for container credentials (grafana, caddy, pihole,
+# copyparty, romm, seerr), mode 600:
+#
+#   /home/dank/Containers/jellyfin/provider-keys
+#       TMDB_API_KEY=<key>
+#       TVDB_API_KEY=<key>
+#       TVDB_PIN=<pin>        # optional: only a subscriber-shaped TVDB key
+#                             # needs one; a licensed key does not
+#
+# TVDB itself is not core in 12 (the metadata provider was cut in 10.9), so it
+# is installed from the official catalog, pinned three ways: sha256 of the
+# artifact, the plugin's own guid, and its version. A matching sha proves only
+# that the bytes are the bytes fetched — the guid check is what says it is the
+# official plugin, the version check is what says it is the one this file was
+# reviewed against.
+#
+# Subcommands
+#   plan      print the resolved shape and every pending change, touch nothing
+#   install   fetch+verify the plugin, then write the keys (backing up first)
+#   verify    prove the plugin and both keys are actually in place
+#
+# Nothing here restarts Jellyfin: the deployer owns that
+# (`scripts/deploy.sh dankserver --service jellyfin`), and the plugin is read at
+# start-up, so a key written while the server runs is not live until then.
+set -euo pipefail
+
+# Jellyfin's data root on Dankserver (the rootless quadlet's config volume).
+JF_ROOT="${JF_ROOT:-/home/dank/Containers/jellyfin/config}"
+PROV_KEYS="${PROV_KEYS:-/home/dank/Containers/jellyfin/provider-keys}"
+TRASH_DIR="${TRASH_DIR:-$HOME/.AGENTTRASH/$(date +%Y%m%d)-jellyfin-providers}"
+
+PLUGINS="$JF_ROOT/plugins"
+CONF="$PLUGINS/configurations"
+TMDB_XML="$CONF/Jellyfin.Plugin.Tmdb.xml"
+TVDB_XML="$CONF/Jellyfin.Plugin.Tvdb.xml"
+
+# repo.jellyfin.org catalog entry for TheTVDB (owner: jellyfin), version 24.0.0.0,
+# targetAbi 12.0.0.0 — published 2026-09-09, verified against Jellyfin 12.4.0.
+TVDB_PLUGIN_URL="${TVDB_PLUGIN_URL:-https://repo.jellyfin.org/files/plugin/thetvdb/thetvdb_24.0.0.0.zip}"
+TVDB_PLUGIN_SHA256="${TVDB_PLUGIN_SHA256:-6302fba3857bdde8d186dd099fac09ca43247041f531a528a2c788add70a5319}"
+TVDB_PLUGIN_GUID="${TVDB_PLUGIN_GUID:-a677c0da-fac5-4cde-941a-7134223f14c8}"
+TVDB_PLUGIN_VERSION="${TVDB_PLUGIN_VERSION:-24.0.0.0}"
+PLUGIN_DIR="thetvdb_${TVDB_PLUGIN_VERSION}"
+TVDB_DLL="Jellyfin.Plugin.Tvdb.dll"
+
+die() {
+    printf 'jellyfin-providers: %s\n' "$*" >&2
+    exit 1
+}
+
+usage() {
+    printf 'usage: %s plan|install|verify\n' "$0" >&2
+    exit 2
+}
+
+# check_keys — refuse anything that is not a readable private file. An open
+# mode is an error, not a warning: a provider key readable by every user on the
+# box is the failure this check exists to catch.
+check_keys() {
+    [ -f "$PROV_KEYS" ] ||
+        die "$PROV_KEYS not found (create it with TMDB_API_KEY= and TVDB_API_KEY=, then chmod 600)"
+    local mode
+    mode="$(stat -c '%a' "$PROV_KEYS")"
+    case "$mode" in
+    600 | 400) ;;
+    *) die "$PROV_KEYS is mode $mode; provider keys must be 600" ;;
+    esac
+}
+
+# keys_value <VAR> — the value of one variable in the env file, or nothing.
+keys_value() {
+    awk -v k="$1" -F= '$1 == k { sub(/^[^=]*=/, ""); print; exit }' "$PROV_KEYS"
+}
+
+# keys_len <VAR> — the length, and a refusal naming the variable when absent.
+# The value itself is never printed.
+keys_len() {
+    local v
+    v="$(keys_value "$1")"
+    [ -n "$v" ] || die "$PROV_KEYS has no $1= line"
+    printf '%s' "${#v}"
+}
+
+# xml_value <file> <element> — the element's text, empty when it is absent or
+# self-closing.
+xml_value() {
+    python3 - "$1" "$2" <<'PYEOF'
+import re, sys
+src = open(sys.argv[1], encoding="utf-8").read()
+m = re.search(r"<%s(?:\s*/>|>(.*?)</%s>)" % (sys.argv[2], sys.argv[2]), src, re.S)
+print(m.group(1).strip() if (m and m.group(1)) else "")
+PYEOF
+}
+
+# xml_secret <file> <element> <var> — set <element> to that variable from the
+# env file. The python process reads the env file itself, so the value never
+# appears in argv.
+xml_secret() {
+    python3 - "$1" "$2" "$3" "$PROV_KEYS" <<'PYEOF'
+import os, re, sys, tempfile
+from xml.sax.saxutils import escape
+
+xml_path, elem, var, keys_path = sys.argv[1:5]
+val = ""
+for line in open(keys_path, encoding="utf-8"):
+    key, sep, rest = line.partition("=")
+    if sep and key.strip() == var:
+        val = rest.strip()
+        break
+if not val:
+    raise SystemExit(f"{keys_path} has no {var}= line")
+
+src = open(xml_path, encoding="utf-8").read()
+pat = re.compile(r"([ \t]*)<%s(?:\s*/>|>.*?</%s>)" % (elem, elem), re.S)
+text = escape(val)
+if pat.search(src):
+    out = pat.sub(lambda m: f"{m.group(1)}<{elem}>{text}</{elem}>", src, count=1)
+else:
+    out = src.replace("</Configuration>", f"  <{elem}>{text}</{elem}>\n</Configuration>", 1)
+if out == src:
+    raise SystemExit(f"{xml_path}: no place to write <{elem}>")
+
+d, base = os.path.split(xml_path)
+fd, tmp = tempfile.mkstemp(prefix=base + ".", dir=d)
+with os.fdopen(fd, "w", encoding="utf-8") as fh:
+    fh.write(out)
+os.chmod(tmp, os.stat(xml_path).st_mode & 0o777)
+os.replace(tmp, xml_path)
+PYEOF
+}
+
+# plugin_meta <zip> — "guid version" out of the artifact's own meta.json.
+plugin_meta() {
+    python3 - "$1" <<'PYEOF'
+import json, sys, zipfile
+meta = json.loads(zipfile.ZipFile(sys.argv[1]).read("meta.json").decode("utf-8"))
+print(meta.get("guid", ""), meta.get("version", ""))
+PYEOF
+}
+
+check_root() {
+    # Never fabricate a data root: creating plugins/ beside a typo'd path would
+    # install a plugin Jellyfin never loads and hide the operator's mistake.
+    [ -d "$CONF" ] || die "no Jellyfin configuration directory at $CONF (is JF_ROOT right?)"
+}
+
+check_keys_report() {
+    check_keys
+    # A refusal inside a command substitution used as a printf argument would
+    # be swallowed, so the guard runs as its own statement.
+    keys_len TMDB_API_KEY >/dev/null
+    keys_len TVDB_API_KEY >/dev/null
+    printf '= keys read from %s (mode %s)\n' "$PROV_KEYS" "$(stat -c '%a' "$PROV_KEYS")"
+}
+
+# The plugin's state, as one line in the shared vocabulary.
+plugin_state() {
+    if [ -d "$PLUGINS/$PLUGIN_DIR" ]; then
+        printf '= %s already installed\n' "$PLUGIN_DIR"
+        return 0
+    fi
+    local other
+    other="$(find "$PLUGINS" -maxdepth 1 -type d -name 'thetvdb_*' ! -name "$PLUGIN_DIR" -printf '%f\n' 2>/dev/null | sort | tr '\n' ' ')"
+    if [ -n "$other" ]; then
+        printf '+ install %s (a different version is present: %s)\n' "$PLUGIN_DIR" "$other"
+        return 0
+    fi
+    printf '+ install %s from %s\n' "$PLUGIN_DIR" "${TVDB_PLUGIN_URL#*://}"
+}
+
+tmdb_state() {
+    if [ ! -f "$TMDB_XML" ]; then
+        printf '! %s is missing — save the TMDb plugin settings once in the UI, then re-run\n' "$TMDB_XML" >&2
+        return 1
+    fi
+    local now want
+    now="$(xml_value "$TMDB_XML" TmdbApiKey)"
+    want="$(keys_value TMDB_API_KEY)"
+    if [ -n "$now" ] && [ "$now" = "$want" ]; then
+        printf '= TmdbApiKey already set (%s chars)\n' "${#now}"
+    elif [ -n "$now" ]; then
+        printf '+ replace the TmdbApiKey in Jellyfin.Plugin.Tmdb.xml\n'
+    else
+        printf '+ set TmdbApiKey in Jellyfin.Plugin.Tmdb.xml\n'
+    fi
+}
+
+tvdb_state() {
+    local now
+    if [ -f "$TVDB_XML" ]; then
+        now="$(xml_value "$TVDB_XML" ApiKey)"
+        if [ -n "$now" ]; then
+            printf '= Jellyfin.Plugin.Tvdb.xml already has an ApiKey (kept)\n'
+        else
+            printf '+ fill the empty ApiKey in Jellyfin.Plugin.Tvdb.xml\n'
+        fi
+    else
+        printf '+ create Jellyfin.Plugin.Tvdb.xml with ApiKey\n'
+    fi
+}
+
+write_plugin() {
+    local tmp guid version other
+    other="$(find "$PLUGINS" -maxdepth 1 -type d -name 'thetvdb_*' ! -name "$PLUGIN_DIR" -printf '%f\n' 2>/dev/null | sort | tr '\n' ' ')"
+    [ -z "$other" ] ||
+        die "another TheTVDB version is already installed ($other); move it to the trash first — two copies of one plugin means two loads"
+
+    tmp="$(mktemp "${TMPDIR:-/tmp}/thetvdb.XXXXXX.zip")"
+    # The pins are checked before anything is unpacked, so a bad artifact never
+    # reaches the plugins directory.
+    curl -fsSL --max-time 300 -o "$tmp" "$TVDB_PLUGIN_URL" ||
+        { rm -f "$tmp"; die "could not fetch $TVDB_PLUGIN_URL"; }
+    local got
+    got="$(sha256sum "$tmp" | cut -d' ' -f1)"
+    if [ "$got" != "$TVDB_PLUGIN_SHA256" ]; then
+        rm -f "$tmp"
+        die "artifact sha256 is $got, pinned $TVDB_PLUGIN_SHA256 — refusing to install"
+    fi
+    read -r guid version <<<"$(plugin_meta "$tmp")"
+    if [ "$guid" != "$TVDB_PLUGIN_GUID" ]; then
+        rm -f "$tmp"
+        die "artifact guid is $guid, expected the official $TVDB_PLUGIN_GUID — refusing to install"
+    fi
+    if [ "$version" != "$TVDB_PLUGIN_VERSION" ]; then
+        rm -f "$tmp"
+        die "artifact version is $version, pinned $TVDB_PLUGIN_VERSION — refusing to install"
+    fi
+    mkdir -p "$PLUGINS/$PLUGIN_DIR"
+    python3 -m zipfile -e "$tmp" "$PLUGINS/$PLUGIN_DIR/"
+    rm -f "$tmp"
+    [ -f "$PLUGINS/$PLUGIN_DIR/$TVDB_DLL" ] ||
+        die "$PLUGIN_DIR has no $TVDB_DLL — the artifact does not match this plugin"
+    printf '+ installed %s\n' "$PLUGIN_DIR"
+}
+
+backup_to_trash() {
+    local f="$1"
+    mkdir -p "$TRASH_DIR"
+    local dest="$TRASH_DIR/$(basename "$f").$(date +%H%M%S)"
+    cp -p "$f" "$dest"
+    printf '+ previous copy kept at %s\n' "$dest"
+}
+
+cmd_write_keys() {
+    local now want
+    KEYS_CHANGED=0
+    # TMDb: the running container owns this file and every setting in it. Only
+    # the key element is touched, so country, language and adult settings
+    # survive the edit.
+    [ -f "$TMDB_XML" ] ||
+        die "$TMDB_XML is missing — save the TMDb plugin settings once in the UI, then re-run"
+    now="$(xml_value "$TMDB_XML" TmdbApiKey)"
+    keys_len TMDB_API_KEY >/dev/null
+    want="$(keys_value TMDB_API_KEY)"
+    if [ "$now" = "$want" ]; then
+        printf '= TmdbApiKey already set (%s chars)\n' "${#now}"
+    else
+        backup_to_trash "$TMDB_XML"
+        xml_secret "$TMDB_XML" TmdbApiKey TMDB_API_KEY
+        printf '+ set TmdbApiKey (%s chars)\n' "${#want}"
+        KEYS_CHANGED=1
+    fi
+
+    # TVDB: never overwrite a config the operator saved through the UI.
+    if [ -f "$TVDB_XML" ]; then
+        now="$(xml_value "$TVDB_XML" ApiKey)"
+        if [ -n "$now" ]; then
+            printf '= Jellyfin.Plugin.Tvdb.xml already has an ApiKey (kept)\n'
+        else
+            keys_len TVDB_API_KEY >/dev/null
+            backup_to_trash "$TVDB_XML"
+            xml_secret "$TVDB_XML" ApiKey TVDB_API_KEY
+            if [ -n "$(keys_value TVDB_PIN)" ] && [ -z "$(xml_value "$TVDB_XML" Pin)" ]; then
+                xml_secret "$TVDB_XML" Pin TVDB_PIN
+            fi
+            printf '+ filled the ApiKey in Jellyfin.Plugin.Tvdb.xml\n'
+            KEYS_CHANGED=1
+        fi
+    else
+        keys_len TVDB_API_KEY >/dev/null
+        cat >"$TVDB_XML" <<XML
+<?xml version="1.0" encoding="utf-8"?>
+<Configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+  <ApiKey />
+</Configuration>
+XML
+        xml_secret "$TVDB_XML" ApiKey TVDB_API_KEY
+        if [ -n "$(keys_value TVDB_PIN)" ]; then
+            xml_secret "$TVDB_XML" Pin TVDB_PIN
+        fi
+        printf '+ created Jellyfin.Plugin.Tvdb.xml with ApiKey\n'
+        KEYS_CHANGED=1
+    fi
+}
+
+cmd_plan() {
+    check_root
+    check_keys_report
+    printf 'root  %s\n' "$JF_ROOT"
+    plugin_state
+    tmdb_state
+    tvdb_state
+}
+
+cmd_install() {
+    check_root
+    check_keys_report
+    KEYS_CHANGED=0
+    if [ ! -d "$PLUGINS/$PLUGIN_DIR" ]; then
+        write_plugin
+        KEYS_CHANGED=1
+    else
+        printf '= %s already installed\n' "$PLUGIN_DIR"
+    fi
+    cmd_write_keys
+    if [ "$KEYS_CHANGED" -eq 1 ]; then
+        # Nothing is restarted here on purpose: this host may be playing media
+        # right now, and both the plugin and its keys are read at start-up.
+        printf 'next: scripts/deploy.sh dankserver --service jellyfin (the plugin and keys load at start-up)\n'
+    else
+        printf '= no change\n'
+    fi
+}
+
+cmd_verify() {
+    check_root
+    local bad=0
+    if [ ! -d "$PLUGINS/$PLUGIN_DIR" ]; then
+        printf 'MISSING plugin directory %s (thetvdb not installed)\n' "$PLUGIN_DIR" >&2
+        bad=1
+    else
+        local dlls abi
+        dlls="$(find "$PLUGINS/$PLUGIN_DIR" -maxdepth 1 -name '*.dll' | wc -l)"
+        [ "$dlls" -ge 1 ] || {
+            printf 'MISSING %s inside %s\n' "$TVDB_DLL" "$PLUGIN_DIR" >&2
+            bad=1
+        }
+        abi="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("targetAbi","?"))' \
+            "$PLUGINS/$PLUGIN_DIR/meta.json" 2>/dev/null || echo '?')"
+        printf 'plugin %s: %s dll(s), targetAbi %s\n' "$PLUGIN_DIR" "$dlls" "$abi"
+    fi
+    local pair elem n
+    for pair in "$TMDB_XML|TmdbApiKey" "$TVDB_XML|ApiKey"; do
+        elem="${pair##*|}"
+        if [ ! -f "${pair%%|*}" ]; then
+            printf 'MISSING %s\n' "${pair%%|*}" >&2
+            bad=1
+            continue
+        fi
+        if ! python3 -c 'import sys, xml.dom.minidom; xml.dom.minidom.parse(sys.argv[1])' "${pair%%|*}"; then
+            printf 'UNPARSEABLE %s\n' "${pair%%|*}" >&2
+            bad=1
+            continue
+        fi
+        n="$(xml_value "${pair%%|*}" "$elem")"
+        if [ -z "$n" ]; then
+            printf 'MISSING key: %s has no %s value\n' "$(basename "${pair%%|*}")" "$elem" >&2
+            bad=1
+        else
+            printf '%s %s chars\n' "$elem" "${#n}"
+        fi
+    done
+    [ "$bad" -eq 0 ] || return 1
+    printf 'providers configured\n'
+}
+
+cmd="${1:-}"
+case "$cmd" in
+plan) cmd_plan ;;
+install) cmd_install ;;
+verify) cmd_verify ;;
+*) usage ;;
+esac

scripts/kld-gate-qwen38.sh

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+# kld-gate-qwen38.sh — RUNS ON DANKSERVER (or anywhere the reports are). The campaign's
+# acceptance gate: one verdict from the KLD reports of the containers that ride one reference
+# recording.
+#
+# The reference is the BF16 container built from the vendor checkpoint (run-qwen38-bf16.sh
+# kldref, in the same drained window that records the calibration grams). Every scored
+# container is measured against that one recording, so the KL scale runs from 0 = the
+# checkpoint itself downward in faithfulness.
+#
+# TWO MODES, because the disk decides how many anchors can exist at once:
+#
+#   gate IQ4R.json CAND.json                    candidate vs the served payload
+#   gate FP8.json IQ4R.json CAND.json           the same, inside the absolute interval
+#
+# iq4r.json is the payload frank-archon serves — the bar to beat. When the FP8-native release
+# (172.8 GiB, fetched last because of the peak-disk law) is on disk, pass its report first:
+# "half the gap" is then half of the distance between the 4-bit payload and the near-lossless
+# release, not half of an invented distance to the checkpoint. Without it the gate still
+# answers the campaign's question — does the candidate beat what is served — and says so.
+#
+# The law, on the summary of one slice (--slice all | a corpus source):
+#
+#   candidate vs served (2 reports)
+#   1. mean KL(cand) < mean KL(iq4r)         the served payload is beaten on the corpus
+#   2. p99.9(cand) <= p99.9(iq4r)            and the tail is no worse — a mean can hide a few
+#                                            positions taking the whole hit
+#   3. top-1(cand) >= top-1(iq4r)            the agreement the harness keys on does not fall
+#   4. every source slice passes 1-3         a win bought by regressing one distribution
+#                                            (code, prose, tools) is not a win
+#
+#   inside the interval (3 reports), laws 1-4 above (1-3 renumbered to the interval and the
+#   recovery fraction, as agreed 2026-10-27) plus:
+#   5. p99.9(cand) <= p99.9(iq4r)            the tail law
+#   6. every source slice passes            the slice law
+#
+# Booting the containers is NOT this script's job: `run-qwen38-bf16.sh score RAD OUT.json`
+# writes the reports and owns the single engine argv (same pin, --tp 2, --kv-cache-dtype fp8,
+# same step size, --deterministic, no speculation) that makes the numbers comparable.
+# This script is arithmetic, so it re-runs against old reports and is testable without a GPU.
+#
+# Usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]
+#        kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]
+set -uo pipefail
+
+if [ "$#" -lt 2 ]; then
+    echo "usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]" >&2
+    echo "       kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]" >&2
+    exit 2
+fi
+python3 - "$@" <<'PY'
+import json, sys
+
+a = sys.argv[1:]
+opts, files = [], []
+i = 0
+while i < len(a):
+    if a[i].startswith("--"):
+        opts += [a[i], a[i + 1]]
+        i += 2
+    else:
+        files.append(a[i])
+        i += 1
+if len(files) not in (2, 3):
+    sys.exit("usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]\n"
+             "       kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]")
+o = dict(zip(opts[0::2], opts[1::2]))
+slice_ = o.get("--slice", "all")
+recover = float(o.get("--recover", 0.5))
+if not 0.0 < recover <= 1.0:
+    sys.exit(f"--recover must be in (0,1], got {recover}")
+interval = len(files) == 3
+
+
+def die(msg):
+    print(msg, file=sys.stderr)
+    raise SystemExit(2)
+
+
+def load(path):
+    try:
+        return json.load(open(path))
+    except Exception as e:
+        die(f"cannot read the KLD report {path}: {e}")
+
+
+def summary(path, d):
+    """The slice this gate grades: 'all', or one corpus source."""
+    if slice_ == "all":
+        g = d.get("all")
+    else:
+        g = (d.get("by_source") or {}).get(slice_)
+        if g is None:
+            die(f"{path} has no slice {slice_!r}; it has: "
+                f"{', '.join(sorted(d.get('by_source') or {})) or '(none)'}")
+    if g is None or "kld" not in g or "top1_agreement" not in g:
+        die(f"{path} has no {slice_!r} summary — wrong report version?")
+    return g
+
+
+docs = {label: load(path) for label, path in
+        zip(("fp8", "iq4r", "cand") if interval else ("iq4r", "cand"), files)}
+rows = []
+for label, path in zip(("fp8", "iq4r", "cand") if interval else ("iq4r", "cand"), files):
+    g = summary(path, docs[label])
+    rows.append((label, path, docs[label].get("model", "?"), g["kld"]["mean"],
+                 g["kld"]["p99_9"], g["top1_agreement"], g["positions"]))
+
+print(f"reference: BF16 container; slice: {slice_}; "
+      + ("inside the FP8 interval" if interval else "candidate vs served (no FP8 anchor)"))
+print(f"{'':6} {'model':38} {'KL mean':>10} {'KL p99.9':>10} {'top-1':>8}  positions")
+for label, path, model, kl, p999, top1, n in rows:
+    print(f"{label:6} {str(model)[:38]:38} {kl:10.6f} {p999:10.4f} {100*top1:7.2f}%  {n}")
+r = {label: (kl, p999, top1) for label, _p, _m, kl, p999, top1, _n in rows}
+kl_cand, p_cand, t_cand = r["cand"]
+kl_iq4r, p_iq4r, t_iq4r = r["iq4r"]
+ok = True
+
+
+def law(n, text, passed):
+    global ok
+    ok = ok and passed
+    print(f"  [{'PASS' if passed else 'FAIL'}] {n}. {text}")
+
+
+def tail_law(n):
+    law(n, f"candidate KL p99.9 {p_cand:.4f} <= served iq4r {p_iq4r:.4f}",
+        p_cand <= p_iq4r)
+
+
+# --- the slice law: every corpus source must pass the metric laws -------------
+def slice_law(n):
+    """A win that regresses one source class is not a win: check them all."""
+    mine = docs["cand"].get("by_source") or {}
+    theirs = docs["iq4r"].get("by_source") or {}
+    if not mine or not theirs:
+        law(n, "no per-source summaries to check", True)
+        return
+    missing = sorted(set(theirs) - set(mine))
+    extra = sorted(set(mine) - set(theirs))
+    if missing or extra:
+        law(n, "same source slices in both reports"
+            + (f" (missing from cand: {', '.join(missing)})" if missing else "")
+            + (f" (only in cand: {', '.join(extra)})" if extra else ""), False)
+        return
+    bad = []
+    for src in sorted(theirs):
+        c, q = mine[src], theirs[src]
+        worse_mean = c["kld"]["mean"] >= q["kld"]["mean"]
+        worse_tail = c["kld"]["p99_9"] > q["kld"]["p99_9"]
+        worse_top1 = c["top1_agreement"] < q["top1_agreement"]
+        if worse_mean or worse_tail or worse_top1:
+            why = ",".join(m for m, w in (("mean", worse_mean), ("p99.9", worse_tail),
+                                          ("top-1", worse_top1)) if w)
+            bad.append(f"{src} ({why})")
+    law(n, "every source slice passes (mean, p99.9, top-1)"
+        + (f" — regressed: {'; '.join(bad)}" if bad else ""), not bad)
+
+
+if interval:
+    kl_fp8 = r["fp8"][0]
+    gap = kl_iq4r - kl_fp8
+    law(1, f"anchors ordered: fp8 {kl_fp8:.6f} < iq4r {kl_iq4r:.6f} (gap {gap:.6f})",
+        gap > 0)
+    law(2, f"candidate inside the interval: {kl_fp8:.6f} < {kl_cand:.6f} < {kl_iq4r:.6f}",
+        kl_fp8 < kl_cand < kl_iq4r)
+    need = kl_iq4r - recover * gap
+    law(3, f"candidate closes {100*max((kl_iq4r - kl_cand)/gap, 0):.1f}% of the "
+           f"{gap:.6f} gap (needs {100*recover:.0f}%, i.e. KL <= {need:.6f})"
+           if gap > 0 else "gap is not positive; cannot score recovery",
+        gap > 0 and kl_cand <= need)
+    law(4, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%",
+        t_cand >= t_iq4r)
+    tail_law(5)
+    slice_law(6)
+else:
+    law(1, f"candidate beats the served payload: KL {kl_cand:.6f} < iq4r {kl_iq4r:.6f}",
+        kl_cand < kl_iq4r)
+    tail_law(2)
+    law(3, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%",
+        t_cand >= t_iq4r)
+    slice_law(4)
+
+print("KLD GATE " + ("PASS" if ok else "FAIL"))
+sys.exit(0 if ok else 1)
+PY

scripts/make-calib-corpus.py

@@ -34,14 +34,25 @@ import json
 import os
 import random
 import re
+import sys
 
 from jinja2 import Environment, FileSystemLoader
-from tokenizers import Tokenizer
+
+_HERE = os.path.dirname(os.path.abspath(__file__)) if "__file__" in globals() else os.getcwd()
+if _HERE not in sys.path:
+    sys.path.insert(0, _HERE)
+
+import qwen38_common as qc  # noqa: E402 - sibling module, same directory
 
 CODE_EXT = (".sh", ".py", ".jinja", ".recipe", ".env", ".example",
             ".yaml", ".yml", ".bats", ".service", ".container", ".toml")
 JSONL_EXTS = (".jsonl", ".json")
 DOC_DIR = "serverAppConfigs-docs"
+FRAGMENT_TOKENS = 2400   # what one fragment may contribute
+CLASS_MIX = {"code": .45, "prose": .30, "tools": .25}
+HF_CAP = {"code": 40000, "prose": 60000, "tools": 0}
+POOL_CYCLES = 40         # how many times a class pool may repeat before giving up
+KLD_GUARD = 5000         # rendered conversations tried before declaring the pool short
 
 FLEET_CMDS = [
     "systemctl status frank-archon --no-pager",
@@ -142,7 +153,7 @@ def load_parquet_texts(path: str, kind: str, rng: random.Random,
     return out[:cap]
 
 
-def main() -> None:
+def parse_args() -> argparse.Namespace:
     ap = argparse.ArgumentParser(description=__doc__)
     ap.add_argument("--repo", required=True)
     ap.add_argument("--tokenizer-json", required=True)
@@ -156,22 +167,26 @@ def main() -> None:
     ap.add_argument("--hf-code-parquet", default="")
     ap.add_argument("--hf-prose-parquet", default="")
     ap.add_argument("--seed", type=int, default=20261006)
-    a = ap.parse_args()
+    return ap.parse_args()
 
-    rng = random.Random(a.seed)
-    tokenizer = Tokenizer.from_file(a.tokenizer_json)
-    env = Environment(loader=FileSystemLoader(os.path.dirname(a.template)),
+
+def load_template(template: str) -> tuple[Environment, str]:
+    """The chat template as the engine will render it, plus its short name."""
+    env = Environment(loader=FileSystemLoader(os.path.dirname(template)),
                       trim_blocks=True, lstrip_blocks=True)
     env.filters["tojson"] = lambda v: json.dumps(v, ensure_ascii=False)
     env.filters["raise_exception"] = lambda m: (_ for _ in ()).throw(
         ValueError(m))
-    tname = os.path.basename(a.template)
+    return env, os.path.basename(template)
 
+
+def classify(repo: str) -> dict[str, list[str]]:
+    """Walk the checkout into the three source classes the mix draws from."""
     code, prose, jsonl = [], [], []
-    for root, dirs, files in os.walk(a.repo):
+    for root, dirs, files in os.walk(repo):
         dirs[:] = [d for d in dirs if not d.startswith(".")]
         for f in sorted(files):
-            p = os.path.relpath(os.path.join(root, f), a.repo)
+            p = os.path.relpath(os.path.join(root, f), repo)
             if DOC_DIR in p and f.endswith(".md"):
                 prose.append(p)
             elif p.startswith(DOC_DIR):
@@ -180,137 +195,185 @@ def main() -> None:
                 code.append(p)
             elif f.endswith(JSONL_EXTS) and f != "package-lock.json":
                 jsonl.append(p)
-    # file-level split: bucket 0 -> KLD pool, 1..4 -> calibration pool
-    pools: dict[str, tuple[list, list]] = {}
-    for cls, lst in (("code", code), ("prose", prose), ("tools", jsonl)):
-        pools[cls] = ([p for p in lst if fnv(p) % 5 == 0],
-                      [p for p in lst if fnv(p) % 5 != 0])
+    return {"code": code, "prose": prose, "tools": jsonl}
 
-    def text_of(p: str) -> str:
-        path = os.path.join(a.repo, p)
-        raw = open(path, encoding="utf-8", errors="replace").read()
-        if p.endswith(JSONL_EXTS):
-            lines = [json.loads(l)["content"] for l in raw.splitlines()
-                     if l.strip() and "content" in l]
-            raw = "\n".join(lines)
-        toks = tokenizer.encode(raw, add_special_tokens=False).ids
-        return tokenizer.decode(toks[:2400], skip_special_tokens=True)
 
+def split_pools(classes: dict[str, list[str]]) -> dict[str, tuple[list, list]]:
+    """File-level split: bucket 0 -> KLD pool, everything else -> calibration."""
+    return {cls: ([p for p in lst if fnv(p) % 5 == 0],
+                  [p for p in lst if fnv(p) % 5 != 0])
+            for cls, lst in classes.items()}
+
+
+def jsonl_content(raw: str) -> str:
+    """The content fields of a content-keyed JSONL, or "" for anything else.
+
+    A .json in this repo is not necessarily a content-keyed JSONL: the substring
+    test passes on any line that merely mentions "content", and a list-shaped
+    document raises TypeError — both used to crash the whole build. Skip what
+    does not fit instead.
+    """
+    lines = []
+    for line in raw.splitlines():
+        if not line.strip() or "content" not in line:
+            continue
+        try:
+            rec = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(rec, dict) and isinstance(rec.get("content"), str):
+            lines.append(rec["content"])
+    return "\n".join(lines)
+
+
+def fragment_text(repo: str, rel: str, tok) -> str:
+    """One file as a fragment: its first FRAGMENT_TOKENS tokens, decoded."""
+    raw = open(os.path.join(repo, rel), encoding="utf-8",
+               errors="replace").read()
+    if rel.endswith(JSONL_EXTS):
+        raw = jsonl_content(raw)
+    ids = tok.encode(raw, add_special_tokens=False).ids
+    return tok.decode(ids[:FRAGMENT_TOKENS], skip_special_tokens=True)
+
+
+def build_frags(a: argparse.Namespace, tok,
+                rng: random.Random) -> dict[str, dict[str, list[str]]]:
+    """Both fragment pools per class, seeded so the same seed rebuilds the corpus."""
+    pools = split_pools(classify(a.repo))
     cmds = harvest_cmds(a.repo, rng)
-    # fragment pools per class, corpus-split first
     frags: dict[str, dict[str, list[str]]] = {}
-    for cls in pools:
-        kld_files, cal_files = pools[cls]
-        fr = {}
-        src = {"kld": kld_files, "cal": cal_files}
+    for cls, (kld_files, cal_files) in pools.items():
         if cls == "tools":
-            fr["kld"] = [tool_call(rng, cmds) for _ in range(600)]
-            fr["cal"] = [tool_call(rng, cmds) for _ in range(40000)]
+            frags[cls] = {"kld": [tool_call(rng, cmds) for _ in range(600)],
+                          "cal": [tool_call(rng, cmds) for _ in range(40000)]}
         else:
-            fr["kld"] = [text_of(p) for p in src["kld"]]
-            fr["cal"] = [text_of(p) for p in src["cal"]]
-        frags[cls] = fr
+            frags[cls] = {"kld": [fragment_text(a.repo, p, tok) for p in kld_files],
+                          "cal": [fragment_text(a.repo, p, tok) for p in cal_files]}
     # per-class fragment caps: prose rows are short (~66 tok) so need many;
     # codeparrot files are long (~2.6k tok) so a small cap suffices
-    hfcap = {"code": 40000, "prose": 60000, "tools": 0}
     for cls, spec in (("code", a.hf_code_parquet),
                       ("prose", a.hf_prose_parquet)):
         for path in [p for p in spec.split(",") if p]:
             frags[cls]["cal"].extend(
-                load_parquet_texts(path, cls, rng, cap=hfcap[cls]))
+                load_parquet_texts(path, cls, rng, cap=HF_CAP[cls]))
     for cls in frags:
         rng.shuffle(frags[cls]["cal"])
+    return frags
 
-    def toks(s: str) -> int:
-        return len(tokenizer.encode(s, add_special_tokens=False).ids)
 
-    def emit_calib() -> tuple[int, list[dict]]:
-        targets = {"code": .45, "prose": .30, "tools": .25}
-        rows: list[dict] = []
-        for cls, frac in targets.items():
-            want = int(a.calib_tokens * frac)
-            pool = frags[cls]["cal"]
+def pack_class(cls: str, pool: list[str], want: int, tok,
+               chunk_tokens: int) -> list[dict]:
+    """Greedily pack `pool` into chunk_tokens-sized calibration prompts."""
+    if not pool:
+        return []
+    rows: list[dict] = []
+    # Count every fragment once: the loop cycles the pool up to POOL_CYCLES
+    # times, and re-tokenizing a 2.4k-token fragment on each pass dominated the
+    # build's wall time.
+    counts = [qc.n_tokens(tok, f) for f in pool]
+    got = 0
+    i = 0
+    buf: list[str] = []
+    nbuf = 0
+    while got < want:
+        j = i % len(pool)
+        frag = pool[j]
+        i += 1
+        n = counts[j]
+        if nbuf + n > chunk_tokens and buf:
+            rows.append({"prompt": "\n".join(buf), "cls": cls})
+            got += nbuf
+            buf, nbuf = [], 0
+            if got >= want:
+                break
+        buf.append(frag)
+        nbuf += n
+        if i > len(pool) * POOL_CYCLES:
+            break  # pool exhausted with repeats; stop
+    if buf and got < want:
+        rows.append({"prompt": "\n".join(buf), "cls": cls})
+        got += nbuf
+    print(f"calib {cls}: {got} tokens, {i} fragments")
+    return rows
+
+
+def emit_calib(a: argparse.Namespace, frags, tok,
+               rng: random.Random) -> tuple[int, list[dict]]:
+    """calib.jsonl rows: the class mix, shuffled, prompts only."""
+    rows: list[dict] = []
+    for cls, frac in CLASS_MIX.items():
+        rows.extend(pack_class(cls, frags[cls]["cal"], int(a.calib_tokens * frac),
+                               tok, a.chunk_tokens))
+    rng.shuffle(rows)
+    return len(rows), [{"prompt": r["prompt"]} for r in rows]
+
+
+def emit_kld(a: argparse.Namespace, frags, tok, rng: random.Random,
+             env: Environment, tname: str) -> tuple[int, list[dict], int]:
+    """kld.jsonl rows: multi-turn conversations, each with a scored tail."""
+    template = env.get_template(tname)
+    rows: list[dict] = []
+    total = scored_total = 0
+    classes = ["code", "prose", "tools"]
+    cursor = {c: 0 for c in classes}
+    guard = 0
+    while total < a.kld_tokens and guard < KLD_GUARD:
+        guard += 1
+        msgs, srcs = [], []
+        for t in range(rng.choice([2, 3, 4])):
+            cls = classes[(len(rows) + t) % 3]
+            pool = frags[cls]["kld"]
             if not pool:
                 continue
-            got, i, buf, nbuf = 0, 0, [], 0
-            while got < want:
-                frag = pool[i % len(pool)]
-                i += 1
-                n = toks(frag)
-                if nbuf + n > a.chunk_tokens and buf:
-                    rows.append({"prompt": "\n".join(buf), "cls": cls})
-                    got += nbuf
-                    buf, nbuf = [], 0
-                    if got >= want:
-                        break
-                buf.append(frag)
-                nbuf += n
-                if i > len(pool) * 40:
-                    break  # pool exhausted with repeats; stop
-            if buf and got < want:
-                rows.append({"prompt": "\n".join(buf), "cls": cls})
-                got += nbuf
-            print(f"calib {cls}: {got} tokens, {i} fragments")
-        rng.shuffle(rows)
-        return len(rows), [{k: v for k, v in r.items() if k == "prompt"}
-                           for r in rows]
+            frag = pool[cursor[cls] % len(pool)]
+            cursor[cls] += 1
+            msgs.append({"role": "user",
+                         "content": "Summarize and explain this "
+                                    f"engineering material:\n{frag}"})
+            msgs.append({"role": "assistant",
+                         "content": frag[:4000]})
+            srcs.append(cls)
+        rendered = template.render(messages=msgs, add_generation_prompt=False)
+        n = qc.n_tokens(tok, rendered)
+        if n < a.kld_min_prompt:
+            continue
+        rows.append({"prompt": rendered,
+                     "score_from": n - a.score_tail,
+                     "source": "+".join(sorted(set(srcs)))})
+        total += n
+        scored_total += a.score_tail
+    return len(rows), rows, scored_total
 
-    def emit_kld() -> tuple[int, list[dict]]:
-        rows, total = [], 0
-        classes = ["code", "prose", "tools"]
-        i = {c: 0 for c in classes}
-        guard = 0
-        while total < a.kld_tokens and guard < 5000:
-            guard += 1
-            msgs, srcs = [], []
-            for t in range(rng.choice([2, 3, 4])):
-                cls = classes[(len(rows) + t) % 3]
-                pool = frags[cls]["kld"]
-                if not pool:
-                    continue
-                frag = pool[i[cls] % len(pool)]
-                i[cls] += 2
-                msgs.append({"role": "user",
-                             "content": "Summarize and explain this "
-                                        f"engineering material:\n{frag}"})
-                msgs.append({"role": "assistant",
-                             "content": frag[:4000]})
-                srcs.append(cls)
-            rendered = env.get_template(tname).render(
-                messages=msgs, add_generation_prompt=False)
-            n = toks(rendered)
-            if n < a.kld_min_prompt:
-                continue
-            rows.append({"prompt": rendered,
-                         "score_from": n - a.score_tail,
-                         "source": "+".join(sorted(set(srcs)))})
-            total += n
-        return len(rows), rows
 
-    os.makedirs(a.out_dir, exist_ok=True)
-    nc, rows = emit_calib()
-    with open(os.path.join(a.out_dir, "calib.jsonl"), "w") as f:
-        for r in rows:
-            f.write(json.dumps(r) + "\n")
-    nk, krows = emit_kld()
-    with open(os.path.join(a.out_dir, "kld.jsonl"), "w") as f:
-        for r in krows:
-            f.write(json.dumps(r) + "\n")
-    scored = sum(len(tokenizer.encode(r["prompt"], add_special_tokens=False).ids)
-                 - r["score_from"] for r in krows)
-    manifest = {
+def manifest(a: argparse.Namespace, tname: str, nc: int, nk: int,
+             scored: int) -> dict:
+    return {
         "seed": a.seed, "template": tname,
         "tokenizer": os.path.basename(a.tokenizer_json),
         "calib_prompts": nc, "kld_prompts": nk,
         "kld_scored_positions": scored,
         "kld_score_tail": a.score_tail,
         "disjoint_file_pools": "local KLD material only; HF code feeds calib",
-        "calib_mix": {"code": .45, "prose": .30, "tools": .25},
+        "calib_mix": dict(CLASS_MIX),
         "hf_code": a.hf_code_parquet, "hf_prose": a.hf_prose_parquet,
     }
-    with open(os.path.join(a.out_dir, "manifest.json"), "w") as f:
-        json.dump(manifest, f, indent=1)
-    print(json.dumps(manifest, indent=1))
+
+
+def main() -> None:
+    a = parse_args()
+    rng = random.Random(a.seed)
+    tok = qc.load_tokenizer(a.tokenizer_json)
+    env, tname = load_template(a.template)
+
+    frags = build_frags(a, tok, rng)
+    os.makedirs(a.out_dir, exist_ok=True)
+    nc, calib_rows = emit_calib(a, frags, tok, rng)
+    qc.write_jsonl(os.path.join(a.out_dir, "calib.jsonl"), calib_rows)
+    nk, kld_rows, scored = emit_kld(a, frags, tok, rng, env, tname)
+    qc.write_jsonl(os.path.join(a.out_dir, "kld.jsonl"), kld_rows)
+    man = manifest(a, tname, nc, nk, scored)
+    qc.write_json(os.path.join(a.out_dir, "manifest.json"), man)
+    print(json.dumps(man, indent=1))
 
 
 if __name__ == "__main__":

scripts/media-apply.py

@@ -0,0 +1,454 @@
+#!/usr/bin/env python3
+"""media-apply.py — the only thing that moves movie-library files.
+
+Reads the plan written by scripts/media-plan.py (rows: `action  src  dst  kind`,
+plus `mkdir` rows) and performs it with rename(2) — bytes are never copied, so
+an interrupted run has destroyed nothing. Every run is resumable and every run
+is undoable: the journal is a write-ahead log, one JSON line per intent, fsynced
+BEFORE the rename it authorises. That ordering is the whole design: a crash
+leaves either an intent whose move happened or an intent whose move never ran,
+and both are recognised on the next run. A file that moved with no journal line
+cannot have been moved by this tool, and saying so is better than guessing.
+
+Four subcommands:
+
+    apply    --plan PLAN [--root ROOT] [--journal J] [--apply]
+             no --apply => report only and write NOTHING, not even folders
+    rollback --journal J [--root ROOT] [--apply]
+             back to the paths the census saw; folders this run created are
+             removed only when empty, never anything else
+    verify   --plan PLAN --journal J [--root ROOT]
+             every planned file is at its target with the journalled byte count
+    sweep    --journal J [--root ROOT] [--apply]
+             removes the folders THIS RUN emptied — the journal has to show a
+             file that moved out of one, so a folder someone made by hand can
+             never be removed by this tool, and rollback puts them back
+
+Refusals (exit 2, before the first rename, with nothing touched): a target
+already occupied by a foreign file, a source that has gone, a duplicate target,
+a path outside --root, a journal that was already rolled back, a move that would
+cross a filesystem (rename cannot do it and a silent copy at 24 TB would not).
+
+--force relaxes only the guards that are opinion (journal already rolled back,
+paths outside --root); it never allows overwriting a file.
+
+--metadata-view names the overlay the media servers read their library through
+(default /var/lib/media-meta/view). The sidecar follow-up printed after an apply
+or rollback is conditioned on that path being a mountpoint: where no view is
+mounted the servers bind the array itself, the sidecars moved with the media,
+and a reap order would point at a directory that is not there.
+
+The journal defaults to `media-relayout.jsonl` BESIDE the library root, never
+inside it: a log inside the library is a file Jellyfin will index.
+"""
+import argparse
+import datetime
+import json
+import os
+import sys
+
+DEFAULT_ROOT = os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies")
+
+
+class Refuse(Exception):
+    """A gate the operator has to see; every refusal leaves the library alone."""
+
+
+def journal_default(root):
+    return os.path.join(os.path.dirname(os.path.abspath(root)), "media-relayout.jsonl")
+
+
+class Journal:
+    """Append-only JSONL of intents. Writes are fsynced before the tool acts."""
+
+    def __init__(self, path, writing):
+        self.path, self.writing = path, writing
+        self.records = []
+        if os.path.exists(path):
+            for line in open(path):
+                line = line.strip()
+                if line:
+                    self.records.append(json.loads(line))
+        self._fh = None
+
+    def already(self, dst, src):
+        """An intent for this exact (src, dst) was written by an earlier run."""
+        return any(r.get("op") == "move" and r.get("dst") == dst
+                   and r.get("src") == src for r in self.records)
+
+    @property
+    def rolled_back(self):
+        return any(r.get("op") == "rollback" for r in self.records)
+
+    def record(self, **fields):
+        self.records.append(fields)
+        if not self.writing:
+            return
+        if self._fh is None:
+            self._fh = open(self.path, "a")
+        self._fh.write(json.dumps(fields) + "\n")
+        self._fh.flush()
+        os.fsync(self._fh.fileno())
+
+    def close(self):
+        if self._fh:
+            self._fh.close()
+
+
+def read_plan(path, root, force):
+    """Plan rows: (mkdirs, keeps, moves), header dropped, all paths under root."""
+    mkdirs, keeps, moves = [], [], []
+    with open(path) as fh:
+        header = fh.readline()
+        if not header.startswith("action\t"):
+            raise Refuse(f"{path} is not a plan (first column is "
+                         f"{header.split(chr(9))[0]!r}, expected 'action') —"
+                         " regenerate it with scripts/media-plan.py")
+        for line in fh:
+            f = line.rstrip("\n").split("\t")
+            if f[0] == "mkdir":
+                mkdirs.append(f[2])
+            elif f[0] == "keep":
+                keeps.append(f[1])
+            elif f[0] == "move":
+                moves.append((f[1], f[2], f[3] if len(f) > 3 else ""))
+    if not force:
+        for p in mkdirs + keeps + [p for m in moves for p in m[:2]]:
+            if p and not (p == root or p.startswith(root + os.sep)):
+                raise Refuse(f"{p} is outside --root {root}"
+                             " (a plan built for another library?)")
+    return mkdirs, keeps, moves
+
+
+# The media servers may read the library through the SSD overlay view
+# (scripts/media-view.sh). A relayout moves bytes on the raw array, so sidecars
+# the servers wrote through that view stay at the old coordinates — naming the
+# follow-up here is the point of use for that fact, but only where the view
+# exists: where it is not mounted the servers bind the array itself, the
+# sidecars moved with the media, and a reap order points at a directory that
+# is not there. See the Media Metadata View page for the host's state.
+METADATA_VIEW = "/var/lib/media-meta/view"
+METADATA_HINT = ("note: sidecars written through the metadata view did not follow"
+                 " these moves — next: scripts/media-view.sh reap")
+
+
+def metadata_note(args):
+    """Print the sidecar follow-up only when the overlay is really mounted."""
+    if os.path.ismount(args.metadata_view):
+        print(METADATA_HINT)
+
+
+def do_apply(args, root, journal, mkdirs, keeps, moves):
+    """Preflight everything, then folders, then renames. Never a partial start."""
+    problems = []
+    for p in keeps:
+        if not os.path.exists(p):
+            problems.append(f"missing: keep row source is gone: {p}")
+    seen = {}
+    for src, dst, _kind in moves:
+        if src == dst:
+            problems.append(f"no-op: move row whose target is its source: {src}")
+        if dst in seen:
+            problems.append(f"double-claim: {dst} claimed by {seen[dst]} and {src}")
+        seen[dst] = src
+        if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
+            continue                                    # this intent already ran
+        if not os.path.exists(src):
+            if os.path.exists(dst):
+                problems.append(f"missing: {src} is gone and {dst} carries no journal"
+                                " line — it moved outside this tool, refusing to guess")
+            else:
+                problems.append(f"missing: {src} is gone and {dst} is not there either")
+            continue
+        if os.path.exists(dst):
+            problems.append(f"occupied: target already holds another file: {dst}")
+    if problems:
+        print(f"refusing: {len(problems)} problem(s), nothing moved")
+        for p in sorted(problems)[:20]:
+            print("  " + p)
+        if len(problems) > 20:
+            print(f"  ... and {len(problems) - 20} more")
+        return 2
+
+    if not args.apply:
+        print(f"dry run: {len(moves)} moves, {len(mkdirs)} folders, {len(keeps)} files stay"
+              " — nothing written (pass --apply to run)")
+        for src, dst, _k in moves[:5]:
+            print(f"  would move {src} -> {dst}")
+        return 0
+
+    made = {r.get("dst") for r in journal.records if r.get("op") == "mkdir"}
+    dirs_new = 0
+    for d in sorted(mkdirs, key=lambda q: q.count(os.sep)):
+        if d in made or os.path.isdir(d):
+            continue
+        journal.record(op="mkdir", dst=d)
+        os.makedirs(d, exist_ok=True)
+        dirs_new += 1
+
+    # same-filesystem gate: rename cannot cross a device, and a silent copy of a
+    # library this size is not an option anyone would choose.
+    cross = []
+    for src, dst, _k in moves:
+        if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
+            continue
+        parent = os.path.dirname(dst)
+        if os.path.exists(parent) and os.stat(parent).st_dev != os.stat(src).st_dev:
+            cross.append(dst)
+    if cross:
+        print(f"refusing: {len(cross)} moves would cross a filesystem"
+              f" (first: {cross[0]}) — this tool only renames")
+        return 2
+
+    moved = done = 0
+    for src, dst, _kind in moves:
+        if journal.already(dst, src):
+            if os.path.exists(dst) and not os.path.exists(src):
+                done += 1
+                continue
+            if os.path.exists(src) and not os.path.exists(dst):
+                os.rename(src, dst)             # the intent ran, the move did not
+                moved += 1
+                continue
+        st = os.stat(src)
+        journal.record(op="move", src=src, dst=dst, bytes=st.st_size, mode=st.st_mode,
+                       mtime=st.st_mtime, ino=st.st_ino, dev=st.st_dev)
+        os.rename(src, dst)
+        moved += 1
+    print(f"applied: {moved} moved, {done} already done in an earlier run,"
+          f" {dirs_new} folders created, {len(keeps)} files stayed put")
+    print(f"journal: {journal.path} — undo with: media-apply.py rollback"
+          f" --journal {journal.path} --root {root} --apply")
+    metadata_note(args)
+    return 0
+
+
+def do_verify(root, journal, moves, keeps):
+    """Every planned file sits at its target carrying the journalled bytes."""
+    journalled = {(r["src"], r["dst"]): r for r in journal.records
+                  if r.get("op") == "move"}
+    todo = moves or [(r["src"], r["dst"], "") for r in journal.records
+                     if r.get("op") == "move"]
+    bad, missing, bytes_total = [], [], 0
+    for src, dst, _kind in todo:
+        rec = journalled.get((src, dst))
+        if not os.path.exists(dst):
+            missing.append(dst)
+            continue
+        size = os.path.getsize(dst)
+        bytes_total += size
+        if rec and rec.get("bytes") not in (None, size):
+            bad.append(f"{dst}: {size} bytes on disk, {rec['bytes']} at move time")
+        if os.path.exists(src) and src != dst:
+            bad.append(f"{src}: still in place as well as at {dst}")
+    missing += [p for p in keeps if not os.path.exists(p)]
+    if bad or missing:
+        print(f"verify FAILED: {len(missing)} absent, {len(bad)} wrong")
+        for line in (missing + bad)[:20]:
+            print("  " + line)
+        return 2
+    empty = [r["dst"] for r in journal.records
+             if r.get("op") == "mkdir" and os.path.isdir(r["dst"])
+             and not os.listdir(r["dst"])]
+    print(f"verified: {len(todo)} files at their targets"
+          f" ({bytes_total / 2**40:.2f} TiB), {len(keeps)} kept")
+    if empty:
+        print(f"note: {len(empty)} folders are empty (left in place — see the trash law)")
+    return 0
+
+
+def sweep_targets(root, journal):
+    """The folders this run emptied, deepest first — never one it did not touch.
+
+    Eligibility is evidence, not appearance. A folder is a target only when the
+    journal shows a file moved out of it or out of something beneath it, it holds
+    nothing but empty directories now, it is not a folder this run created, and no
+    journalled destination lives under it. A symlink anywhere inside disqualifies
+    the whole subtree: this tool never follows a link, and deleting the folder
+    that holds one would delete someone's mount point.
+
+    Proof carries downward: once a folder qualifies, every empty directory inside
+    it qualifies too. A release's junk (``Screens/``, ``Sample/``) is never itself
+    a move source, so without that rule it holds its emptied parent on disk
+    forever — the parent can never be removed while the child stands.
+    """
+    moves = [r for r in journal.records if r.get("op") == "move"]
+    made = {r["dst"] for r in journal.records if r.get("op") == "mkdir"}
+    # An rmdir INTENT that failed left its folder standing, and the next sweep
+    # must be able to finish it: only a folder that is really gone is out of
+    # scope. Counting every intent as done made 146 live survivors permanent.
+    gone = {r["path"] for r in journal.records if r.get("op") == "rmdir"
+            and not os.path.exists(r["path"])}
+    dsts = [r["dst"] for r in moves]
+    marked = set()
+    for r in moves:
+        d = os.path.dirname(r["src"])
+        while d.startswith(root + os.sep):
+            marked.add(d)
+            d = os.path.dirname(d)
+
+    def removable(d):
+        if d in made or d in gone or os.path.islink(d):
+            return False
+        if any(p == d or p.startswith(d + os.sep) for p in dsts):
+            return False
+        try:
+            entries = os.listdir(d)
+        except OSError:
+            return False
+        for e in entries:
+            c = os.path.join(d, e)
+            if os.path.islink(c) or not os.path.isdir(c) or not removable(c):
+                return False
+        return True
+
+    roots = [d for d in marked if removable(d)]
+    targets = set(roots)
+    for d in roots:
+        for dirpath, dirnames, _ in os.walk(d):
+            for c in dirnames:
+                p = os.path.join(dirpath, c)
+                if removable(p):
+                    targets.add(p)
+    return sorted(targets, key=lambda q: (-q.count(os.sep), q))
+
+
+def do_sweep(args, root, journal):
+    """Remove the empty husks an apply left standing at the library root."""
+    moves = [r for r in journal.records if r.get("op") == "move"]
+    if not moves:
+        raise Refuse("the journal holds no moves — a sweep follows an apply, it"
+                     " does not guess at emptiness")
+    outside = sorted({p for r in moves for p in (r["src"], r["dst"])
+                      if not (p == root or p.startswith(root + os.sep))})
+    if outside and not args.force:
+        raise Refuse(f"{len(outside)} journalled paths are outside --root {root}"
+                     f" (first: {outside[0]}) — this journal belongs to another"
+                     " library")
+    targets = sweep_targets(root, journal)
+    if not targets:
+        print("swept: 0 folders — nothing this run emptied is still standing empty")
+        return 0
+    if not args.apply:
+        print(f"dry run: {len(targets)} folders this run emptied would be removed"
+              " (pass --apply to run)")
+        for d in targets[:5]:
+            print(f"  would remove {d}")
+        return 0
+    removed = 0
+    for d in targets:
+        journal.record(op="rmdir", path=d)
+        try:
+            os.rmdir(d)
+        except OSError as e:
+            print(f"left in place: {d} ({e})")
+            continue
+        removed += 1
+    print(f"swept: {removed} folders removed, {len(targets) - removed} left in place")
+    print(f"journal: {journal.path} — rollback recreates them with their files")
+    return 0
+
+
+def do_rollback(args, root, journal):
+    """Undo every journalled move, newest first, after a full preflight."""
+    moves = [r for r in journal.records if r.get("op") == "move"]
+    made = [r["dst"] for r in journal.records if r.get("op") == "mkdir"]
+    swept = [r["path"] for r in journal.records if r.get("op") == "rmdir"]
+    problems = []
+    for r in reversed(moves):
+        src, dst = r["src"], r["dst"]
+        if not args.force and not (src.startswith(root + os.sep)
+                                   or dst.startswith(root + os.sep)):
+            problems.append(f"outside --root: {dst}")
+            continue
+        if os.path.exists(dst) and os.path.exists(src):
+            problems.append(f"occupied: both paths exist, refusing to choose:"
+                            f" {src} | {dst}")
+        elif os.path.exists(dst):
+            pass
+        elif os.path.exists(src):
+            continue                                    # already undone
+        else:
+            problems.append(f"missing: {dst} is gone and {src} never came back")
+    if problems:
+        print(f"refusing: {len(problems)} problem(s), nothing moved back")
+        for p in problems[:20]:
+            print("  " + p)
+        return 2
+    if not args.apply:
+        print(f"dry run: would move {len(moves)} files back to their census paths"
+              f" and recreate {len(set(swept))} swept folders (pass --apply)")
+        return 0
+    undone = 0
+    recreated = 0
+    for d in sorted(set(swept), key=lambda q: q.count(os.sep)):
+        if not os.path.isdir(d):
+            os.makedirs(d, exist_ok=True)
+            recreated += 1
+    for r in reversed(moves):
+        if os.path.exists(r["dst"]) and not os.path.exists(r["src"]):
+            os.makedirs(os.path.dirname(r["src"]), exist_ok=True)
+            os.rename(r["dst"], r["src"])
+            undone += 1
+    dirs_gone = 0
+    for d in sorted(set(made), key=lambda q: -q.count(os.sep)):
+        if d == root or not os.path.isdir(d) or os.listdir(d):
+            continue
+        try:
+            os.rmdir(d)
+            dirs_gone += 1
+        except OSError as e:
+            print(f"left in place: {d} ({e})")
+    journal.record(op="rollback", undone=undone, dirs_removed=dirs_gone,
+                   t=datetime.datetime.now(datetime.timezone.utc).isoformat())
+    print(f"rolled back: {undone} files back where the census found them,"
+          f" {dirs_gone} empty folders removed, {recreated} swept folders recreated")
+    metadata_note(args)
+    return 0
+
+
+def main():
+    ap = argparse.ArgumentParser(description="apply or undo a media relayout plan")
+    ap.add_argument("cmd", choices=["apply", "rollback", "verify", "sweep"])
+    ap.add_argument("--plan")
+    ap.add_argument("--root", default=DEFAULT_ROOT)
+    ap.add_argument("--journal")
+    ap.add_argument("--apply", action="store_true", help="actually change the library")
+    ap.add_argument("--force", action="store_true", help="relax the opinion guards")
+    ap.add_argument("--metadata-view", default=METADATA_VIEW,
+                    help="overlay the media servers read through; the sidecar"
+                         " note prints only when this path is a mountpoint")
+    args = ap.parse_args()
+
+    root = os.path.abspath(args.root)
+    if not os.path.isdir(root):
+        raise Refuse(f"no library at {root}")
+    journal_path = args.journal or journal_default(root)
+    journal = Journal(journal_path, args.apply)
+    try:
+        if args.cmd in ("rollback", "verify", "sweep") and not os.path.exists(journal_path):
+            raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
+        if args.cmd in ("rollback", "sweep") and not args.force and journal.rolled_back:
+            raise Refuse("this journal has already been rolled back"
+                         " (--force to replay it)")
+        if args.cmd in ("apply", "verify") and not args.plan:
+            raise Refuse(f"{args.cmd} needs --plan")
+        mkdirs, keeps, moves = (read_plan(args.plan, root, args.force)
+                                if args.plan else ([], [], []))
+        if args.cmd == "apply":
+            return do_apply(args, root, journal, mkdirs, keeps, moves)
+        if args.cmd == "verify":
+            return do_verify(root, journal, moves, keeps)
+        if args.cmd == "sweep":
+            return do_sweep(args, root, journal)
+        return do_rollback(args, root, journal)
+    finally:
+        journal.close()
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except Refuse as r:
+        print(f"refusing: {r}")
+        sys.exit(2)

scripts/media-audiobooks.py

@@ -0,0 +1,425 @@
+#!/usr/bin/env python3
+r"""media-audiobooks.py — a plan that re-files the audiobook library into one folder per book.
+
+    media-audiobooks.py --root DIR --enum TSV --rows TSV [--out TSV] [--exceptions FILE]
+
+Plex reads this library and Jellyfin does not: its music scanner has no audiobook
+concept, and the tree is indexed entirely by Plex, which matched 58,975 of the
+62,879 files here into its Artist/Album/Track shape. So the authority is
+`scripts/media-census.py audiobooks` — the BOOK (album row), its NARRATOR (that
+book's single artist row) and its CHAPTERS (track rows) — and never the folder a
+release pack made. The plan is:
+
+    Audiobooks/<Narrator>/<Book> (<year>)/<the file's own name>.<ext>
+                     ... the same folder, " (2)" on the name, when two chapters
+                         of one book carry the same name
+
+What that buys is the one property both media servers' book scanners assume: a
+folder holds exactly one book. Today 212 top-level folders are a mix of
+author-named shelves and collection packs, and the books inside the packs are
+split across `… [Part 1]`, `… [Part 2]` folders three or four deep.
+
+Three laws here are the opposite of what a music shelf does, and each is forced
+by what the rows actually hold:
+
+ * A chapter is NEVER renumbered. 7,556 of the 10,485 books contain at least one
+   chapter whose index Plex does not know, so a number written into a name would
+   be invented — and it would buy nothing, because both servers order playback by
+   the embedded track tag, not by the file name. A wrong number in a name is worse
+   than none: it is the number a human later trusts. A file already carrying its
+   own number keeps it, untouched.
+ * The shelf is the NARRATOR, not the author. Every book has exactly one artist row
+   and it is the narrator; the author survives only inside 10% of the album titles
+   (`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving the
+   author would mean parsing titles for one book in ten and inventing it for the
+   rest. One rule that always resolves beats a shelf that is half author and half
+   narrator; the author stays visible in the book folder's own name, which is the
+   title Plex shows.
+ * Nothing is guessed at. A chapter row with no book or no narrator parks — a
+   tag-scan placeholder (`[Unknown Album]`) names neither, since unrelated books
+   share it and filing on one merges them, and so does a chapter number the ripper
+   mistagged as the artist (`0.4`). An audio file Plex never matched parks, a
+   video container parks, and a loose file in a folder holding two books parks.
+
+Nothing is deleted and nothing is overwritten. The output is a list of rename(2)
+intents for `scripts/media-apply.py`, whose preflight refuses any target that is
+already claimed and whose journal is the rollback.
+"""
+import argparse
+import csv
+import os
+import re
+import sys
+from collections import Counter, defaultdict
+
+# What a chapter file can be. .m4b is the audiobook container; the rest are the
+# audio a pack ships when it was ripped as music.
+AUDIO_EXT = {".mp3", ".m4a", ".m4b", ".ogg", ".oga", ".opus", ".wav", ".flac",
+             ".wma", ".aiff", ".aif", ".m4v", ".aac", ".mka"}
+# Never an audiobook, whatever the library it sits in says: Plex reads these as
+# video, and renaming one into an audio extension would claim it is a chapter.
+VIDEO_EXT = {".mkv", ".mp4", ".webm", ".avi", ".mov", ".mpg", ".mpeg", ".ts",
+             ".m2ts", ".vob", ".ifo", ".wmv", ".iso", ".img", ".bdmv", ".ogm",
+             ".divx"}
+
+# Tag-scan placeholders. These strings are not identities: 1,106 chapters from
+# 14 unrelated top-level folders carry the album `[Unknown Album]`, so filing on
+# them would merge books that have nothing to do with each other — the exact
+# shape this relayout exists to make impossible. Such a chapter parks with the
+# path it has today and the fix belongs in its tags. `Various Artists` is NOT a
+# placeholder: that is what a multi-cast read carries, and the one book here
+# with 1,189 chapters under it is genuinely one book.
+PLACEHOLDER_BOOK = {"[unknown album]", "unknown album", "no title", "untitled",
+                    "[untitled]", "unknown title", "n/a", "-"}
+PLACEHOLDER_NARRATOR = {"[unknown artist]", "unknown artist", "unknown author",
+                        "<unknown>", "[unknown]", "unknown"}
+
+
+def identity(value, placeholders, needs_letter=False):
+    """The tag value Plex matched, when it names something; '' when it does not.
+
+    A trailing parenthesized group is stripped for the test only, because rippers
+    glue timestamps to the placeholder (`Unknown album (03/03/2015 21:39:55)`).
+
+    `needs_letter` is for a NARRATOR alone: a shelf is a person's name, and 272
+    chapters here carry a chapter number in the artist tag (`0.4`, `31`). The same
+    test must never judge a BOOK title, which may honestly be `1984`.
+    """
+    v = (value or "").strip()
+    bare = re.sub(r"\s*\([^()]*\)\s*$", "", v).strip().casefold()
+    if bare in placeholders or (needs_letter and not any(c.isalpha() for c in v)):
+        return ""
+    return v
+
+
+# The characters Plex documents as breaking playback. They are dropped, not
+# substituted: a replacement character is a guess at what the title meant.
+BAD_CHARS = set('<>:"/\\|?*')
+TRASH_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir"
+                       r"|lost\+found)(/|$)")
+
+ARGS = None
+HELD = {}                            # path -> (bytes, ext)
+ROWS = {}                            # path -> the chapter row Plex matched
+PLACED = {}                          # chapter path -> the destination the plan gave it
+CLAIMED = set()                      # every destination path the plan has handed out
+MOVES, KEEPS = [], []
+PARKED = defaultdict(list)           # reason -> [path]
+NOTES = defaultdict(list)            # section -> [line]
+AUDIO_BY_DIR = defaultdict(set)      # source dir -> chapter files a row names
+CHILD_OF = defaultdict(set)          # dir -> the dirs directly below it holding chapters
+FOLDERS_SEEN = {}                    # (dir, depth) -> book folders placed below it
+SUBTREE_DEPTH = 2                    # how far below a folder its chapters may sit
+BOOKS = {}                           # (narrator, book, year) -> the book group
+
+AMBIGUOUS = object()
+
+
+def dominant(values):
+    """The spelling the rows mostly agree on: the most frequent value, ties broken
+    by sort so the answer never depends on the order rows arrived in."""
+    counts = Counter(values)
+    return min(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0]
+
+
+def rel(path):
+    """The path as seen from the library root."""
+    return os.path.relpath(path, ARGS.root)
+
+
+def safe(name):
+    """A name every client can carry: no reserved character, nothing unprintable,
+    no trailing dot or space, and short enough to rename."""
+    out = "".join(c for c in (name or "")
+                  if c not in BAD_CHARS and (c.isprintable() or c == " "))
+    out = re.sub(r"\s+", " ", out).strip().rstrip(". ")
+    while out and len(out.encode("utf-8", "ignore")) > 200:
+        out = out[:-4].rstrip()
+    return out
+
+
+def assign(src, folder, name, ext, kind, note):
+    """Book one file into `folder` under `name`, renamed if that exact destination
+    is already taken — by another file in the plan, or by a file standing there
+    today. Two files claiming one path is the fault that costs a re-run of a 60k
+    plan, and so is a plan that only works if the renames happen in the right
+    order: the executor refuses any target that is occupied, so an occupied path
+    is never a destination, even when its occupant is scheduled to leave.
+    A file already standing where the contract puts it is a keep, not a move."""
+    stem = safe(name) or "file"
+    dst = os.path.join(folder, stem + ext)
+    if dst == src:
+        CLAIMED.add(dst.lower())          # it is standing on its own target
+        KEEPS.append((src, dst, kind))
+        return dst
+    # A file already in this folder under this name — or under the numbered
+    # variant the collision rule gives it — keeps the name it has. Renumbering it
+    # every run would turn the plan into a shuffle instead of a fixed point.
+    base = os.path.splitext(os.path.basename(src))[0]
+    if os.path.dirname(src) == folder and (base == stem
+                                           or base.startswith(stem + " (")):
+        CLAIMED.add(src.lower())
+        KEEPS.append((src, src, kind))
+        return src
+    n = 1
+    while dst.lower() in CLAIMED:
+        n += 1
+        dst = os.path.join(folder, f"{stem} ({n}){ext}")
+    if n > 1:
+        NOTES[note].append(rel(dst))
+    CLAIMED.add(dst.lower())
+    MOVES.append((src, dst, kind))
+    return dst
+
+
+def park(path, reason):
+    """Hold a file where it is, under the park folder, keeping its relative path."""
+    PARKED[reason].append(path)
+    dst = os.path.join(ARGS.park_dir, rel(path))
+    if dst != path:
+        CLAIMED.add(dst.lower())
+        MOVES.append((path, dst, "park"))
+
+
+def under_park(path):
+    """True for a file already parked: that pile is for a human to review, and
+    re-parking it every run would only push it deeper."""
+    return path.startswith(ARGS.park_dir + os.sep)
+
+
+# ------------------------------------------------------------------------ books
+def group_books():
+    """Every matched chapter into one group per (narrator, book, year).
+
+    The key is the identity Plex asserts in its own database, not the folder the
+    file sits in: one book arrives as four part folders inside a collection pack,
+    and two copies of one book in one pack are one book to Plex, whose album item
+    IS that narrator, title and year. The year is part of the key so a 1965
+    recording and a 2020 recording that share a title stay two books.
+    """
+    for path, r in sorted(ROWS.items()):
+        if path not in HELD or under_park(path) or TRASH_DIR.search(path):
+            continue
+        d = os.path.dirname(path)
+        AUDIO_BY_DIR[d].add(path)
+        CHILD_OF[os.path.dirname(d)].add(d)   # so a cover above Part 1/Part 2 sees both
+        narrator = identity(r["narrator"], PLACEHOLDER_NARRATOR, needs_letter=True)
+        book = identity(r["book"], PLACEHOLDER_BOOK)
+        if not book:
+            park(path, "a chapter row that names no book — Plex's album title is a "
+                       "placeholder like [Unknown Album], so filing on it would "
+                       "merge unrelated books onto one shelf")
+            continue
+        if not narrator:
+            park(path, "a chapter row that names no narrator — the artist row is a "
+                       "placeholder like [Unknown Artist] or a chapter number a "
+                       "ripper mistagged as the artist, so no shelf owns it")
+            continue
+        key = (narrator.casefold(), book.casefold(), (r["year"] or "").strip())
+        g = BOOKS.setdefault(key, {"narrators": [], "books": [], "year": key[2],
+                                   "members": [], "indexed": 0})
+        g["members"].append(path)
+        g["narrators"].append(narrator)
+        g["books"].append(book)
+        if str(r["track"] or "").isdigit():
+            g["indexed"] += 1
+
+
+def book_folder(g):
+    """The shelf Plex's own rows name: <narrator>/<book> (<year>). None when a
+    name is made of characters no path can hold — inventing a shelf is filing it
+    wrong, so those chapters park instead."""
+    narrator = safe(dominant(g["narrators"]))
+    book = safe(dominant(g["books"]))
+    if not narrator or not book:
+        return None
+    return os.path.join(ARGS.root, narrator,
+                        f"{book} ({g['year']})" if g["year"] else book)
+
+
+def place_books():
+    """One folder per book; every chapter keeps the name it has today."""
+    for key in sorted(BOOKS):
+        g = BOOKS[key]
+        folder = book_folder(g)
+        if folder is None:
+            for path in sorted(g["members"]):
+                park(path, "a book or narrator Plex spells with characters no "
+                           "folder name can hold")
+            continue
+        for path in sorted(g["members"]):
+            ext = HELD[path][1]
+            base = os.path.splitext(os.path.basename(path))[0]
+            # The name the ripper wrote, unchanged: the track tag inside the file
+            # is what orders playback, and 72% of the books here have chapters no
+            # row numbers, so any number added here would be invented.
+            PLACED[path] = assign(path, folder, base, ext, "chapter",
+                                  "two chapters of one book carried the same name")
+
+
+# --------------------------------------------------------------------- sidecars
+def folders_below(d, depth):
+    """The book folders the chapters in `d`, or within `depth` directories below
+    it, were placed in. Memoised, because every loose file in a big collection
+    folder asks the same question about it."""
+    if (d, depth) not in FOLDERS_SEEN:
+        seen = {os.path.dirname(PLACED[a]) for a in AUDIO_BY_DIR.get(d, ())
+                if a in PLACED}
+        if depth:
+            for child in CHILD_OF.get(d, ()):
+                seen |= folders_below(child, depth - 1)
+        FOLDERS_SEEN[(d, depth)] = seen
+    return FOLDERS_SEEN[(d, depth)]
+
+
+def owner_folder(d):
+    """The book folder that owns this directory — the first place up the tree
+    where chapters sit, in the directory itself or within SUBTREE_DEPTH levels
+    below it, which is how a pack whose parts are `… [Part 1]`/`… [Part 2]` owns
+    the cover art lying above them. None when no chapter is near enough to say,
+    AMBIGUOUS when the chapters there belong to more than one book."""
+    while d.startswith(ARGS.root + os.sep) and d != ARGS.root:
+        folders = folders_below(d, SUBTREE_DEPTH)
+        if folders:
+            return next(iter(folders)) if len(folders) == 1 else AMBIGUOUS
+        d = os.path.dirname(d)
+    return None
+
+
+def place_other():
+    """Everything that is not a chapter Plex matched: covers, booklets, EPUBs,
+    logs, checksums, playlists — and the clutter that is none of those."""
+    for path, (_size, ext) in sorted(HELD.items()):
+        if path in ROWS or under_park(path) or TRASH_DIR.search(path):
+            continue
+        d = os.path.dirname(path)
+        base = os.path.splitext(os.path.basename(path))[0]
+        if ext in VIDEO_EXT:
+            park(path, "a video container inside the audiobook library: Plex reads "
+                       "it as video, and renaming it would claim it is a chapter")
+        elif ext in AUDIO_EXT:
+            park(path, "an audio file no chapter row names — it is not in the "
+                       "library Plex matched, so nothing says which book it is")
+        else:
+            owner = owner_folder(d)
+            if owner is AMBIGUOUS:
+                park(path, "a file whose folder holds more than one book, so "
+                           "nothing says which one it belongs to")
+            elif owner:
+                assign(path, owner, base, ext,
+                       "playlist" if ext in (".m3u", ".m3u8") else "sidecar",
+                       "one book folder would have carried two files of the same"
+                       " sidecar name")
+            else:
+                park(path, "not an audiobook file, and no book's folder lies with"
+                           " it to carry it")
+
+
+# ---------------------------------------------------------------------- outputs
+def write_plan():
+    """mkdir rows for every destination folder and its whole chain, then the moves.
+    Parents first: the executor creates directories in the order they are listed."""
+    chain = set()
+    for _s, dst, _k in MOVES:
+        d = os.path.dirname(dst)
+        while d and d != ARGS.root and d != os.path.dirname(d):
+            chain.add(d)
+            d = os.path.dirname(d)
+    dirs = sorted(chain, key=lambda p: (p.count("/"), p))
+    with open(ARGS.out, "w") as fh:
+        fh.write("action\tsrc\tdst\tkind\n")
+        for d in dirs:
+            fh.write(f"mkdir\t\t{d}\tdir\n")
+        for src, dst, kind in sorted(MOVES, key=lambda m: (m[1], m[0])):
+            fh.write(f"move\t{src}\t{dst}\t{kind}\n")
+        for src, dst, kind in sorted(KEEPS, key=lambda m: (m[1], m[0])):
+            fh.write(f"keep\t{src}\t{dst}\t{kind}\n")
+    parked = [m for m in MOVES if m[2] == "park"]
+    sized = sum(HELD[s][0] for s, _d, _k in MOVES if s in HELD)
+    print(f"plan: {len(MOVES) - len(parked)} moves, {len(parked)} parked"
+          f" ({sized / 2**40:.2f} TiB touched), {len(KEEPS)} already placed,"
+          f" {len(dirs)} folders -> {ARGS.out}")
+
+
+def write_exceptions():
+    with open(ARGS.exceptions, "w") as fh:
+        fh.write(f"Audiobook relayout exceptions for {ARGS.root}\n\n")
+        fh.write(f"Everything below is parked under {ARGS.park_dir}, keeping the\n"
+                 "relative path it has today. Nothing is deleted and nothing is\n"
+                 "overwritten: a parked file is one rename from wherever the plan is\n"
+                 "pointed next.\n\n")
+        for reason in sorted(PARKED, key=lambda r: -len(PARKED[r])):
+            rs = PARKED[reason]
+            fh.write(f"{len(rs):>6} files  {reason}\n")
+            for p in sorted(rs, key=rel)[:400]:
+                fh.write(f"          {rel(p)}\n")
+            if len(rs) > 400:
+                fh.write(f"          … and {len(rs) - 400} more\n")
+            fh.write("\n")
+        for section in sorted(NOTES, key=lambda s: -len(NOTES[s])):
+            lines = sorted(set(NOTES[section]))
+            fh.write(f"{len(lines):>6}  {section}\n")
+            for line in lines[:200]:
+                fh.write(f"          {line}\n")
+            if len(lines) > 200:
+                fh.write(f"          … and {len(lines) - 200} more\n")
+            fh.write("\n")
+        unindexed = sorted(f"{safe(dominant(g['narrators']))}/{safe(dominant(g['books']))}"
+                           for g in BOOKS.values()
+                           if 0 < g["indexed"] < len(g["members"]))
+        if unindexed:
+            fh.write(f"{len(unindexed):>6} books have chapters Plex does not number. Their\n")
+            fh.write("       files keep the names they carry: playback order comes from the\n"
+                     "       track tag inside each file, and a number written here would be\n"
+                     "       invented for the chapters with no index. Fix it in the tags, not\n"
+                     "       in the names — Plex re-reads tags on a scan and never names files.\n\n")
+        fh.write("       Playlists move with their book and their contents are never\n"
+                 "       rewritten: the paths inside an .m3u go stale.\n")
+    print(f"exceptions -> {ARGS.exceptions}")
+
+
+def main():
+    global ARGS
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--root", default=os.environ.get("MEDIA_AUDIOBOOKS_ROOT",
+                                                     "/mnt/entropy/Audiobooks"))
+    ap.add_argument("--enum", default="/tmp/audiobooks-enum.tsv",
+                    help="the enumerate TSV (bytes, ext, path)")
+    ap.add_argument("--rows", default="/tmp/plex-audiobooks.tsv",
+                    help="the chapter rows scripts/media-census.py audiobooks wrote")
+    ap.add_argument("--out", default="/tmp/audiobooks-plan.tsv")
+    ap.add_argument("--exceptions", default="/tmp/audiobooks-exceptions.txt")
+    ap.add_argument("--park", default="_unsorted",
+                    help="the folder inside the root that holds what cannot be filed")
+    ARGS = ap.parse_args()
+    ARGS.root = os.path.normpath(ARGS.root)
+    ARGS.park_dir = os.path.join(ARGS.root, ARGS.park)
+    read_inputs()
+    # Every path a file occupies today is claimed before a single destination is
+    # handed out: the executor refuses an occupied target, so a plan that aimed at
+    # one would only fail after the renames that were supposed to clear it.
+    CLAIMED.update(p.lower() for p in HELD)
+    group_books()
+    place_books()
+    place_other()
+    write_plan()
+    write_exceptions()
+
+
+def read_inputs():
+    with open(ARGS.enum) as fh:
+        for line in fh:
+            parts = line.rstrip("\n").split("\t")
+            if len(parts) >= 3:
+                HELD[parts[2]] = (int(parts[0]), parts[1].lower())
+    with open(ARGS.rows, newline="") as fh:
+        for row in csv.DictReader(fh, delimiter="\t"):
+            if row.get("file"):
+                ROWS[row["file"]] = row
+    gone = [p for p in ROWS if p not in HELD]
+    if gone:
+        NOTES["a chapter row whose file the census no longer sees"].extend(gone)
+
+
+if __name__ == "__main__":
+    main()

scripts/media-census.py

@@ -0,0 +1,526 @@
+#!/usr/bin/env python3
+r"""Read-only census of a media library — the inputs a relayout plan is built from.
+
+  media-census.py enumerate [--root DIR] [--out TSV]
+  media-census.py probe     [--enum TSV] [--out TSV] [--jobs N]
+  media-census.py plex      [--root DIR] [--db PATH] [--out TSV]
+  media-census.py jellyfin  [--root DIR] [--db PATH] [--container PREFIX] [--out TSV]
+  media-census.py audiobooks [--root DIR] [--db PATH] [--out TSV]
+
+Run it on the machine that mounts the library. Nothing here writes inside the
+library and nothing is deleted; the only outputs are the TSVs below, and the
+scripts/media-plan.py and scripts/media-shows.py read them instead of touching media.
+
+  enum    bytes <TAB> ext <TAB> path                  every file under the root
+  probed  bytes <TAB> path <TAB> dur <TAB> vcodec <TAB> WxH <TAB> fps <TAB> vbit
+          <TAB> pix <TAB> audio <TAB> subs <TAB> spare <TAB> format
+          one row per video-container file; `audio` is "codec:lang+codec:lang",
+          `subs` the subtitle-stream count, `spare` the audio streams carrying
+          neither a default nor a forced tag
+  plex    file <TAB> leaf_title <TAB> … (see PLEX_COLS)
+          the titles Plex already matched — the authority over any filename guess
+
+  jellyfin file <TAB> series <TAB> year <TAB> season <TAB> episode <TAB> extra
+          <TAB> tmdb <TAB> title <TAB> tvdb      (see JF_COLS)
+          one row per episode Jellyfin has matched and numbered, its path mapped
+          from the container's view back to the host; `year`, `tmdb` and `tvdb`
+          are the SERIES' values, `title` is the episode title the row carries,
+          `extra` is Jellyfin's own extra type when the row is bonus material
+          rather than an episode
+
+  music file <TAB> artist <TAB> album <TAB> year <TAB> disc <TAB> track <TAB> title
+         <TAB> mb <TAB> performer   (see MUSIC_COLS)
+         one row per track Jellyfin has tagged: the tag read is what the row IS,
+         so `artist` is the album artist (the track artist when that is empty) and
+         `album`/`year` are the album item's provider values when the folder is an
+         album item and the tags' otherwise; `mb` is the album's MusicBrainz
+         release group, `performer` the track artist the tags carry
+
+  audiobooks file <TAB> narrator <TAB> book <TAB> year <TAB> track <TAB> chapter
+         (see AB_COLS)
+         one row per chapter Plex matched under the audiobook root: `book` is the
+         album row Plex built from the files' own tags, `narrator` that book's
+         artist row — Plex keeps exactly one artist per book, which is why it, and
+         not an author parsed out of a title, names the shelf. `track` is empty
+         wherever Plex has no chapter number, which is most of this library.
+
+`probe` is the slow step (every container is opened for its header, ~10 min for
+4,800 files on a spindle array at 6 workers) so it is `nice`d: a census beside a
+live service must not starve it. `enumerate` is a stat pass, so re-censusing
+before a plan is cheap — and a plan is only ever as good as these files.
+"""
+import argparse
+import csv
+import json
+import os
+import re
+import sqlite3
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor
+
+# Containers ffprobe can read. .iso/.img are deliberately NOT here: a disc image is
+# a mounted volume, not a stream, and the planner treats it as a movie file instead.
+PROBE_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg",
+             ".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".bdmv", ".vob"}
+# Trash and thumbnail caches are not library content, and this repo's trash policy
+# means .AGENTTRASH can sit inside a movie folder at any time.
+SKIP_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir|lost\+found)(/|$)")
+PLEX_DB = ("/var/lib/plexmediaserver/Library/Application Support/Plex Media Server"
+           "/Plug-in Support/Databases/com.plexapp.plugins.library.db")
+PLEX_COLS = ["file", "leaf_title", "leaf_orig", "leaf_edition", "leaf_year",
+             "parent_title", "parent_orig", "parent_year", "parent_edition",
+             "container", "vcodec", "acodec", "width", "height"]
+PLEX_SQL = """
+select p.file, coalesce(mi.title,''), coalesce(mi.original_title,''),
+       coalesce(mi.edition_title,''), coalesce(mi.year,0),
+       coalesce(par.title,''), coalesce(par.original_title,''),
+       coalesce(par.year,0), coalesce(par.edition_title,''),
+       m.container, m.video_codec, m.audio_codec, m.width, m.height
+from media_parts p
+join media_items m on m.id = p.media_item_id
+join metadata_items mi on mi.id = m.metadata_item_id
+left join metadata_items par on par.id = mi.parent_id
+where p.file like ? and p.deleted_at is null and mi.deleted_at is null
+"""
+JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db"
+JF_COLS = ["file", "series", "year", "season", "episode", "extra",
+           "tmdb", "title", "tvdb"]
+# Jellyfin stores the entity's .NET class as its type; the TV tree is these three.
+JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode",
+         "MediaBrowser.Controller.Entities.TV.Season",
+         "MediaBrowser.Controller.Entities.TV.Series")
+JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType,
+                   IsVirtualItem, Name from BaseItems where Type = ? and Path like ?""")
+
+# The music tree is these two entity classes. The track row carries the tags
+# Jellyfin read out of the file; the album row carries what the provider matched,
+# which is the better folder name of the two.
+JF_AUDIO = "MediaBrowser.Controller.Entities.Audio.Audio"
+JF_ALBUM = "MediaBrowser.Controller.Entities.Audio.MusicAlbum"
+MUSIC_COLS = ["file", "artist", "album", "year", "disc", "track", "title", "mb",
+              "performer"]
+MUSIC_SQL = ("""select b.Path, b.AlbumArtists, b.Artists, b.Album, b.ProductionYear,
+                       b.ParentIndexNumber, b.IndexNumber, b.Name, b.Id,
+                       a.Name, a.ProductionYear, a.Id
+                  from BaseItems b
+                  left join BaseItems a on a.Id = b.ParentId and a.Type = ?
+                 where b.Type = ? and b.Path like ?""")
+# The album's identity, in preference order. A release group is the album; a bare
+# release id is one reissue of it.
+MUSIC_PROVIDERS = ("MusicBrainzReleaseGroup", "MusicBrainzAlbum")
+
+# Plex's audiobook tree is the same Artist/Album/Track shape as its music tree:
+# the leaf (10) is a CHAPTER, its parent (9) is the BOOK, and that book's artist
+# (8) is the NARRATOR. The track's own `index` is exported because the exceptions
+# report counts how many books have chapters Plex does not number — the reason the
+# relayout refuses to invent numbers into file names.
+PLEX_ARTIST, PLEX_ALBUM, PLEX_TRACK = 8, 9, 10
+AB_COLS = ["file", "narrator", "book", "year", "track", "chapter"]
+AB_SQL = f"""
+select p.file, coalesce(ar.title,''), coalesce(al.title,''), coalesce(al.year,0),
+       coalesce(mi."index",-1), coalesce(mi.title,'')
+from media_parts p
+join media_items m on m.id = p.media_item_id
+join metadata_items mi on mi.id = m.metadata_item_id
+join metadata_items al on al.id = mi.parent_id and al.metadata_type = {PLEX_ALBUM}
+left join metadata_items ar on ar.id = al.parent_id
+                          and ar.metadata_type = {PLEX_ARTIST}
+where p.file like ? and p.deleted_at is null and mi.deleted_at is null
+  and al.deleted_at is null
+"""
+
+def open_plex_ro(db):
+    """Plex's library, read the way Plex writes it.
+
+    `immutable=1` tells sqlite the file never changes, so it skips the
+    write-ahead log: every row a scan has committed but not yet checkpointed
+    is invisible, and reading a live database that way can fail outright as
+    "database disk image is malformed". Plain ro reads the WAL through its
+    -shm; immutable is only the fallback for a database whose -shm cannot be
+    read. The same law scripts/media-verify.py states.
+    """
+    last = None
+    for opt in ("mode=ro", "mode=ro&immutable=1"):
+        try:
+            return sqlite3.connect(f"file:{db}?{opt}", uri=True)
+        except sqlite3.Error as e:
+            last = e
+    sys.exit(f"cannot open the Plex database read-only ({db}): {last}\n"
+             "Plex runs rootless here: run this as the container's owner, never "
+             "with sudo — rootless storage is per-user, so root sees nothing.")
+
+
+def enumerate_lib(root, out):
+    """One row per real file under root: size, extension, path."""
+    rows = []
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames
+                       if not SKIP_DIR.search(os.path.join(dirpath, d))]
+        for fn in filenames:
+            p = os.path.join(dirpath, fn)
+            if SKIP_DIR.search(p):
+                continue
+            try:
+                st = os.lstat(p)
+            except OSError:
+                continue
+            if not os.path.isfile(p) or os.path.islink(p):
+                continue                # links and sockets are not library content
+            rows.append((st.st_size, os.path.splitext(fn)[1].lower(), p))
+    with open(out, "w") as fh:
+        for size, ext, p in rows:
+            fh.write(f"{size}\t{ext}\t{p}\n")
+    vid = [r for r in rows if r[1] in PROBE_EXT]
+    print(f"files={len(rows)} bytes={sum(r[0] for r in rows)} "
+          f"probes={len(vid)} probe_bytes={sum(r[0] for r in vid)} -> {out}")
+
+
+def probe_one(path):
+    """ffprobe header read -> the probed TSV row (never the whole file)."""
+    try:
+        os.nice(19)                     # a census must never outrank the media server
+    except OSError:
+        pass
+    try:
+        size = os.stat(path).st_size
+    except OSError:
+        return None
+    cmd = ["ffprobe", "-v", "error", "-print_format", "json",
+           "-show_format", "-show_streams", path]
+    try:
+        done = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
+        data = json.loads(done.stdout or "{}")
+    except (subprocess.TimeoutExpired, subprocess.SubprocessError,
+            json.JSONDecodeError, OSError):
+        data = {}
+    st = data.get("streams", [])
+    fmt = data.get("format", {})
+    v = next((s for s in st if s.get("codec_type") == "video"), None)
+    aud = [s for s in st if s.get("codec_type") == "audio"]
+    subs = [s for s in st if s.get("codec_type") == "subtitle"]
+    fps = "-"
+    if v and v.get("r_frame_rate"):
+        try:
+            n, d = (int(x) for x in v["r_frame_rate"].split("/", 1))
+            fps = f"{round(n / d * 100) / 100:g}" if d else "0"
+        except ValueError:
+            pass
+
+    def num(x):
+        try:
+            return round(float(x))
+        except (TypeError, ValueError):
+            return 0
+
+    return [size, path, num(fmt.get("duration")),
+            (v or {}).get("codec_name", "-"),
+            f"{v['width']}x{v['height']}" if v else "-",
+            fps, num(fmt.get("bit_rate")), (v or {}).get("pix_fmt", "-"),
+            "+".join(f"{a.get('codec_name', '?')}:{a.get('language', '?')}" for a in aud),
+            len(subs),
+            sum(1 for a in aud if not re.search(
+                r"default|forced", json.dumps(a.get("tags", {})), re.I)),
+            fmt.get("format_name", "-") or "-"]
+
+
+def probe(enum, out, jobs):
+    """Probe every video-container row of the enum TSV, in parallel."""
+    paths = []
+    with open(enum) as fh:
+        for line in fh:
+            f = line.rstrip("\n").split("\t", 2)
+            if len(f) == 3 and f[1] in PROBE_EXT:
+                paths.append(f[2])
+    print(f"probing {len(paths)} files with {jobs} workers", flush=True)
+    done = unreadable = 0
+    with open(out, "w") as fh, ThreadPoolExecutor(max_workers=jobs) as pool:
+        for row in pool.map(probe_one, paths):
+            if row is None:
+                continue
+            if row[3] == "-":
+                unreadable += 1         # header unreadable: the row still gets written
+            done += 1
+            fh.write("\t".join(str(x) for x in row) + "\n")
+            if done % 250 == 0:
+                print(f"  {done}/{len(paths)}", flush=True)
+    print(f"probed={done} unreadable={unreadable} -> {out}")
+
+
+def plex(root, db, out):
+    """Plex's matched titles for files under root, from its DB, read-only."""
+    con = open_plex_ro(db)
+    with open(out, "w", newline="") as fh:
+        w = csv.writer(fh, delimiter="\t", lineterminator="\n")
+        w.writerow(PLEX_COLS)
+        n = 0
+        for r in con.execute(PLEX_SQL, (root + "/%",)):
+            w.writerow([str(x).replace("\t", " ").replace("\n", " ") for x in r])
+            n += 1
+    print(f"plex-matched={n} -> {out}")
+
+
+def jellyfin(root, db, container, out):
+    """The episodes Jellyfin has already numbered, for the library at root.
+
+    Jellyfin stores each path as the CONTAINER sees it (its quadlet binds the
+    share at /data), so `container` is the in-container prefix that stands for
+    root and every exported path is mapped back to the host before it is written
+    — the planner reads host paths, because it plans moves on the array.
+
+    The numbering is what matters, not the name: a release group's file name says
+    nothing about which episode a file is, and Jellyfin has already answered that
+    question against a provider. Season/episode and the episode's own title come
+    from the episode row; the series name, its year and its TMDb and Tvdb ids come
+    from the Series row the episode climbs to, because those are the only place
+    Jellyfin keeps them.
+    """
+    try:
+        con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
+    except sqlite3.Error as e:
+        sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n"
+                 "It is a WAL database: copy it with its -wal and -shm beside it and"
+                 " point --db at the copy. Never sudo it — the data dir belongs to"
+                 " the container's owner, and rootless storage is per-user.")
+    have = {r[1] for r in con.execute("pragma table_info(BaseItems)")}
+    need = {"Id", "ParentId", "Type", "Path", "IndexNumber", "ParentIndexNumber",
+            "ExtraType", "IsVirtualItem", "ProductionYear", "Data"}
+    if not need <= have:
+        sys.exit(f"{db} is not a Jellyfin library database (BaseItems has no"
+                 f" {','.join(sorted(need - have))})")
+    node = {}
+    for i, pid, typ, name, year, data in con.execute(
+            "select Id, ParentId, Type, Name, ProductionYear, Data from BaseItems"
+            " where Type in (?,?,?)", JF_TV):
+        node[i] = (pid, typ, name, year, data)
+
+    # Jellyfin 12 keeps provider ids in their own table; older builds kept them in
+    # the row's Data blob. Both shapes are read, because the tvdb id the series
+    # naming depends on is wherever this build happened to write it.
+    prov = {}
+    if "BaseItemProviders" in {r[0] for r in con.execute(
+            "select name from sqlite_master where type='table'")}:
+        for iid, pid, val in con.execute(
+                "select ItemId, ProviderId, ProviderValue from BaseItemProviders"
+                " where ProviderId in ('Tmdb', 'Tvdb')"):
+            prov[(iid, pid)] = val
+
+    def provider_of(item_id, want):
+        got = prov.get((item_id, want))
+        if got:
+            return str(got)
+        try:
+            blob = json.loads(node[item_id][4] or "{}").get("ProviderIds") or {}
+        except (KeyError, ValueError, TypeError):
+            return ""
+        return str(blob.get(want, "") or "")
+
+    def series_of(item_id):
+        """Climb episode -> season -> series; empty when the chain is broken."""
+        for _ in range(4):
+            if item_id not in node:
+                return "", "", "", ""
+            pid, typ = node[item_id][0], node[item_id][1]
+            if typ.endswith(".Series"):
+                return (node[item_id][2] or "", node[item_id][3] or "",
+                        provider_of(item_id, "Tmdb"), provider_of(item_id, "Tvdb"))
+            item_id = pid
+        return "", "", "", ""
+
+    # The fleet's quadlets bind the share at /data, so the container's view of
+    # this library is /data/<the library's own folder name> unless told otherwise.
+    if not container:
+        container = "/data/" + os.path.basename(os.path.normpath(root))
+    prefix = container.rstrip("/") + "/"
+    written = skipped = 0
+    with open(out, "w", newline="") as fh:
+        w = csv.writer(fh, delimiter="\t", lineterminator="\n")
+        w.writerow(JF_COLS)
+        for iid, path, idx, pidx, extra, virt, epname in con.execute(
+                JF_SQL, (JF_TV[0], prefix + "%")):
+            if virt or not path or not path.startswith(prefix):
+                skipped += 1          # a virtual item is a variant, not a file
+                continue
+            sname, syear, tmdb, tvdb = series_of(iid)
+            w.writerow([root + "/" + path[len(prefix):],
+                        sname.replace("\t", " "), syear,
+                        "" if pidx is None else pidx,
+                        "" if idx is None else idx,
+                        (extra or "").replace("\t", " "), tmdb,
+                        (epname or "").replace("\t", " "), tvdb])
+            written += 1
+    print(f"episodes={written} virtual-or-blank={skipped} -> {out}")
+
+
+def music(root, db, container, out):
+    """The tracks Jellyfin has already tagged, for the library at root.
+
+    Jellyfin reads music metadata out of the file itself, so an Audio row IS the
+    tag read, done once, by the vendor's own parser: its album-artist, album,
+    year, disc and track columns are what the embedded tags say and its Name is
+    the track title. That makes this export the whole authority for a music
+    relayout — the path a ripper chose is not evidence of anything, and the
+    folder Jellyfin put a track under is often a release pack, not an album.
+
+    Two gaps this export answers from the row rather than the tree. Many tracks
+    sit under a folder that is not an album item at all (a disc folder, or a
+    mixed one), so the album name and year fall back to the tags when the album
+    row is absent. And an album's MusicBrainz ids are not in the row's own Data
+    — the current schema leaves that empty for music — they are rows of
+    BaseItemProviders keyed by the item's Id.
+    """
+    try:
+        con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
+    except sqlite3.Error as e:
+        sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n"
+                 "It is a WAL database: copy it with its -wal and -shm beside it and"
+                 " point --db at the copy. Never sudo it — the data dir belongs to"
+                 " the container's owner, and rootless storage is per-user.")
+    have = {r[1] for r in con.execute("pragma table_info(BaseItems)")}
+    need = {"Id", "ParentId", "Type", "Path", "Name", "IndexNumber",
+            "ParentIndexNumber", "ProductionYear", "Album", "AlbumArtists",
+            "Artists"}
+    if not need <= have:
+        sys.exit(f"{db} is not a Jellyfin library database with music columns"
+                 f" (BaseItems has no {','.join(sorted(need - have))})")
+    # Provider ids, best identity first: the release group is the album, a bare
+    # release id is one reissue of it and never overrides a group.
+    mb = {}
+    if "BaseItemProviders" in {r[0] for r in con.execute(
+            "select name from sqlite_master where type='table'")}:
+        rank = {p: i for i, p in enumerate(MUSIC_PROVIDERS)}
+        best = {}
+        for iid, pid, val in con.execute(
+                "select ItemId, ProviderId, ProviderValue from BaseItemProviders"
+                " where ProviderId in (?, ?)", MUSIC_PROVIDERS):
+            if iid not in best or rank[pid] < best[iid][0]:
+                best[iid] = (rank[pid], val)
+        mb = {k: v[1] for k, v in best.items()}
+
+    # The fleet's quadlets bind the share at /data, so the container's view of
+    # this library is /data/<the library's own folder name> unless told otherwise.
+    if not container:
+        container = "/data/" + os.path.basename(os.path.normpath(root))
+    prefix = container.rstrip("/") + "/"
+    written = skipped = 0
+    with open(out, "w", newline="") as fh:
+        w = csv.writer(fh, delimiter="\t", lineterminator="\n")
+        w.writerow(MUSIC_COLS)
+        for (path, aal, arts, tag_album, tag_year, disc, track, name, iid,
+             alb_name, alb_year, alb_id) in con.execute(
+                MUSIC_SQL, (JF_ALBUM, JF_AUDIO, prefix + "%")):
+            if not path or not path.startswith(prefix):
+                skipped += 1          # a row for a file outside this library
+                continue
+            # The album artist owns the folder; the track artist is the fallback
+            # for the compilations whose album-artist tag was left empty.
+            w.writerow([root + "/" + path[len(prefix):],
+                        ((aal or "").strip() or (arts or "").strip()),
+                        (alb_name or tag_album or "").replace("\t", " "),
+                        alb_year or tag_year or "",
+                        "" if disc is None else disc,
+                        "" if track is None else track,
+                        (name or "").replace("\t", " "),
+                        mb.get(alb_id or "") or mb.get(iid or "") or "",
+                        (arts or "").replace("\t", " ")])
+            written += 1
+    print(f"tracks={written} outside-root-or-blank={skipped} -> {out}")
+
+
+def audiobooks(root, db, out):
+    """The chapters Plex matched inside the audiobook library, read-only.
+
+    Plex stores HOST paths here (unlike Jellyfin, which stores the container's
+    view), so no prefix mapping is needed: the part's `file` is the path the
+    operator has today. Rows for files outside `root` are skipped, because one
+    Plex library DB carries every library on the box.
+
+    The chapter's `index` is exported as an empty field when Plex has none. That
+    is not a gap to fill in here: three quarters of the books on this box contain
+    at least one un-indexed chapter, so the relayout that reads this file must
+    not renumber anything from it.
+    """
+    con = open_plex_ro(db)
+    tables = {r[0] for r in con.execute(
+        "select name from sqlite_master where type='table'")}
+    missing = {"media_parts", "media_items", "metadata_items"} - tables
+    if missing:
+        sys.exit(f"{db} is not a Plex library database (no "
+                 f"{', '.join(sorted(missing))})")
+    written = skipped = 0
+    with open(out, "w", newline="") as fh:
+        w = csv.writer(fh, delimiter="\t", lineterminator="\n")
+        w.writerow(AB_COLS)
+        for file, narrator, book, year, track, chapter in con.execute(
+                AB_SQL, (root.rstrip("/") + "/%",)):
+            if not file or not file.startswith(root.rstrip("/") + "/"):
+                skipped += 1              # a row for a file outside this library
+                continue
+            w.writerow([file.replace("\t", " "), narrator.replace("\t", " "),
+                        book.replace("\t", " "), "" if not year else year,
+                        "" if track is None or int(track) < 0 else track,
+                        chapter.replace("\t", " ")])
+            written += 1
+    print(f"chapters={written} outside-root={skipped} -> {out}")
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    root = {"default": os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies"),
+            "help": "library root"}
+    e = sub.add_parser("enumerate", help="stat every file under the root")
+    e.add_argument("--root", **root)
+    e.add_argument("--out", default="/tmp/media-enum.tsv")
+    p = sub.add_parser("probe", help="ffprobe every video container")
+    p.add_argument("--enum", default="/tmp/media-enum.tsv")
+    p.add_argument("--out", default="/tmp/media-probed.tsv")
+    p.add_argument("--jobs", type=int, default=6)
+    x = sub.add_parser("plex", help="export Plex's matched titles for the root")
+    x.add_argument("--root", **root)
+    x.add_argument("--db", default=PLEX_DB)
+    x.add_argument("--out", default="/tmp/plex-movies.tsv")
+    j = sub.add_parser("jellyfin",
+                       help="export the episodes Jellyfin has numbered for the root")
+    j.add_argument("--root", default=os.environ.get("MEDIA_ROOT",
+                                                    "/mnt/entropy/TVShows"),
+                   help="TV library root on the host")
+    j.add_argument("--db", default=JF_DB)
+    j.add_argument("--container", default=None,
+                   help="the in-container prefix that stands for --root"
+                        " (default /data/<root's folder name>)")
+    j.add_argument("--out", default="/tmp/jellyfin-tv.tsv")
+    m = sub.add_parser("music",
+                       help="export the tracks Jellyfin has tagged for the root")
+    m.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT",
+                                                    "/mnt/entropy/Music"),
+                   help="music library root on the host")
+    m.add_argument("--db", default=JF_DB)
+    m.add_argument("--container", default=None,
+                   help="the in-container prefix that stands for --root"
+                        " (default /data/<root's folder name>)")
+    m.add_argument("--out", default="/tmp/jellyfin-music.tsv")
+    b = sub.add_parser("audiobooks",
+                       help="export the chapters Plex matched for the audiobook root")
+    b.add_argument("--root", default=os.environ.get("MEDIA_AUDIOBOOKS_ROOT",
+                                                    "/mnt/entropy/Audiobooks"),
+                   help="audiobook library root on the host")
+    b.add_argument("--db", default=PLEX_DB)
+    b.add_argument("--out", default="/tmp/plex-audiobooks.tsv")
+    a = ap.parse_args()
+    if a.cmd == "enumerate":
+        enumerate_lib(a.root, a.out)
+    elif a.cmd == "probe":
+        probe(a.enum, a.out, a.jobs)
+    elif a.cmd == "plex":
+        plex(a.root, a.db, a.out)
+    elif a.cmd == "music":
+        music(a.root, a.db, a.container, a.out)
+    elif a.cmd == "audiobooks":
+        audiobooks(a.root, a.db, a.out)
+    else:
+        jellyfin(a.root, a.db, a.container, a.out)
+
+
+if __name__ == "__main__":
+    main()

scripts/media-music.py

@@ -0,0 +1,445 @@
+#!/usr/bin/env python3
+r"""media-music.py — a plan that re-files the music library into Jellyfin's own shape.
+
+    media-music.py --root DIR --enum TSV --rows TSV [--out TSV] [--exceptions FILE]
+
+Jellyfin's music contract (https://jellyfin.org/docs/general/server/media/music/)
+is not the movie one. Jellyfin reads the metadata out of the file, so a folder is
+an ALBUM and nothing else, every disc of an album belongs in that one folder, the
+tags decide the track order, and a file name only matters when it carries a
+character known to break playback (`< > : " / \ | ? *`) or when a lyric file has
+to match the basename of the track it belongs to. The plan is therefore:
+
+    Music/<album artist>/<Album> (<year>)/NN - <Title>.<ext>
+                       ... the same folder, D.NN - <Title>  when the album has discs
+                       Music/Various Artists/...            when its tracks name
+                                                            more than one artist
+
+The authority is the tag row Jellyfin already read — `scripts/media-census.py
+music` — never the release-pack folder a ripper made. A file with no row is not
+guessed at: it parks under `<root>/_unsorted` with the relative path it has today.
+
+The rules below exist because of what a downloaded discography pack actually
+contains:
+
+ * A cuesheet beside exactly one audio file means the album is one image with a
+   track list inside it. That file keeps its own name: numbering it `01` would
+   claim it is one track, and Jellyfin shows one track either way.
+ * Sidecars — images, cuesheets, EAC logs, checksums, booklets, playlists —
+   follow the album the audio in their own folder belongs to. A folder holding
+   two albums gives a sidecar no single owner, so it parks instead of being
+   guessed into one.
+ * A video container (.mkv, .vob, .ifo, …) inside a music library is not music.
+   Renaming it to .mka/.m4a would tell Jellyfin to index video as audio, so it
+   parks for a human decision.
+ * Playlists move with their album and their CONTENTS are never rewritten: the
+   paths inside an .m3u go stale. Editing a file the operator owns to keep a
+   convenience file tidy is not this tool's job.
+
+Nothing is deleted and nothing is overwritten. The output is a list of rename(2)
+intents for `scripts/media-apply.py`, whose preflight refuses any target that is
+already claimed and whose journal is the rollback.
+"""
+import argparse
+import csv
+import os
+import re
+import sys
+from collections import Counter, defaultdict
+
+# What Jellyfin will index as music. .mp4/.mkv/.webm are deliberately NOT here:
+# the vendor doc says an audio-only stream in those containers is never read as
+# music, and this tool renames for a reason it can prove, not to guess at one.
+AUDIO_EXT = {".flac", ".mp3", ".m4a", ".m4b", ".ogg", ".oga", ".opus", ".wav",
+             ".wma", ".aiff", ".aif", ".ape", ".dsf", ".dff", ".wv", ".mka",
+             ".mpc", ".aac"}
+VIDEO_EXT = {".mkv", ".vob", ".ifo", ".bup", ".mpg", ".mpeg", ".avi", ".mov",
+             ".ts", ".m2ts", ".wmv", ".webm", ".mp4", ".m4v", ".iso", ".img",
+             ".bdmv", ".divx", ".ogm"}
+# A lyric has to carry the basename of its track, so it is named after the audio
+# file and not after itself. .txt counts only when its stem matches an audio stem:
+# a loose .txt in an album folder is a note.
+LYRIC_EXT = {".lrc", ".elrc", ".txt"}
+IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tiff"}
+# The album-artist strings that mean "compilation", so such an album is not filed
+# under one artist who does not own it.
+VARIOUS = {"various", "various artists", "unknown", "unknown artist", "unknow"}
+# The characters Jellyfin documents as breaking playback. They are dropped, not
+# substituted: a replacement character is a guess at what the title meant.
+BAD_CHARS = set('<>:"/\\|?*')
+TRASH_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir"
+                       r"|lost\+found)(/|$)")
+
+ARGS = None
+HELD = {}                            # path -> (bytes, ext)
+ROWS = {}                            # path -> the tag row
+PLACED = {}                          # audio path -> the destination the plan gave it
+CLAIMED = set()                      # every destination path the plan has handed out
+MOVES, KEEPS = [], []
+PARKED = defaultdict(list)           # reason -> [path]
+NOTES = defaultdict(list)            # section -> [line]
+CUE_DIRS = set()                     # source dirs holding a .cue
+AUDIO_BY_DIR = defaultdict(set)      # source dir -> audio files a row names
+CHILD_OF = defaultdict(set)          # dir -> the dirs directly below it holding audio
+FOLDERS_SEEN = {}                    # (dir, depth) -> album folders placed below it
+SUBTREE_DEPTH = 2                    # how far below a folder its audio may sit
+ALBUMS = {}                          # (artist, album, year) -> the album group
+
+AMBIGUOUS = object()
+
+
+def dominant(values):
+    """The spelling the tags mostly agree on: the most frequent value, ties broken
+    by sort so the answer never depends on the order rows arrived in."""
+    counts = Counter(values)
+    return min(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0]
+
+
+def rel(path):
+    """The path as seen from the library root."""
+    return os.path.relpath(path, ARGS.root)
+
+
+def safe(name):
+    """A name every client can carry: no reserved character, nothing
+    unprintable, no trailing dot or space, and short enough to rename."""
+    out = "".join(c for c in (name or "")
+                  if c not in BAD_CHARS and (c.isprintable() or c == " "))
+    out = re.sub(r"\s+", " ", out).strip().rstrip(". ")
+    while out and len(out.encode("utf-8", "ignore")) > 200:
+        out = out[:-4].rstrip()
+    return out
+
+
+def assign(src, folder, name, ext, kind, note):
+    """Book one file into `folder` under `name`, renamed if that exact destination
+    is already taken — by another file in the plan, or by a file standing there
+    today. Two files claiming one path is the fault that costs a re-run of a 90k
+    plan, and so is a plan that only works if the renames happen in the right
+    order: the executor refuses any target that is occupied, so an occupied path
+    is never a destination, even when its occupant is scheduled to leave.
+    A file already standing where the contract puts it is a keep, not a move."""
+    stem = safe(name) or "file"
+    dst = os.path.join(folder, stem + ext)
+    if dst == src:
+        CLAIMED.add(dst.lower())          # it is standing on its own target
+        KEEPS.append((src, dst, kind))
+        return dst
+    # A file already in this folder under this name — or under the numbered
+    # variant the collision rule gives it — keeps the name it has. Renumbering it
+    # every run would turn the plan into a shuffle instead of a fixed point.
+    base = os.path.splitext(os.path.basename(src))[0]
+    if os.path.dirname(src) == folder and (base == stem
+                                           or base.startswith(stem + " (")):
+        CLAIMED.add(src.lower())
+        KEEPS.append((src, src, kind))
+        return src
+    n = 1
+    while dst.lower() in CLAIMED:
+        n += 1
+        dst = os.path.join(folder, f"{stem} ({n}){ext}")
+    if n > 1:
+        NOTES[note].append(rel(dst))
+    CLAIMED.add(dst.lower())
+    MOVES.append((src, dst, kind))
+    return dst
+
+
+def park(path, reason):
+    """Hold a file where it is, under the park folder, keeping its relative path."""
+    PARKED[reason].append(path)
+    dst = os.path.join(ARGS.park_dir, rel(path))
+    if dst != path:
+        CLAIMED.add(dst.lower())
+        MOVES.append((path, dst, "park"))
+
+
+def under_park(path):
+    """True for a file already parked: that pile is for a human to review, and
+    re-parking it every run would only push it deeper."""
+    return path.startswith(ARGS.park_dir + os.sep)
+
+
+# ---------------------------------------------------------------------- albums
+def group_albums():
+    """Every tagged track into one group per (album-artist tag, album, year).
+
+    The key is the identity the tags assert, not the folder the file sits in: one
+    album arrives as three discs in three folders, and two copies of an album in
+    one pack are one album to Jellyfin, whose album item is that name, artist and
+    year. The artist in the key is the ALBUM-artist tag and never the performer,
+    because a compilation is one album whose tracks name many performers — keying
+    on the performer would file one album once per artist.
+    """
+    for path, r in sorted(ROWS.items()):
+        if path not in HELD or under_park(path):
+            continue
+        d = os.path.dirname(path)
+        AUDIO_BY_DIR[d].add(path)
+        CHILD_OF[os.path.dirname(d)].add(d)   # so a cover above CD1/CD2 sees both
+        tag = (r["artist"] or "").strip()
+        album = (r["album"] or "").strip()
+        performer = (r["performer"] or "").strip()
+        if not album:
+            park(path, "a track no album names")
+            continue
+        if not tag and not performer:
+            park(path, "a track no artist names")
+            continue
+        key = (tag.casefold(), album.casefold(), (r["year"] or "").strip())
+        g = ALBUMS.setdefault(key, {"tags": [], "albums": [], "year": key[2],
+                                    "members": [], "performers": set(),
+                                    "mb": set()})
+        g["members"].append(path)
+        g["tags"].append(tag)
+        g["albums"].append(album)
+        g["performers"].add(performer or tag)
+        if r.get("mb"):
+            g["mb"].add(r["mb"])
+
+
+def folder_artist(g):
+    """Whose shelf the album sits on. The album-artist tag owns it when it names
+    one; when the tags name only performers, a single performer names the album
+    and several make it a compilation, which belongs to no one of them. The key is
+    case-folded, so the spelling that reaches the folder is the one the tracks
+    mostly carry — see dominant()."""
+    tags = [t for t in g["tags"] if t]
+    if tags:
+        name = dominant(tags)
+        return "Various Artists" if name.casefold() in VARIOUS else name
+    if len({p.casefold() for p in g["performers"]}) == 1:
+        return dominant(g["performers"])
+    return "Various Artists"
+
+
+def album_folder(g):
+    name = safe(dominant(g["albums"])) or "Untitled"
+    return os.path.join(ARGS.root, safe(folder_artist(g)) or "Unknown Artist",
+                        f"{name} ({g['year']})" if g["year"] else name)
+
+
+def is_cue_image(path):
+    """A folder holding this one audio file and a cuesheet is an album image."""
+    d = os.path.dirname(path)
+    return len(AUDIO_BY_DIR.get(d, ())) == 1 and d in CUE_DIRS
+
+
+def number_of(value, width):
+    return f"{int(value):0{width}d}" if str(value or "").isdigit() else ""
+
+
+def place_albums():
+    for key in sorted(ALBUMS):
+        g = ALBUMS[key]
+        folder = album_folder(g)
+        discs = {g0["disc"] for g0 in (ROWS[p] for p in g["members"]) if g0["disc"]}
+        tracks = [int(ROWS[p]["track"]) for p in g["members"]
+                  if str(ROWS[p]["track"] or "").isdigit()]
+        # More than one disc, or a disc numbered past the first: the track number
+        # alone cannot name a file uniquely, so the disc goes in front of it.
+        multi = len(discs) > 1 or any(str(d).isdigit() and int(d) > 1 for d in discs)
+        width = 3 if multi or (max(tracks, default=0) > 99) else 2
+        for path in sorted(g["members"]):
+            r = ROWS[path]
+            ext = HELD[path][1]
+            base = os.path.splitext(os.path.basename(path))[0]
+            if is_cue_image(path):
+                name = safe(base) or "image"
+            else:
+                title = safe(r["title"] or base) or "untitled"
+                num = number_of(r["track"], 2 if multi else width)
+                if multi and num:
+                    num = f"{r['disc'] or '1'}.{num}"
+                name = f"{num} - {title}" if num else title
+            PLACED[path] = assign(path, folder, name, ext, "track",
+                                  "two tracks claimed one name inside one album")
+
+
+# --------------------------------------------------------------------- sidecars
+def folders_below(d, depth):
+    """The album folders the audio in `d`, or within `depth` directories below it,
+    was placed in. Memoised, because every loose file in a big discography folder
+    asks the same question about it."""
+    if (d, depth) not in FOLDERS_SEEN:
+        seen = {os.path.dirname(PLACED[a]) for a in AUDIO_BY_DIR.get(d, ())
+                if a in PLACED}
+        if depth:
+            for child in CHILD_OF.get(d, ()):
+                seen |= folders_below(child, depth - 1)
+        FOLDERS_SEEN[(d, depth)] = seen
+    return FOLDERS_SEEN[(d, depth)]
+
+
+def owner_folder(d):
+    """The album folder that owns this directory — the first place up the tree
+    where audio sits, in the directory itself or within SUBTREE_DEPTH levels below
+    it, which is how a box with its discs in CD1/CD2 owns the cover lying above
+    them. None when no audio is near enough to say, AMBIGUOUS when the audio there
+    belongs to more than one album."""
+    while d.startswith(ARGS.root + os.sep) and d != ARGS.root:
+        folders = folders_below(d, SUBTREE_DEPTH)
+        if folders:
+            return next(iter(folders)) if len(folders) == 1 else AMBIGUOUS
+        d = os.path.dirname(d)
+    return None
+
+
+def place_sidecars():
+    """Everything that is not a tagged track: lyrics, images, cuesheets, logs,
+    booklets, playlists — and the clutter that is none of those."""
+    # The lyric rule needs the audio's stem as it is named TODAY, before the plan
+    # renames it, because that is what the sidecar's own name was copied from.
+    stem = {(d, os.path.splitext(os.path.basename(a))[0]): a
+            for d, audios in AUDIO_BY_DIR.items() for a in audios}
+    for path, (_size, ext) in sorted(HELD.items()):
+        if path in ROWS or under_park(path) or TRASH_DIR.search(path):
+            continue
+        d, base = os.path.dirname(path), os.path.splitext(os.path.basename(path))[0]
+        if ext in VIDEO_EXT:
+            park(path, "a video container inside the music library: Jellyfin will "
+                       "not read it as music, and renaming it would claim it is")
+        elif ext in AUDIO_EXT:
+            park(path, "an audio file no provider row names — its tags are not in "
+                       "the library Jellyfin has matched")
+        elif ext in (LYRIC_EXT | IMAGE_EXT) and (d, base) in stem:
+            # A lyric, or a picture named after its track: it follows that track
+            # and takes the name the track now has, or Jellyfin cannot find it.
+            audio = stem[(d, base)]
+            dst = PLACED.get(audio)
+            if dst:
+                folder, name = os.path.dirname(dst), os.path.splitext(
+                    os.path.basename(dst))[0]
+                assign(path, folder, name, ext, "lyric",
+                       "a sidecar whose name collided with another in its album")
+            else:
+                park(path, "a sidecar of a track the plan could not place")
+        else:
+            owner = owner_folder(d)
+            if owner is AMBIGUOUS:
+                park(path, "a sidecar whose folder holds more than one album, so "
+                           "nothing says which one it belongs to")
+            elif owner:
+                assign(path, owner, base, ext,
+                       "playlist" if ext in (".m3u", ".m3u8") else "sidecar",
+                       "one album folder would have carried two files of the same "
+                       "sidecar name")
+            else:
+                park(path, "not music, or music with nothing beside it to belong to")
+
+
+# ---------------------------------------------------------------------- outputs
+def write_plan():
+    """mkdir rows for every destination folder and its whole chain, then the moves.
+    Parents first: the executor creates directories in the order they are listed."""
+    chain = set()
+    for _s, dst, _k in MOVES:
+        d = os.path.dirname(dst)
+        while d and d != ARGS.root and d != os.path.dirname(d):
+            chain.add(d)
+            d = os.path.dirname(d)
+    dirs = sorted(chain, key=lambda p: (p.count("/"), p))
+    with open(ARGS.out, "w") as fh:
+        fh.write("action\tsrc\tdst\tkind\n")
+        for d in dirs:
+            fh.write(f"mkdir\t\t{d}\tdir\n")
+        for src, dst, kind in sorted(MOVES, key=lambda m: (m[1], m[0])):
+            fh.write(f"move\t{src}\t{dst}\t{kind}\n")
+        for src, dst, kind in sorted(KEEPS, key=lambda m: (m[1], m[0])):
+            fh.write(f"keep\t{src}\t{dst}\t{kind}\n")
+    parked = [m for m in MOVES if m[2] == "park"]
+    sized = sum(HELD[s][0] for s, _d, _k in MOVES if s in HELD)
+    print(f"plan: {len(MOVES) - len(parked)} moves, {len(parked)} parked"
+          f" ({sized / 2**40:.2f} TiB touched), {len(KEEPS)} already placed,"
+          f" {len(dirs)} folders -> {ARGS.out}")
+
+
+def write_exceptions():
+    with open(ARGS.exceptions, "w") as fh:
+        fh.write(f"Music relayout exceptions for {ARGS.root}\n\n")
+        fh.write(f"Everything below is parked under {ARGS.park_dir}, keeping the\n"
+                 "relative path it has today. Nothing is deleted and nothing is\n"
+                 "overwritten: a parked file is one rename from wherever the plan is\n"
+                 "pointed next.\n\n")
+        for reason in sorted(PARKED, key=lambda r: -len(PARKED[r])):
+            rs = PARKED[reason]
+            fh.write(f"{len(rs):>6} files  {reason}\n")
+            for p in sorted(rs, key=rel)[:400]:
+                fh.write(f"          {rel(p)}\n")
+            if len(rs) > 400:
+                fh.write(f"          … and {len(rs) - 400} more\n")
+            fh.write("\n")
+        for section in sorted(NOTES, key=lambda s: -len(NOTES[s])):
+            lines = sorted(set(NOTES[section]))
+            fh.write(f"{len(lines):>6}  {section}\n")
+            for line in lines[:200]:
+                fh.write(f"          {line}\n")
+            if len(lines) > 200:
+                fh.write(f"          … and {len(lines) - 200} more\n")
+            fh.write("\n")
+        no_year = sorted(f"{folder_artist(g)}/{dominant(g['albums'])}"
+                         for g in ALBUMS.values() if not g["year"])
+        if no_year:
+            fh.write(f"{len(no_year):>6} albums carry no year in any tag, so their folders\n"
+                     "       are named without one; Jellyfin still matches them.\n")
+            fh.write("       " + ", ".join(no_year[:300]) + "\n\n")
+        clash = sorted(f"{folder_artist(g)}/{dominant(g['albums'])}"
+                       for g in ALBUMS.values() if len(g["mb"]) > 1)
+        if clash:
+            fh.write(f"{len(clash):>6} albums put one folder name on more than one\n"
+                     "       MusicBrainz release group — different releases, one name.\n"
+                     "       They share the folder Jellyfin's own album item names.\n")
+            fh.write("       " + ", ".join(clash[:200]) + "\n\n")
+        fh.write("       Playlists move with their album and their contents are never\n"
+                 "       rewritten: the paths inside an .m3u go stale.\n")
+    print(f"exceptions -> {ARGS.exceptions}")
+
+
+def main():
+    global ARGS
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT",
+                                                     "/mnt/entropy/Music"))
+    ap.add_argument("--enum", default="/tmp/music-enum.tsv",
+                    help="the enumerate TSV (bytes, ext, path)")
+    ap.add_argument("--rows", default="/tmp/jellyfin-music.tsv",
+                    help="the tag rows scripts/media-census.py music wrote")
+    ap.add_argument("--out", default="/tmp/music-plan.tsv")
+    ap.add_argument("--exceptions", default="/tmp/music-exceptions.txt")
+    ap.add_argument("--park", default="_unsorted",
+                    help="the folder inside the root that holds what cannot be filed")
+    ARGS = ap.parse_args()
+    ARGS.root = os.path.normpath(ARGS.root)
+    ARGS.park_dir = os.path.join(ARGS.root, ARGS.park)
+    read_inputs()
+    # Every path a file occupies today is claimed before a single destination is
+    # handed out: the executor refuses an occupied target, so a plan that aimed at
+    # one would only fail after the renames that were supposed to clear it.
+    CLAIMED.update(p.lower() for p in HELD)
+    for path, (_size, ext) in HELD.items():
+        if ext == ".cue":
+            CUE_DIRS.add(os.path.dirname(path))
+    group_albums()
+    place_albums()
+    place_sidecars()
+    write_plan()
+    write_exceptions()
+
+
+def read_inputs():
+    with open(ARGS.enum) as fh:
+        for line in fh:
+            parts = line.rstrip("\n").split("\t")
+            if len(parts) >= 3:
+                HELD[parts[2]] = (int(parts[0]), parts[1].lower())
+    with open(ARGS.rows, newline="") as fh:
+        for row in csv.DictReader(fh, delimiter="\t"):
+            if row.get("file"):
+                ROWS[row["file"]] = row
+    gone = [p for p in ROWS if p not in HELD]
+    if gone:
+        NOTES["a tag row whose file the census no longer sees"].extend(gone)
+
+
+if __name__ == "__main__":
+    main()

scripts/media-plan.py

@@ -0,0 +1,881 @@
+#!/usr/bin/env python3
+r"""Plan a Jellyfin-format relayout of a movie library.  DRY RUN ONLY: nothing moves.
+
+  media-plan.py [--root DIR] [--enum TSV] [--probed TSV] [--plex TSV]
+                [--out TSV] [--exceptions TXT]
+
+Inputs are the TSVs scripts/media-census.py exports, plus the optional Plex
+metadata-cache export whose clean titles outrank the filename parser; the title
+authority for everything Plex never matched is scripts/media-titles.py beside this
+file. No media file is opened, so planning the whole library costs seconds.
+
+PLAN is a TSV the executor (scripts/media-apply.py) replays verbatim —
+`action<TAB>src<TAB>dst<TAB>kind` with action in mkdir | move | keep, ordered so
+parents exist before children. EXCEPTIONS is the human review list: every file the
+plan could not name a film for, every collision it refused, the READMEs the
+executor must write into the piles it created.
+
+Target contract (jellyfin.org/docs/general/server/media/movies):
+  * one folder per movie, named  Title (year)
+  * the video inside is named exactly like the folder
+  * extra copies of one film are versions: "<base> - <label>.ext"; the base must
+    match the folder name character-for-character or Jellyfin splits them apart
+  * extras live in canonical subfolders (trailers, featurettes, deleted scenes,
+    interviews, samples, shorts, clips, other, extras, backdrops, theme-music)
+  * sidecars <base>.<lang>.<flag>.<ext>; images <base>-<poster|backdrop|...>
+  * <>:"/\|?* break the matcher, so they never survive here
+  * VIDEO_TS/BDMV folders support neither multiple versions nor parts
+"""
+import argparse
+import collections
+import csv
+import importlib.util
+import os
+import re
+
+# Inputs are the census TSVs scripts/media-census.py writes; the engine beside
+# this file is the title authority. Nothing here opens a media file.
+def parse_args():
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--root", default=os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies"),
+                    help="library root the plan moves files inside")
+    ap.add_argument("--enum", default="/tmp/media-enum.tsv", help="file census TSV")
+    ap.add_argument("--probed", default="/tmp/media-probed.tsv", help="ffprobe census TSV")
+    ap.add_argument("--plex", default="/tmp/plex-movies.tsv",
+                    help="Plex metadata export (clean titles); absent = parser only")
+    ap.add_argument("--out", default="/tmp/jf-plan.tsv")
+    ap.add_argument("--exceptions", default="/tmp/jf-exceptions.txt")
+    return ap.parse_args()
+
+
+ARGS = parse_args()
+ROOT = ARGS.root
+ENUM, PROBE, PLEX = ARGS.enum, ARGS.probed, ARGS.plex
+PLAN, EXCEPTS = ARGS.out, ARGS.exceptions
+ENGINE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "media-titles.py")
+
+spec = importlib.util.spec_from_file_location("eng", ENGINE)
+eng = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(eng)
+
+VIDEO_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg",
+             ".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".flv", ".vob", ".iso",
+             ".img", ".mpv", ".3gp", ".rm", ".rmvb", ".m2v", ".mpv"}
+DISC_DIRS = {"BDMV", "VIDEO_TS", "CERTIFICATE", "AACS", "MLPI", "CLIPINF"}
+SUB_EXT = {".srt", ".ass", ".ssa", ".sub", ".vtt"}
+IMG_EXT = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif", ".tbn"}
+AUDIO_DIR_RE = re.compile(r"soundtrack|score|\bost\b|music|song|album", re.I)
+SUB_KEEP_RE = re.compile(r"^(?!.*\d)(?!.*[._])(?!.*\[)(?!.*\()([A-Za-z][A-Za-z -]{1,24})$")
+MUSIC_TOP_RE = re.compile(r"(?i)soundtrack|score|\bost\b|music|album|song")
+SUB_DIR_OK = re.compile(r"(?i)^$|subs?|subtitles?|srt|custom ?subs?(\[.*\])?"
+                        r"|(additional|extra) languages|languages|resources")
+AUDIO_DIR_OK = re.compile(r"(?i)^$|audio(s)?|extra audio|additional audio"
+                          r"|(additional|extra) languages|commentar(y|ies)"
+                          r"|audio commentary|audio tracks?")
+
+
+AUDIO_EXT = {".mp3", ".flac", ".m4a", ".aac", ".dts", ".wav", ".ac3", ".mka",
+             ".opus", ".ogg", ".wma"}
+NFO_EXT = {".nfo"}
+IMG_PRIMARY = {"poster", "folder", "cover", "default", "movie", "show"}
+IMG_OTHER = {"backdrop": "backdrop", "fanart": "backdrop", "background": "backdrop",
+             "banner": "banner", "logo": "logo", "clearlogo": "logo",
+             "thumb": "thumb", "landscape": "thumb", "art": "backdrop",
+             "clearart": "backdrop", "jacket": "poster", "cdart": "poster",
+             "discart": "poster", "disc": "poster"}
+RES_RE = re.compile(r"\b(2160|4320|4k|1080|720|576|480|360|240)(i|p)\b", re.I)
+# The characters Jellyfin's matcher cannot carry in a file or folder name. "/" is
+# one of them and it is the dangerous one: Plex titles KEEP it ("V/H/S/2", and this
+# library holds six of those films), so leaving it in a name turns ONE movie folder
+# into a nested tree — Movies/V/H/S/2 (2013)/… — and Jellyfin then sees four
+# folders, none of them a movie.
+BAD_CHARS = re.compile(r'[<>:"/|?*]')
+PART_RE = re.compile(r"(?:^|[\s._-])(cd|dvd|part|pt|disc|disk|disco)"
+                     r"[\s._-]?([1-9abcd])(?:$|[\s._-])", re.I)
+LANGS = {"en", "eng", "it", "ita", "ja", "jpn", "de", "deu", "fr", "fra", "es",
+         "spa", "pt", "por", "ru", "rus", "zh", "chi", "ko", "kor", "sv", "no",
+         "da", "nl", "pl", "cs", "hu", "fi", "tr", "el", "he", "ar", "uk", "ro",
+         "bg", "hr", "sr", "sk", "et", "lv", "lt", "id", "th", "vi", "fa", "la"}
+FLAGS = {"default", "forced", "foreign", "sdh", "cc"}
+NOISE = {"sub", "subs", "subtitle", "subtitles", "srt", "ass", "ext", "external",
+         "the", "a", "an", "of", "and", "in", "on", "to", "for", "with", "ita",
+         "eng", "english", "italian", "dual", "audio", "forced", "default"}
+# Canonical title where the library only ever carries a localised name.
+OVERRIDES = {"28 giorni dopo": "28 Days Later",
+             "28 settimane dopo": "28 Weeks Later"}
+
+rows = []
+for line in open(ENUM):
+    size, ext, path = line.rstrip("\n").split("\t")
+    rows.append({"bytes": int(size), "ext": ext.lower(), "path": path})
+ext_by = {r["path"]: r["ext"] for r in rows}
+size_by = {r["path"]: r["bytes"] for r in rows}
+
+
+def rel(p):
+    return p[len(ROOT) + 1:] if p.startswith(ROOT + "/") else p
+
+
+def parts_of(p):
+    return [q for q in rel(p).split("/") if q]
+
+
+def is_qb(p):
+    return ".!q" in p.lower()
+
+
+def disc_root(p, fold=True):
+    """Absolute root of the disc tree containing p (the MOVIE folder that holds
+    BDMV/ or VIDEO_TS/), else None. `fold` also folds bookkeeping dirs that sit
+    beside the tree (CERTIFICATE/, AACS/) into the same movie folder."""
+    ps = parts_of(p)
+    kinds = DISC_DIRS if fold else {"BDMV", "VIDEO_TS"}
+    for i, q in enumerate(ps):
+        if q.upper() in kinds:
+            return ROOT + "/" + "/".join(ps[:i])
+    return None
+
+
+def row_name(r):
+    """The name a disc-tree entry should be identified by: the engine stores the
+    disc's own folder for it, because "BDMV/index.bdmv" is not a title."""
+    return r.get("name_src") or r["path"]
+
+
+def sanitize(name):
+    name = BAD_CHARS.sub("-", name).replace("\\", "-")
+    name = re.sub(r"\s+", " ", name).strip().strip(". ").strip()
+    return (name[:180] or "untitled")
+
+
+def ok_year(y):
+    """True for a year a real film could carry. Plex stores 0 for a file it indexed
+    but never MATCHED, and "0" is a digit string: trusting it named 67 movie
+    folders "Title (0)" — Jellyfin items with no identity, no year and no poster."""
+    return bool(re.fullmatch(r"(18|19|20)\d\d", y or "")) and 1870 <= int(y) <= 2035
+
+
+WORD_SPLIT = re.compile(r"[^0-9a-z]+")
+
+
+def words_of(text):
+    """The matching words of a name — or of a whole path, which is how callers
+    pass a file together with the folders it sits in.
+
+    Split on everything that is not a letter or a digit: eng.norm does not touch
+    "/", so feeding it "…/Psycho (1960)/Extras" left glued tokens like
+    "(1960)/extras" and "(1954)" that no other name could ever agree with, and a
+    film could not be found by the folder that holds it — which is how one box
+    set's featurettes ended up in another film's folder. Both sides of every
+    comparison below go through here. A bare number counts only when it could be
+    a release year, so release junk ("1080", "2160", "2", "7") never poses as
+    agreement between two names.
+    """
+    out = set()
+    for w in WORD_SPLIT.split(eng.norm(text)):
+        if not w or w in eng.STOP:
+            continue
+        if w.isdigit() and not 1870 <= int(w) <= 2100:
+            continue
+        out.add(w)
+    return out
+
+
+# ------------------------------------------------------------------ Plex titles
+# Plex has matched a large part of the library's media files to real titles
+# ("Alien", not "Alien.Directors.Cut.1979.1080p.BRrip.x265…"). Where the cache
+# holds a MATCH, its title wins; the filename parser carries everything else
+# (extras, disc trees, TV, and whatever Plex never matched).
+def plex_titles():
+    if not os.path.exists(PLEX):
+        return {}
+    out = {}
+    with open(PLEX, newline="") as fh:
+        for r in csv.DictReader(fh, delimiter="\t"):
+            t, y = r["leaf_title"].strip(), r["leaf_year"].strip()
+            # Only a match is an authority, and a match carries a year: an
+            # unmatched row echoes the filename, which the parser reads better
+            # than a copied release name. The junk filter stays for the matches
+            # whose title still carries the release.
+            if t and ok_year(y) and not re.search(
+                    r"2160p|1080p|x26[45]|hevc|bluray|web-?dl", t.lower()):
+                out[r["file"]] = (t, y)
+    return out
+
+
+PLEX_T = plex_titles()
+
+# ------------------------------------------------------------- classification
+erows = eng.load(PROBE)
+for r in erows:
+    for k, v in OVERRIDES.items():
+        if k in r["title"]:
+            r["title"] = v
+            r["key"] = eng.key_of(r["title"])
+            r["base"] = eng.base_words(r["title"], r["key"])
+marks = {r["path"] for r in erows if eng.is_watermark(r["path"], r["bytes"])}
+extras = {r["path"] for r in erows
+          if r["path"] not in marks
+          and eng.is_extra(r["path"], r["title"], r["dur"], ROOT,
+                           matched=r["path"] in PLEX_T)}
+eps = collections.defaultdict(list)
+for r in erows:
+    if r["path"] in marks | extras:
+        continue
+    ep = eng.ep_of(r["path"])
+    if ep:
+        eps[ep[0]].append(r)      # keyed by the SERIES, never the episode title
+tv_keys = {k for k, v in eps.items() if len(v) >= 5}
+tv_set = {r["path"] for k, v in eps.items() if k in tv_keys for r in v}
+# A folder INSIDE a movie's folder is bonus territory even when its own name is
+# not bonus vocabulary: "Saboteur (1942)/Exrtas/Saboteur - A Closer Look" is a
+# misspelled Extras folder, so the engine's folder rule never fired and the
+# featurette became a movie folder of its own — whose words then out-scored the
+# real Saboteur and drew that box set's other extras into it. The rule needs no
+# metadata: the folder holding the longest film owns the shorter video filed
+# under it, provided that shorter file's own name speaks the film's name — a
+# whole other film filed there stays a film, exactly as the engine's folder rule
+# exempts it. A different year in the file's own name IS that other film
+# speaking, so it vetoes the rule.
+def longest_of(rows):
+    """The longest video in each folder: that folder's feature."""
+    out = {}
+    for r in rows:
+        if r["dur"] > 0:
+            d = os.path.dirname(r["path"])
+            if d not in out or r["dur"] > out[d]["dur"]:
+                out[d] = r
+    return out
+
+
+def not_disc(r):
+    """A disc tree's innards belong to the tree, never to the folder's film."""
+    return not any(c in DISC_DIRS for c in r["path"].split("/"))
+
+
+FEATURE_OF = longest_of([r for r in erows if r["path"] not in marks and not_disc(r)])
+
+
+def under_a_feature(path, dur):
+    """True when a folder above this file holds a longer film that this file's
+    own name names."""
+    d = os.path.dirname(os.path.dirname(path))
+    while len(d) > len(ROOT):
+        f = FEATURE_OF.get(d)
+        if f:
+            if f["dur"] < dur:
+                return False
+            theirs = (words_of(os.path.basename(f["path"]))
+                      | words_of(os.path.basename(d)))
+            if not words_of(os.path.basename(path)) & theirs:
+                return False
+            y1 = re.search(r"(?:18|19|20)\d\d", os.path.basename(path))
+            y2 = re.search(r"(?:18|19|20)\d\d", os.path.basename(f["path"]))
+            return not (y1 and y2 and y1.group() != y2.group())
+        d = os.path.dirname(d)
+    return False
+
+
+extras |= {r["path"] for r in erows
+           if r["dur"] > 0 and r["path"] not in marks | extras | tv_set
+           and not_disc(r) and under_a_feature(r["path"], r["dur"])}
+film_rows = [r for r in erows if r["path"] not in marks | extras | tv_set]
+
+buckets = collections.defaultdict(list)
+for r in film_rows:
+    buckets[eng.key_of(r["title"])].append(r)
+
+
+def res_label(r):
+    m = RES_RE.search(r["res"] or "")
+    if m:
+        n, kind = m.group(1).lower(), m.group(2).lower()
+        return "2160p" if n in ("2160", "4k") else f"{n}{kind}"
+    try:
+        w = int((r["res"] or "0x0").split("x")[0])
+    except (ValueError, IndexError):
+        w = 0
+    return "2160p" if w >= 3000 else ("1080p" if w >= 1400 else "SD")
+
+
+def version_label(r):
+    bits = [res_label(r)]
+    vc = (r["vcodec"] or "").lower()
+    if vc in ("av1", "libaom", "libsvtav1"):
+        bits.append("AV1")
+    elif vc in ("vc1", "wmv3"):
+        bits.append("VC1")
+    if r["bytes"] > 40 * 2**30:
+        bits.append("REMUX")
+    if "imax" in r["path"].lower():
+        bits.append("IMAX")
+    return sanitize(" ".join(bits))
+
+
+def group_name(items):
+    titles = collections.Counter(r["title"] for r in items)
+    ascii_t = [t for t in titles if t.isascii()]
+    title = (collections.Counter(ascii_t) if ascii_t else titles).most_common(1)[0][0]
+    years = [r["year"] for r in items if r["year"] != "?"] or \
+            [r["soft"] for r in items if r["soft"]]
+    year = collections.Counter(years).most_common(1)[0][0] if years else None
+    return sanitize(f"{title} ({year})" if year else title)
+
+
+groups = []
+for key, items in buckets.items():
+    for c in eng.dur_clusters(items):
+        g = {"rows": c, "base": group_name(c)}
+        known = [PLEX_T[r["path"]] for r in c if r["path"] in PLEX_T]
+        if known:
+            # one film, one name: a Plex match for any copy names every copy
+            t, yr = collections.Counter(known).most_common(1)[0][0]
+            g["plex"] = t
+            g["base"] = sanitize(f"{t} ({yr})" if ok_year(yr) else t)
+        groups.append(g)
+clash = {b for b, n in collections.Counter(g["base"] for g in groups).items() if n > 1}
+for g in groups:
+    if g["base"] in clash:
+        # two genuinely different films share title+year: separate folders, and
+        # say so in the name, because Jellyfin can only match one item per folder
+        g["base"] = sanitize(g["base"] + " - " + version_label(g["rows"][0]))
+        g["flag"] = "same title and year as another film"
+
+moves, mkdirs, problems, readmes = [], set(), [], []
+# --------------------------------------------------------------- unnamed groups
+# A movie folder has to READ as a title. The engine's runtime veto says any file
+# over 45 minutes that is not named like an extra is a film whatever it is called
+# — true for Jellyfin, but a 118-minute documentary named "Extra 1" then produced
+# a movie folder named "1", and folders named "1", "2", "3" are Jellyfin items
+# with no identity whose numeric keys went on to capture every subtitle named
+# "3_Eng.srt" in the library. A year proves a dated feature ("300 (2006)" is a
+# real film), so the gate is: a year, or a word that is not release noise. With
+# neither, the rows are bonus material until a human names them.
+def named(g):
+    if re.search(r"\(\d{4}\)$", g["base"]):
+        return True
+    return any(w.isalpha() and len(w) >= 2 for w in eng.name_words(g["base"]))
+
+
+unnamed = [g for g in groups if not named(g)]
+groups = [g for g in groups if named(g)]
+for g in unnamed:
+    for r in g["rows"]:
+        extras.add(r["path"])
+        problems.append(
+            f"UNNAMED film {rel(r['path'])} parses to {g['base']!r} with no year: "
+            f"bonus pile, not a folder — give the file a real name and re-plan")
+
+
+group_of = {}
+for g in groups:
+    g["dir"] = f"{ROOT}/{g['base']}"
+    for r in g["rows"]:
+        group_of[r["path"]] = g
+        dr = disc_root(r["path"])
+        if dr:
+            group_of[dr] = g
+by_path = {r["path"]: r for r in erows}
+
+# -------------------------------------------------------------- bookkeeping
+claimed = {}
+existing = set(rows_map := {r["path"] for r in rows})
+
+
+at_target, booked_src = set(), set()
+
+
+def book(src, dst, kind):
+    if src == dst:
+        at_target.add(src)
+        booked_src.add(src)
+        return
+    if os.environ.get("JF_TRACE") and (src in booked_src or dst.lower() in claimed):
+        why = "rebook" if src in booked_src else "collision"
+        problems.append(f"TRACE {why} kind={kind} src={rel(src)} dst={rel(dst)}")
+    if src in booked_src:          # one source may only ever move once
+        problems.append(f"RE-BOOK {rel(src)} -> {rel(dst)} (already planned)")
+        return
+    if dst.lower() in claimed:
+        problems.append(f"COLLISION {rel(dst)}\n    from {rel(src)}\n    and  {rel(claimed[dst.lower()])}")
+        return
+    claimed[dst.lower()] = src
+    booked_src.add(src)
+    comp = dst.split("/")
+    for i in range(1, len(comp)):
+        mkdirs.add("/".join(comp[:i]))
+    moves.append((src, dst, kind))
+
+
+def uniq_dir(d, stem, ext, hint=""):
+    """A destination that no other source already took."""
+    cand = f"{d}/{stem}{ext}"
+    if cand.lower() not in claimed:
+        return cand
+    n = 2
+    while True:
+        cand = f"{d}/{stem} {n}{ext}"
+        if cand.lower() not in claimed:
+            return cand
+        n += 1
+
+
+def media_dir(p):
+    dr = disc_root(p)
+    return os.path.dirname(dr) if dr else os.path.dirname(p)
+
+
+# ------------------------------------------------------------------ films
+# film / disc moves: the largest copy carries the folder's own name, the rest
+# become Jellyfin versions; a disc tree moves as a whole into the movie folder
+
+
+def vext(p):
+    return ext_by.get(p) or os.path.splitext(p)[1].lower()
+disc_children = collections.defaultdict(list)
+for q in rows:
+    dr = disc_root(q["path"])
+    if dr:
+        disc_children[dr].append(q["path"])
+for g in groups:
+    primary_taken = False
+    disc_kinds = set()
+    for r in sorted(g["rows"], key=lambda r: -r["bytes"]):
+        dr = disc_root(r["path"])
+        if dr:
+            inner = os.path.basename(dr).upper()
+            if inner in disc_kinds:
+                problems.append(
+                    f"DISC-VERSION {g['base']}: extra disc tree {rel(dr)} cannot "
+                    f"share the folder — discs have no multiple versions")
+                continue
+            disc_kinds.add(inner)
+            for src in disc_children[dr]:
+                tail = os.path.relpath(src, dr)
+                fold = disc_root(src, fold=False)
+                if fold != dr and os.path.dirname(src) != dr:
+                    tail = os.path.basename(dr) + "/" + tail
+                book(src, f"{g['dir']}/{tail}", "disc-file")
+            continue
+        label = "" if not primary_taken else f" - {version_label(r)}"
+        dst = f"{g['dir']}/{g['base']}{label}{vext(r['path'])}"
+        n = 2
+        while dst.lower() in claimed:
+            dst = f"{g['dir']}/{g['base']}{label} {n}{vext(r['path'])}"
+            n += 1
+        r["dst"] = dst
+        primary_taken = True
+        book(r["path"], dst, "film")
+    g["primary"] = primary_taken
+
+# ------------------------------------------------------------------ extras
+EXTRA_KIND = [("trailers", r"trailer|teaser|preview|promo|tv spot|advert"),
+              ("deleted scenes", r"deleted|alt scene|gag reel|blooper|outtake"),
+              ("interviews", r"interview|cast and crew|session piece|commentary"),
+              ("featurettes", r"featurette|behind the scenes|making of|documentary"
+                              r"|heritage|archival|video archive|music video"),
+              ("samples", r"sample"),
+              ("shorts", r"\bshort\b|short film"),
+              ("clips", r"\bclip\b|scene selection|montage|epilogue|isolated score"),
+              ("other", r"menu|logo|gallery|still|trivia|storyboard|screen test"
+                        r"|audition|test footage|watermark|workprint|poster"
+                        r"|picture in picture|singalong|karaoke|original score")]
+THEME_RE = re.compile(r"theme|opening song|ending song|soundtrack|score"
+                      r"|\bost\b|music|song", re.I)
+BACKDROP_RE = re.compile(r"backdrop|intro", re.I)
+
+
+def extras_kind(path):
+    low = path.lower()
+    if THEME_RE.search(low):
+        return "theme-music"
+    if BACKDROP_RE.search(low):
+        return "backdrops"
+    for name, rx in EXTRA_KIND:
+        if re.search(rx, low):
+            return name
+    return "extras"
+
+
+def attachable_title(path):
+    t, _y = eng.title_of(path)
+    return eng.key_of(t) if t and not eng.is_watermark(path, size_by.get(path, 0)) else None
+
+
+for g in groups:
+    g["tops"] = {parts_of(r["path"])[0] for r in g["rows"]}
+top_groups = collections.defaultdict(list)
+for g in groups:
+    for t in g["tops"]:
+        top_groups[t].append(g)
+group_by_key = collections.defaultdict(list)
+for g in groups:
+    for r in g["rows"]:
+        group_by_key[eng.key_of(r["title"])].append(g)
+group_by_dir = {}
+for g in groups:
+    for r in g["rows"]:
+        group_by_dir.setdefault(media_dir(r["path"]), g)
+        dr = disc_root(r["path"])
+        if dr:
+            group_by_dir.setdefault(os.path.dirname(dr), g)
+
+
+def group_words(g):
+    """The words that identify a film group: its titles and the names of the
+    folders its files live in."""
+    if "words" not in g:
+        w = set()
+        if g.get("plex"):
+            w |= words_of(g["plex"])
+        for r in g["rows"]:
+            w |= words_of(r["title"])
+            w |= words_of(os.path.basename(media_dir(r["path"])))
+        g["words"] = w
+    return g["words"]
+
+
+def score_words(text, groups):
+    """Ranks film groups by how many of their words a path carries."""
+    words = words_of(text)
+    out = [(len(words & group_words(g)), g) for g in groups]
+    out.sort(key=lambda x: -x[0])
+    return out
+
+
+def group_for_orphan(path):
+    """Which movie folder an extra/sidecar belongs to. A collection folder holding
+    several films must be resolved by WORDS, never by position: taking the first
+    film of the folder attached every loose subtitle in a box set to whichever
+    movie happened to sort first."""
+    stem = os.path.splitext(os.path.basename(path))[0]
+    key = attachable_title(path)
+    if key and group_by_key.get(key):
+        cands = group_by_key[key]
+        top = parts_of(path)[0] if len(parts_of(path)) > 1 else ""
+        same_top = [g for g in cands if any(parts_of(r["path"])[0] == top
+                                            for r in g["rows"] if len(parts_of(r["path"])) > 1)]
+        pool = same_top or cands
+        return pool[0] if len(pool) == 1 else score_words(stem, pool)[0][1]
+    gtop = parts_of(path)[0] if len(parts_of(path)) > 1 else ""
+    pool = top_groups.get(gtop, [])
+    if len(pool) > 1:                    # a loose collection folder (box set, bonus
+        n, g = score_words(stem + " " + os.path.dirname(path), pool)[0]
+        return g if n >= 2 else None     # album) is resolved by words, not position
+    if len(pool) == 1:
+        return pool[0]
+    d = os.path.dirname(path)
+    while d and d != ROOT:
+        if d in group_by_dir:
+            return group_by_dir[d]
+        d = os.path.dirname(d)
+    return None
+
+
+tv_tops = {parts_of(p)[0] for p in tv_set}
+def film_folder_for(path):
+    """A movie folder for a lone feature that has no group: derived from its own
+    title when a year proves it, else None (goes to the review pile)."""
+    t, y = eng.title_of(path)
+    if not t or y == "?":
+        return None
+    d = uniq_dir(ROOT, sanitize(f"{t} ({y})"), "", "")
+    mkdirs.add(d)
+    return d
+
+
+for p in sorted(extras):
+    if disc_root(p):
+        continue
+    if len(parts_of(p)) > 1 and parts_of(p)[0] in tv_tops:
+        continue                        # a special inside a TV set: staged with it
+    g = group_for_orphan(p)
+    ext = ext_by.get(p, os.path.splitext(p)[1].lower())
+    stem = os.path.splitext(os.path.basename(p))[0]
+    if g is None:
+        solo = film_folder_for(p)
+        if solo:
+            dst = f"{solo}/{sanitize(stem)}{ext}"
+            problems.append(f"REVIEW lone feature {rel(p)}: gets its own folder "
+                            f"{os.path.basename(solo)}")
+        else:
+            dst = f"{ROOT}/_unsorted bonus/{extras_kind(p)}/{sanitize(stem)}{ext}"
+            problems.append(f"NO-FILM extra {rel(p)}")
+    else:
+        dst = f"{g['dir']}/{extras_kind(p)}/{sanitize(stem)}{ext}"
+    book(p, uniq_dir(os.path.dirname(dst), os.path.basename(dst)[:-(len(ext) or 1)],
+                     ext, os.path.dirname(p)), "extra")
+
+# Disc trees the engine never saw (a MakeMKV scratch folder, an AACS remnant):
+# they belong to the film of the folder they sit in, kept as a named subtree.
+for q in rows:
+    pth = q["path"]
+    dr = disc_root(pth)
+    if not dr or pth in by_path or pth in booked_src:
+        continue
+    g = group_by_dir.get(dr) or (top_groups[parts_of(pth)[0]][0]
+                                 if len(top_groups.get(parts_of(pth)[0], [])) == 1 else None)
+    if g is None:
+        continue
+    inner = os.path.basename(dr)
+    problems.append(f"REVIEW disc remnant {rel(dr)} moves inside {g['base']} "
+                    f"(a second master: compare before keeping both)")
+    for src in disc_children[dr]:
+        if src not in booked_src:
+            book(src, f"{g['dir']}/{inner}/{os.path.relpath(src, dr)}", "disc-file")
+
+
+# ----------------------------------------------------------------- sidecars
+media_by_dir = collections.defaultdict(list)
+for r in erows:
+    if r["path"] in marks:
+        continue
+    dst = r.get("dst")
+    if dst:
+        media_by_dir[media_dir(r["path"])].append(r)
+
+
+
+def sidecar_target(path, e):
+    d = os.path.dirname(path)
+    stem = os.path.splitext(os.path.basename(path))[0]
+    if d == ROOT:
+        # A file loose in the library ROOT names no film. media_by_dir[ROOT] is the
+        # set of root-level MOVIE files, and scoring a stray name against it
+        # attached every root-level leftover to whichever root-level film was
+        # largest — the plan sent ".DS_Store" into "Titanic (1997)/other/" that
+        # way. A root-level stray is nobody's sidecar: it goes to the review pile.
+        return None, None
+    cands = media_by_dir.get(d, [])
+    if not cands:
+        nd = os.path.dirname(d)          # a subs/ or Extras/ folder beside the media
+        while nd and nd != ROOT and not media_by_dir.get(nd):
+            nd = os.path.dirname(nd)
+        if media_by_dir.get(nd):
+            if nd != d:                  # a NAMED subfolder (Subs/, Extras/, the
+                g = group_for_orphan(path)   # box set's own root) carries evidence
+                dirs = {os.path.dirname(r.get("dst", "")) for r in media_by_dir[nd]}
+                if g is None or g["dir"] not in dirs:
+                    # ...but only a STRONG word match outranks the folder the file
+                    # sits in. A weak one is an accident: "3_Eng.srt" scored 1
+                    # against a junk group named "3" and escaped its own movie's
+                    # Subs/ folder, while "03 xXx 3 Return Of Xander Cage" scores 4
+                    # and genuinely belongs to the other film of the set.
+                    n = score_words(stem, [g])[0][0] if g else 0
+                    if n >= 2:
+                        return (g, None)
+                cands = media_by_dir[nd]
+    if not cands:
+        g = group_for_orphan(path)
+        return (g, None) if g else (None, None)
+    if len(cands) > 1:
+        w = [t for t in eng.norm(stem).split() if t not in NOISE]
+        cands = sorted(cands, key=lambda r: (
+            -len([t for t in eng.norm(os.path.basename(r["path"])).split() if t in w]),
+            -r["bytes"]))
+    return (None, cands[0])
+
+
+for q in rows:
+    p, e = q["path"], q["ext"]
+    if p in by_path or p in extras or p in marks or is_qb(p) or disc_root(p):
+        continue
+    if os.path.basename(p).startswith("._"):
+        # macOS AppleDouble resource forks: a few hundred bytes carrying a name
+        # COPIED FROM ANOTHER FILM. Word-matching them to "their" film scattered
+        # dead files through four movie folders (and they are video extensions in
+        # the AppleDouble case, so they also skipped every other filter). They
+        # claim nothing: one pile, and they stay in it — nothing is deleted here.
+        book(p, uniq_dir(f"{ROOT}/_macos sidecars", sanitize(os.path.basename(p))[:120],
+                         "", os.path.dirname(p)), "junk")
+        problems.append(f"JUNK AppleDouble sidecar {rel(p)} (macOS resource fork)")
+        continue
+    if e == ".iso":         # a DVD image: Jellyfin plays it as a movie file
+        g = group_for_orphan(p)
+        stem = os.path.splitext(os.path.basename(p))[0]
+        if g is not None:
+            book(p, uniq_dir(g["dir"], g["base"], e, stem) if g.get("primary") is not None
+                 else f"{g['dir']}/{g['base']}{e}", "iso")
+        else:
+            solo = film_folder_for(p)
+            if solo:
+                book(p, f"{solo}/{sanitize(stem)}.iso", "iso")
+                problems.append(f"REVIEW disc image {rel(p)}: DVD rip, alone in its "
+                                f"folder -> {os.path.basename(solo)}")
+            else:
+                book(p, f"{ROOT}/_unsorted files/{sanitize(os.path.basename(p))}", "orphan")
+                problems.append(f"NO-FILM disc image {rel(p)}")
+        continue
+    if e in VIDEO_EXT:
+        # A video the engine never probed (unreadable header) and not an AppleDouble
+        # remnant: the plan does not guess at it, so it stays where it is.
+        continue
+    stem = os.path.splitext(os.path.basename(p))[0]
+    g, media = sidecar_target(p, e)
+    if media is not None:
+        base = os.path.splitext(os.path.basename(media["dst"]))[0]
+        d = os.path.dirname(media["dst"])
+    elif g is not None:
+        base, d = g["base"], g["dir"]
+    else:
+        # Nothing names this file's film. NEVER scatter it: an archive set or an
+        # album tree must move as the unit it is, so the whole loose folder goes
+        # to the review pile and a film folder's own leftovers stay with it.
+        ps = parts_of(p)
+        if len(ps) < 2:
+            dst = f"{ROOT}/_unsorted files/{sanitize(stem)}{e}"
+        elif len(top_groups.get(ps[0], [])):
+            dst = f"{top_groups[ps[0]][0]['dir']}/other/{'/'.join(sanitize(c) for c in ps[1:])}"
+        else:
+            dst = f"{ROOT}/_unsorted files/{'/'.join(sanitize(c) for c in ps)}"
+        book(dst=dst, src=p, kind="orphan")
+        problems.append(f"NO-OWNER sidecar {rel(p)}")
+        continue
+    if e in IMG_EXT:
+        low = eng.norm(stem)
+        word = next((w for w in IMG_PRIMARY if re.search(rf"\b{w}\b", low)), None)
+        kind = "poster" if word else next(
+            (IMG_OTHER[w] for w in IMG_OTHER if re.search(rf"\b{w}\b", low)), None)
+        if re.search(r"\btheme\b", low):
+            dst = f"{d}/theme{e}"
+        else:
+            dst = f"{d}/{base}-{kind or 'art'}{e}" if kind else f"{d}/{sanitize(stem)}{e}"
+        book(p, uniq_dir(os.path.dirname(dst), os.path.basename(dst)[:-(len(e) or 1)],
+                         e, os.path.dirname(p)), "image")
+    elif e in NFO_EXT:
+        book(p, uniq_dir(d, base, ".nfo", os.path.dirname(p)), "nfo")
+    elif e in SUB_EXT | AUDIO_EXT:
+        toks = [t for t in re.split(r"[._\-\s]+", stem) if t]
+        langs = [t.lower() for t in toks if t.lower() in LANGS]
+        flags = [t.lower() for t in toks if t.lower() in FLAGS]
+        keep = [sanitize(t) for t in toks
+                if t.lower() not in langs + flags + list(NOISE)
+                and not re.fullmatch(r"\d{3,4}p?|cd\d+|\d{4}|\d+ch|\d+.\d", t.lower())]
+        bits = [base] + ([sanitize(" ".join(keep[:4]))] if keep else []) + langs + flags
+        if e in SUB_EXT:
+            # subtitles: <movie>/srt[/language]
+            d = f"{d}/srt"
+            pd = os.path.basename(os.path.dirname(p))
+            if SUB_KEEP_RE.match(pd) and not SUB_DIR_OK.match(pd):
+                d = f"{d}/{sanitize(pd)}"      # a real language folder survives
+        elif e in AUDIO_EXT and AUDIO_DIR_RE.search(os.path.dirname(p)):
+            d = f"{d}/theme-music"                 # bonus audio: one canonical folder
+            pd = os.path.basename(os.path.dirname(p))
+            if not MUSIC_TOP_RE.search(pd):        # an album folder keeps its name
+                d = f"{d}/{sanitize(pd)}"
+        else:
+                pd = os.path.basename(os.path.dirname(p))
+                if not AUDIO_DIR_OK.match(pd):
+                    d = f"{d}/{sanitize(pd)}"
+        stem2 = ".".join(bits) or sanitize(os.path.basename(p))
+        book(p, uniq_dir(d, stem2, e, os.path.basename(p)), "track")
+    else:
+        book(p, uniq_dir(f"{d}/other", sanitize(os.path.basename(p))[:120], e,
+                         os.path.dirname(p)), "misc")
+
+readmes.append(f"README in {ROOT}: this library was relaid out into Jellyfin's "
+               "one-folder-per-movie layout; `_release-group watermarks/`, "
+               "`_unsorted files/` and `_unsorted bonus/` hold files no film claims.")
+for key in tv_keys:
+    readmes.append(f"README in {ROOT}/{sanitize('_TV needs a TV library - ' + key)}: "
+                   "these are TV episodes found in the MOVIES library; move the folder "
+                   "to a TV library source, else Jellyfin groups them wrongly as one "
+                   "movie.")
+
+# --------------------------------------------------------------------- TV
+for key in tv_keys:
+    items = eps[key]
+    year = collections.Counter(r["year"] for r in items if r["year"] != "?")
+    yr = year.most_common(1)[0][0] if year else None
+    base = f"_TV needs a TV library - {sanitize(key)}" + (f" ({yr})" if yr else "")
+    tops = {parts_of(r["path"])[0] for r in items}
+    members = sorted([r["path"] for r in items]
+                     + [q for q in extras | marks if len(parts_of(q)) > 1
+                        and parts_of(q)[0] in tops],
+                     key=lambda q: q)
+    for src in members:
+        rel_parts = rel(src).split("/")
+        tail = "/".join(sanitize(c) for c in rel_parts[:-1] + [rel_parts[-1]])
+        book(src, f"{ROOT}/{base}/{tail}", "tv")
+for p in sorted(marks):
+    stem, ext = os.path.splitext(os.path.basename(p))
+    book(p, uniq_dir(f"{ROOT}/_release-group watermarks", sanitize(stem)[:120], ext,
+                     os.path.dirname(p)), "watermark")
+for q in rows:
+    if is_qb(q["path"]):
+        problems.append(f"SKIPPED qBittorrent partial {rel(q['path'])}")
+        book(q["path"], q["path"], "qb-partial")     # stays: resume data
+
+# --------------------------------------------------------------- debug hook
+if os.environ.get("JF_ISO"):
+    x = ROOT + "/High Anxiety (Mel Brooks) [1977] nordic subs  DVDrip/High Anxiety [1977].iso"
+    print("attachable", attachable_title(x), "| dir", os.path.dirname(x) in group_by_dir,
+          "| g", group_for_orphan(x) and group_for_orphan(x)["base"])
+    raise SystemExit
+if os.environ.get("JF_DEBUG"):
+    import json
+    unp = sorted({r["path"] for r in rows} - {m[0] for m in moves} - at_target)
+    print("at_target", len(at_target))
+    print("kinds-unplanned", collections.Counter(os.path.splitext(u)[1] for u in unp).most_common(8))
+    for up in unp[:20]:
+        d = os.path.dirname(up)
+        print("UNP", up[:90], "| mediadir?", d in media_by_dir, "| by_path", up in by_path,
+              "| extras", up in extras, "| discroot", disc_root(up), "| ext", os.path.splitext(up)[1])
+
+# -------------------------------------------------------------- validation
+srcs = [m[0] for m in moves]
+dup_src = {s for s, n in collections.Counter(srcs).items() if n > 1}
+for s in dup_src:
+    problems.append(f"DOUBLE-CLAIM source {rel(s)}")
+all_src = set(srcs)
+untouched = {r["path"] for r in rows} - all_src
+bad_names, into_alive, lost = [], [], []
+for src, dst, kind in moves:
+    for comp in dst[len(ROOT) + 1:].split("/"):
+        if (BAD_CHARS.search(comp) or comp != comp.strip(". ") or not comp
+                or len(comp) > 200):
+            bad_names.append(f"{kind}: {comp!r} in {rel(dst)}")
+    if dst in untouched:                       # something staying put is there
+        into_alive.append(f"{rel(src)} -> {rel(dst)} (occupied by a file that stays)")
+    if dst.lower() in {s.lower() for s in untouched}:
+        into_alive.append(f"{rel(src)} -> {rel(dst)} (case-collides with a survivor)")
+    if dst.startswith(src + "/") and src in all_src:
+        lost.append(f"{rel(src)} moves into its own subtree")
+# every accounted-for source must exist exactly once in the plan
+covered = {r["path"] for r in rows}
+not_planned = sorted(covered - all_src - at_target)
+
+with open(PLAN, "w") as out:
+    out.write("action\tsrc\tdst\tkind\n")
+    for d in sorted(mkdirs, key=lambda q: (q.count("/"), q)):
+        if d.startswith(ROOT + "/") and d != ROOT:
+            out.write(f"mkdir\t\t{d}\tdir\n")
+    for p in sorted(booked_src - {m[0] for m in moves}):
+        out.write(f"keep\t{p}\t{p}\n")
+    for src, dst, kind in sorted(moves, key=lambda m: m[0]):
+        out.write(f"move\t{src}\t{dst}\t{kind}\n")
+
+kinds = collections.Counter(k for _s, _d, k in moves)
+films_n = sum(1 for _s, _d, k in moves if k == "film")
+folders = len({os.path.dirname(d) for _s, d, k in moves if k in ("film", "disc-file")})
+with open(EXCEPTS, "w") as out:
+    out.write(f"groups={len(groups)} moves={len(moves)} folders={folders}\n")
+    out.write(f"kinds={dict(kinds)}\n")
+    out.write(f"at-target-already={len(at_target)} unplanned={len(not_planned)}\n")
+    out.write("\n=== WHAT THE EXECUTOR MUST ALSO WRITE ===\n")
+    out.write("\n".join(sorted(set(readmes))) + "\n")
+    out.write("\n=== PROBLEMS ===\n")
+    out.write("\n".join(sorted(set(problems))) + "\n")
+    out.write("\n=== BAD NAMES ===\n" + "\n".join(sorted(set(bad_names))) + "\n")
+    out.write("\n=== TARGETS OCCUPIED BY SURVIVORS ===\n"
+              + "\n".join(sorted(set(into_alive))) + "\n")
+    out.write("\n=== LEFTOVER SOURCES (first 60) ===\n"
+              + "\n".join(rel(p) for p in not_planned[:60]) + "\n")
+print(f"groups={len(groups)} folders={folders} moves={len(moves)} "
+      f"mkdirs={len(mkdirs)} film-files={films_n}")
+print(f"kinds={dict(kinds)}")
+print(f"problems={len(set(problems))} bad-names={len(set(bad_names))} "
+      f"occupied={len(set(into_alive))} double-claim={len(dup_src)}")
+print(f"left-unmoved={len(not_planned)}")
+print(f"tv-keys={len(tv_keys)} extras={len(extras)} marks={len(marks)} "
+      f"disc-trees={len(disc_children)}")

scripts/media-shows-tvdb-ids.tsv

@@ -0,0 +1,35 @@
+# The operator's tvdb id map: `folder <TAB> tvdb id <TAB> note`.
+#
+# These are the pack folders whose show tvdb's NAME SEARCH cannot reach: the
+# record exists, but tvdb titles it in the show's own language and the English
+# name rides in aliases the search never reads. Without a row here the folder
+# keeps the release folder's words and no episode title; with one, both matchers
+# read `[tvdbid-NNNN]` and the files rename to tvdb's own name and titles.
+#
+# Every id was read from tvdb's own record (`/series/<id>/extended`) on
+# 2026-10-31, and the note states what that record says plus the neighbouring
+# record a name search would have picked instead — the row is checkable without
+# re-searching, because one wrong id renames a whole pack to the wrong show. The
+# plan refuses a line whose id is not a number and lists every folder the map
+# touched in the exceptions file.
+#
+# Folders the name search DOES reach (the EXACT and PUNCT classes of the proposal
+# table) stay off this file on purpose: a map row would hide a live lookup behind
+# a number nobody re-checks.
+[Anime Time] Pokemon Complete Series (Season 01-25+Movies+Specials+Shorts+OST) [DVD+BD][1080p][HEVC 10bit x265][AAC]	76703	tvdb: ポケットモンスター (1997), the umbrella record for every era this pack spans; 1268 files
+[Anime Time] One Piece (0001-1071+Movies+Specials) [BD+CR] [Dual Audio] [1080p][HEVC 10bit x265][AAC][Multi Sub]	81797	tvdb: ワンピース (1999), alias ONE PIECE; the bare alias "One Piece" belongs to 392276, the 2023 live action; 1072 files
+[Judas] Naruto (Complete Series) [BD 1080p][HEVC x265 10bit][Multi-Audio][Multi-Subs]	78857	tvdb: NARUTO－ナルト－ (2002), alias "Naruto" carries the full-width dashes; 220 files
+Star Trek Deep Space 9 Deluxe DVD Boxset + Extras DVDRip TSV	72073	tvdb: Star Trek: Deep Space Nine (1993), alias "Star Trek Deep Space 9"; the box set's own words match nothing; 279 files
+Top Gear UK 1-17	74608	tvdb: Top Gear (2002), no "UK" alias — the US version is 199051; 131 files
+Black Clover 001	331753	tvdb: ブラッククローバー (2017), which holds NO alias at all; 378762 is Mugyutto! Black Clover (2019), a different show; 108 files
+Shin Chan - Complete Eng Dub Series 2160p Ai Upscaled -Mesc	437356	tvdb: Shin Chan (2006) — the English dub record this pack is; 79654 is the 1992 Japanese original; 78 files
+Fairy Tail (2009-2014) (Seasons 1-8 + OVAs) (BD 1080p)(HEVC x265 10bit)(Dual-Audio)(Eng-Subs)-Judas[TGx]	114801	tvdb: フェアリーテイル (2009), alias FAIRY TAIL; 410031 is FAIRY TAIL 100 NEN QUEST (2024), the sequel; 59 files
+(CBB) Tokyo Ghoul - The Complete Series (BD 1080p)(Dual-Audio)(HEVC-10bit-x265)	281249	tvdb: 東京喰種トーキョーグール (2014) — :re and √A are seasons of THIS record, confirmed by search; 58 files
+[Anime Time] Demon Slayer - Kimetsu no Yaiba (S01+S02+S03+S04+Mugen Train Arc+Movie) [BD] [Dual Audio][1080p][HEVC 10bit x265][AAC][Eng Sub]	348545	tvdb: 鬼滅の刃 (2019), alias "Demon Slayer: Kimetsu no Yaiba" — the folder's hyphen splits it differently; 56 files
+(CBB) Tengen Toppa Gurren Lagann - The Complete Series (BD 1080p)(Dual-Audio)(HEVC-x265-10bit)	80096	tvdb: 天元突破グレンラガン (2007), alias Tengen Toppa Gurren Lagann; 51 files
+Fullmetal Alchemist - Brotherhood	85249	tvdb: 鋼の錬金術師 FULLMETAL ALCHEMIST (2009), alias "Full Metal Alchemist Brotherhood" — spaced, unlike this folder; 64 files
+[NewbSubs] Attack on Titan~Shingeki no Kyojin Series (1080p Blu-ray x265 Dual Audio)	267440	tvdb: 進撃の巨人 (2013), aliases carry "Attack on Titan" and "Shingeki no Kyojin", joined here by a tilde; 27 files
+Kimetsu no Yaiba	348545	the same record as the Anime Time pack above: two release folders, ONE show; 12 files
+Rick and Morty The Anime	420279	tvdb: リック・アンド・モーティ：ジ・アニメ (2024), no English alias on the record; 10 files
+[Anime Time] Dorohedoro (Season 1+OVA+Extra) [Dual Audio][BD][1080p][HEVC 10bit x265][AAC][Eng Sub]	370761	tvdb: ドロヘドロ (2020), alias Dorohedoro; 9 files
+Ken Il Guerriero HOKUTO NO KEN	79156	tvdb: 北斗の拳 (1984), alias "Hokuto no Ken"; the Italian title matches nothing and 465216 is the 2026 remake; 5 files

scripts/media-shows-tvdb.py

@@ -0,0 +1,306 @@
+#!/usr/bin/env python3
+r"""Ask tvdb what it calls the shows a plan renamed WITHOUT a tvdb id.
+
+ media-shows-tvdb.py [--plan TSV] [--root DIR] [--park DIR] [--map TSV]
+                     [--out TSV] [--credentials FILE]
+
+`media-shows.py` renames a series to tvdb's own words only when an id backs the
+name; everything else keeps the metadata provider's words and no episode title,
+and the plan's mkdir rows are where those folders show up. This script turns that
+list into a decision sheet: one row per destination series folder that carries no
+`[tvdbid-NNNN]`, with what tvdb's search answers for it and a verdict that says
+HOW MUCH was guessed to get there.
+
+It changes nothing. The sheet is for the operator; the file that changes
+behaviour is `scripts/media-shows-tvdb-ids.tsv`, whose rows he approves. That is
+the loop: plan -> this sheet -> approved map rows -> re-plan.
+
+Verdicts, weakest last:
+
+  EXACT   tvdb's own name equals the folder's words as they stand.
+  PUNCT   it equals them only after release junk is stripped (pack numbers,
+          codecs, group names, `Complete Series`); the difference is spelling,
+          not identity.
+  NEAR    tvdb answered something whose name is NOT the query. The best hit is
+          printed so a human can judge it, and it is never applied by anything.
+  NONE    nothing left to ask with, or no answer at all.
+
+Acceptance is deliberately unforgiving — a hit counts only when tvdb's name
+equals the query with every non-alphanumeric dropped from both sides, and the
+series' year still has to fall inside any year span the folder names. The
+cleaner is the only thing that got smarter over the passes this tool replaces;
+the acceptance test never moved, because a wrong id renames a whole pack to the
+wrong show. A query that needs the cleaner is reported as PUNCT for exactly that
+reason: the machine stripped words, so a human reads the row.
+
+Read-only against tvdb. The credential file is the planner's own
+(`api_key=`, optional `pin=`), read here and never printed.
+"""
+import argparse
+import collections
+import importlib.util
+import json
+import os
+import re
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+ARGS = argparse.ArgumentParser(description=__doc__.splitlines()[0],
+                               formatter_class=argparse.RawDescriptionHelpFormatter)
+ARGS.add_argument("--plan", default="/tmp/tv-plan.tsv",
+                  help="the plan media-shows.py just wrote (its mkdir rows are "
+                       "the folders this sheet asks about)")
+ARGS.add_argument("--root", default=os.environ.get("TV_ROOT", "/mnt/entropy/TVShows"),
+                  help="the TV library the plan renames")
+ARGS.add_argument("--park", default="",
+                  help="the review pile inside it (its folders are not shows)")
+ARGS.add_argument("--map",
+                  default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                       "media-shows-tvdb-ids.tsv"),
+                  help="the operator's approved map; its folders are reported as "
+                       "MAPPED and never asked again")
+ARGS.add_argument("--credentials",
+                  default=os.path.expanduser("~/.config/secrets/tvdb"),
+                  help="file with an api_key= line (and a pin= for a user-issued "
+                       "key); mode 600, and never printed")
+ARGS.add_argument("--out", default="/tmp/tvdb-proposal.tsv",
+                  help="the decision sheet")
+ARGS = ARGS.parse_args()
+
+ROOT = os.path.abspath(ARGS.root)
+PARK = os.path.abspath(ARGS.park or os.path.join(ROOT, "_unsorted"))
+
+_spec = importlib.util.spec_from_file_location(
+    "eng", os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                        "media-titles.py"))
+eng = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(eng)
+
+# Release metadata, not the show: everything a scene folder name says about the
+# ENCODING, the packaging or the languages, which tvdb has never heard of.
+JUNK = re.compile(
+    r"complete|series|dual|multi|audio|sub|subs|multisub|dualaudio|"
+    r"\d{3,4}p|1080p|720p|2160p|480p|4k|hd|hdtv|cam|hdcam|telesync|x264|x265|hevc|avc|"
+    r"\d+bit|bit|bluray|blu-?ray|bdrip|web-?rip|web-?dl|webrip|hdtvrip|"
+    r"dvdrip|dvdr|dvdscr|tsr|tsv|mkv|avi|mp4|aac|ac3|dts|eac3|flac|ddp?5[.1]1|"
+    r"bd|dvd|rip|boxset|box|season|seasons|episodes?|specials?|shorts|ost|"
+    r"ova|ovas|oad|repack|proper|internal|mi?xed|budgetbits|deluxe|remux|"
+    r"remastered|uncut|collector|edition|extras?|volume|volumes|part|"
+    r"disc|disk|cd|ntsc|pal|region|fixed|patched|proof|english|esp|"
+    r"espagnol|french|vostfr|subtitul\w*|spanish|german|italian|ita|eng|fre|spa|lat|"
+    r"upscaled?|ai|dd|movie|movies",
+    re.I)
+GROUP = re.compile(r"KONTRAST|TGx|Judas|Silence|FreetheFilms|NTb|DEADPOOL|Xploit|"
+                   r"SM737|TSA|3OISL3R|PLAYWEB|EPiC|FLUX|HONEY|Zaxton|Cca3rW|"
+                   r"monikeres|frank|hdh|psi|rtfpemaker|cbm|web|asser|astro|"
+                   r"yfn|uindex|www|org|com|net|mesc|ataraxiaprime|seph\d*|"
+                   r"[a-z]+subs?|edge", re.I)
+BRACKET = re.compile(r"[\[({]([^\])}]*)[\])}]")
+NUM_RANGE = re.compile(r"(?<!\w)\d{1,4}\s*(?:-|–|to|through)\s*\d{1,4}(?!\w)", re.I)
+INSIDE = re.compile(r"all|complete|full|dual|multi|audio|sub|subs|ova|ovas|"
+                    r"movies?|episodes?|extras?|bonus|pack|remux|region\s*\d*|"
+                    r"bd|dvd|cd|ntsc|pal|hr-?dvd", re.I)
+
+
+def drop_bracket(m):
+    """A bracket says nothing when its words are junk; keep it when they aren't."""
+    toks = [t for t in re.split(r"[\s_.,+~&]+", m.group(1)) if t]
+    if not toks or all(INSIDE.fullmatch(t) or JUNK.fullmatch(t)
+                       or GROUP.fullmatch(t)
+                       or re.fullmatch(r"\d+[a-z]?", t, re.I)
+                       or re.fullmatch(r"(19|20)\d\d", t) for t in toks):
+        return " "
+    return " " + m.group(1) + " "
+
+
+def clean(name):
+    """The folder's words with everything about the RELEASE taken out, so what is
+    left is what a person would type into tvdb. `Star Trek Deep Space 9 Deluxe
+    DVD Boxset + Extras DVDRip TSV` becomes `Star Trek Deep Space 9`; `SNL 16-20`
+    becomes `SNL`."""
+    name = re.sub(r"^\s*www\b.*?\b(org|com|net)\b\s*[-–]\s*", " ", name, flags=re.I)
+    name = re.sub(r"\bS\d{1,2}([ .Ee]\d+)?[-–Ee\d.]*", " ", name)
+    name = re.sub(r"\b\d{1,2}[xX]\d{1,3}\b", " ", name)
+    name = NUM_RANGE.sub(" ", name)
+    name = re.sub(r"(19|20)\d\d(?:\s*[-/–]\s*(?:19|20)\d\d)?", " ", name)
+    name = re.sub(r"\bby\s+[A-Za-z0-9_.-]+$", " ", name)
+    name = BRACKET.sub(drop_bracket, name)
+    name = re.sub(r"[\[()\]:,;!?*]", " ", name)
+    toks = re.split(r"[\s_.\-+~&]+", name)
+    out = []
+    for i, t in enumerate(toks):
+        if not t or JUNK.fullmatch(t) or GROUP.fullmatch(t):
+            continue
+        if re.fullmatch(r"\d+", t):
+            after = sum(1 for x in toks[i + 1:] if x
+                        and not JUNK.fullmatch(x) and not GROUP.fullmatch(x)
+                        and not re.fullmatch(r"\d+", x))
+            # A pack index (`001`) and a lead-in number over a real title
+            # (`01 Charles Barkley`) are noise; `30 Rock`, `24`, `Deep Space 9`
+            # are the title itself.
+            if len(t) >= 3 or (not out and after >= 2):
+                continue
+        out.append(t)
+    while out and out[-1].lower() in ("and", "with", "the", "a", "an", "to", "of"):
+        out.pop()
+    return " ".join(out).strip()
+
+
+def years(name):
+    return [int(y) for y in re.findall(r"\b(19\d\d|20\d\d)\b", name)]
+
+
+def bare(s):
+    return re.sub(r"[^a-z0-9]", "", s.lower())
+
+
+CRED = {}
+try:
+    for _line in open(ARGS.credentials):
+        if "=" in _line:
+            k, v = _line.strip().split("=", 1)
+            CRED[k.strip()] = v.strip()
+except OSError:
+    print(f"tvdb: no credentials at {ARGS.credentials} — the sheet would be all "
+          "NEAR and NONE, which is not a decision sheet", file=sys.stderr)
+    sys.exit(2)
+if not CRED.get("api_key"):
+    print(f"tvdb: no api_key in {ARGS.credentials}", file=sys.stderr)
+    sys.exit(2)
+
+TOKEN = {"jwt": ""}
+
+
+def call(path):
+    """One tvdb v4 GET, logging in again once when the JWT expires mid-run."""
+    for attempt in (1, 2):
+        if not TOKEN["jwt"]:
+            body = {"apiKey": CRED["api_key"]}
+            if CRED.get("pin"):
+                body["token"] = CRED["pin"]
+            req = urllib.request.Request("https://api4.thetvdb.com/v4/login",
+                                         data=json.dumps(body).encode(),
+                                         headers={"Content-Type":
+                                                  "application/json"})
+            TOKEN["jwt"] = json.load(
+                urllib.request.urlopen(req, timeout=30))["data"]["token"]
+        try:
+            req = urllib.request.Request("https://api4.thetvdb.com/v4" + path,
+                                         headers={"Authorization":
+                                                  "Bearer " + TOKEN["jwt"]})
+            return json.load(urllib.request.urlopen(req, timeout=30))["data"]
+        except urllib.error.HTTPError as e:
+            if e.code == 401 and attempt == 1:
+                TOKEN["jwt"] = ""
+                continue
+            raise
+
+
+MAPPED = {}
+if ARGS.map and os.path.exists(ARGS.map):
+    for _line in open(ARGS.map):
+        _line = _line.rstrip("\n")
+        if not _line.strip() or _line.startswith("#"):
+            continue
+        cols = _line.split("\t")
+        if len(cols) > 1 and cols[1].strip().isdigit():
+            MAPPED[cols[0].strip()] = cols[1].strip()
+
+# The folders the plan created inside the library, with the file count each one
+# will hold. The review pile is not a show: its files belong to no identity, and
+# naming the pile would put one junk series on the sheet for all of them.
+shows = collections.Counter()
+for line in open(ARGS.plan).read().splitlines()[1:]:
+    parts = line.split("\t")
+    if len(parts) < 4 or parts[0] != "mkdir":
+        continue
+    dst = os.path.abspath(parts[2])
+    if not dst.startswith(ROOT + os.sep) or dst.startswith(PARK + os.sep):
+        continue
+    top = os.path.relpath(dst, ROOT).split(os.sep)[0]
+    if top.startswith((".", "_")):
+        continue
+    if "[tvdbid-" in top:
+        continue                                # an id already backs it
+    shows.setdefault(top, 0)
+for line in open(ARGS.plan).read().splitlines()[1:]:
+    parts = line.split("\t")
+    if len(parts) < 4 or parts[0] != "move":
+        continue
+    dst = os.path.abspath(parts[2])
+    if not dst.startswith(ROOT + os.sep) or dst.startswith(PARK + os.sep):
+        continue
+    top = os.path.relpath(dst, ROOT).split(os.sep)[0]
+    if "[tvdbid-" in top:
+        continue
+    shows[top] += 1
+
+tally = collections.Counter()
+rows = []
+for folder in sorted(shows, key=lambda f: (-shows[f], f)):
+    files, tag, note = shows[folder], "NONE", ""
+    if folder in MAPPED:
+        rows.append([files, "MAPPED", folder, "", "", "", MAPPED[folder],
+                     "the operator's map already states this id"])
+        tally["MAPPED"] += 1
+        continue
+    query, untouched = clean(folder), clean(folder) == folder
+    ys = years(folder)
+    if not query:
+        rows.append([files, "NONE", folder, "", "", "", "",
+                     "nothing is left of the folder's name once release junk"
+                     " is gone"])
+    else:
+        try:
+            hits = call("/search?query=" + urllib.parse.quote(query)
+                        + "&type=series") or []
+        except Exception as e:                                  # noqa: BLE001
+            hits = []
+            note = f"tvdb did not answer: {e}"
+        want = bare(query)
+        pick = None
+        for h in hits:
+            if bare(h.get("name") or "") != want:
+                continue
+            hy = str(h.get("year") or "")
+            if ys and hy.isdigit() and not (min(ys) <= int(hy) <= max(ys)):
+                continue
+            pick = h
+            break
+        if pick:
+            tag = "EXACT" if untouched else "PUNCT"
+            rows.append([files, tag, folder, query, pick.get("name", ""),
+                         str(pick.get("year", "") or ""),
+                         str(pick.get("tvdb_id", "") or ""), ""])
+        elif hits:
+            best = hits[0]
+            rows.append([files, "NEAR", folder, query, best.get("name", ""),
+                         str(best.get("year", "") or ""),
+                         str(best.get("tvdb_id", "") or ""),
+                         f"{len(hits)} answer(s); tvdb's own name is not the "
+                         "query, so nothing would take it"])
+        else:
+            rows.append([files, "NONE", folder, query, "", "", "",
+                         note or "tvdb has no series by that name"])
+    tally[rows[-1][1]] += 1
+    time.sleep(0.12)
+
+# Two folders mapping to one record is not two shows; it is one show the release
+# folders split. Say it on every row involved, or the operator approves the same
+# id twice and reads the second as a mistake.
+by_id = collections.Counter(r[6] for r in rows if r[6] and r[1] != "MAPPED")
+for r in rows:
+    if r[6] and r[1] != "MAPPED" and by_id[r[6]] > 1:
+        r[7] = (f"MERGE: {by_id[r[6]]} folders map to tvdb {r[6]} — one show, "
+                "the releases split it; " + r[7]).strip("; ")
+
+with open(ARGS.out, "w") as fh:
+    fh.write("files\tverdict\tfolder\tquery\ttvdb_name\ttvdb_year\ttvdb_id\tnote\n")
+    for r in rows:
+        fh.write("\t".join(str(c) for c in r) + "\n")
+print(f"sheet: {len(rows)} folders"
+      + "".join(f" {t}={n}" for t, n in sorted(tally.items()))
+      + f" -> {ARGS.out}")