docs(alloy): document the root journal-read cutover

Alloy page rewritten to the shipped root design (why root, the two
mounts that make it work, deploy-pending status); Observability current
state marks the log pipeline fixed-in-code, awaiting deploy.

AGENTS.md

@@ -166,9 +166,11 @@ bare compose container never returned; 2026-09-11: five observability
 services had no units). The observability services now have units under
 `dankclaw/observability/quadlet/` and their compose definitions were removed —
 one owner per container name. The stack is **configured and delivering as of
-2026-09-17**: a Grafana datasource and the `llm-fleet` dashboard are
+2026-09-17**: Grafana datasources (Prometheus and Loki) and the `llm-fleet`
-provisioned from `shared/grafana/provisioning/`, `shared/prometheus/rules/
+dashboard are provisioned from `shared/grafana/provisioning/`, and
-llm-fleet.yml` loads 5 alert rules + 1 recording rule, and firing rules are
+`shared/prometheus/rules/llm-fleet.yml` loads two rule groups — `llm-fleet`
+(llama.cpp/halogen, `llamacpp:*`) and `vllm-fleet` (radiance, `vllm:*`) — alert
++ recording rules, and firing rules are
 delivered: Prometheus → Alertmanager (`dankclaw/observability/quadlet/
 alertmanager.container`, loopback :9093) → host-side Signal relay
 (`scripts/signal-relay.py`, user unit, 192.168.0.143:18093 — the host LAN

dankclaw/observability/quadlet/alloy.container

@@ -1,55 +1,51 @@
-# Quadlet unit: alloy (rootless, boot-persistent).
+# Quadlet unit: alloy (ROOT system service, boot-persistent).
 #
-# Install path on dankclaw: ~/.config/containers/systemd/alloy.container
+# Install path on dankclaw: /etc/containers/systemd/alloy.container
-# Unit name: alloy.service (container name: alloy)
+# Unit name: alloy.service (container name: alloy) — SYSTEM manager, root.
 #
-# Log shipper. This replaces the compose service of the same name in
+# WHY ROOT (not rootless, unlike its observability siblings). Alloy reads the
-# dankclaw/compose.yaml ("Observability" block).
+# host journal. The journal files are root:systemd-journal 0640 (other::---);
+# a rootless container carries no supplementary groups and cannot open them —
+# a rootless read of system.journal returns DENIED (see the Alloy page in the
+# vault). Running as root, one loki.source.journal reads system.journal AND
+# user-1000.journal, so it carries both host logs and the rootless Quadlet
+# containers' logs. Validated 2026-10-20: a root probe read 31,825 journal
+# lines and pushed 26,966 into Loki in 20 s, zero drops.
 #
-# IMAGE. franksdank/alloy is built locally by deploy-remote.sh
+# NETWORK=host. Loki publishes 127.0.0.1:3100 (loopback only) on the host, and
-# (build_compose_image) from dankclaw/alloy/Containerfile: upstream
+# no bridge container — rootless or root — can reach the host's loopback, so
-# grafana/alloy plus a static busybox for the healthcheck. Pull=never keeps a
+# this unit runs on the host network and pushes to 127.0.0.1:3100. The side
-# boot from reaching for that local tag at a registry.
+# effect: the UI must bind loopback too, so Exec sets the listen-addr to
+# 127.0.0.1 (the old rootless unit used 0.0.0.0 only because container loopback
+# was not host loopback; under Network=host they are the same address, so the
+# unauthenticated UI stays loopback-only — nothing off-host reaches it).
 #
-# NETWORK. It joins dankclaw_default so its config's loki.write endpoint
+# THE /etc/machine-id MOUNT IS LOAD-BEARING. sd_journal locates the journal
-# resolves the container name loki:3100.
+# directory by the local machine id; without it the container opens nothing and
+# the journal source reads exactly 0 lines — silently, no error, health green.
+# This mount is the entire reason the pipeline carries data. Never remove it.
 #
-# THE UI PORT EXISTS ON PURPOSE. Alloy's UI listens on the container's own
+# IMAGE. Built rootless by build_compose_image alloy, then mirrored into the
-# 127.0.0.1 by default and cannot be moved from the config file (alloy 1.x
+# root image store by the deployer (a root build runs in the build container's
-# refuses the ui block), so the compose service had no reachable health
+# own resolver and fails on DNS here). Pull=never: boot must not reach for the
-# endpoint at all: the homepage status card for alloy reported "down"
+# local tag at a registry.
-# continuously and there was nothing for a deploy to wait on. The command
-# below therefore moves the UI to the container's wildcard address and the
-# port is published on host loopback only, which puts alloy in the same class
-# as the other three: reachable from the host, by Caddy over the private
-# network, and by nothing off-host. Alloy has no authentication, which is
-# exactly why it is not published wider.
-#
-# SOURCES. The host journald socket and the host log/container-storage trees
-# are bind-mounted read-only; Alloy tails them and pushes to Loki. The config
-# lives at /opt/alloy/config/config.alloy, installed from
-# shared/alloy/config.alloy on every deploy.
-#
-# ORDERING. Wants/After loki.service: without a reachable Loki, Alloy's write
-# client fails and its log pipeline backs up.
 
 [Unit]
 Description=Alloy log shipper (Dankclaw)
-After=network-online.target loki.service
+After=network-online.target
-Wants=network-online.target loki.service
 
 [Container]
 ContainerName=alloy
 Image=localhost/franksdank/alloy:latest
 Pull=never
-Network=dankclaw_default
+Network=host
-Exec=run --server.http.listen-addr=0.0.0.0:12345 /etc/alloy/config.alloy
+Exec=run --server.http.listen-addr=127.0.0.1:12345 /etc/alloy/config.alloy
-PublishPort=127.0.0.1:12345:12345
+Volume=/opt/alloy/config:/etc/alloy:ro
-Volume=/opt/alloy/config:/etc/alloy
+Volume=/etc/machine-id:/etc/machine-id:ro
 Volume=/run/systemd/journal/socket:/run/systemd/journal/socket:ro
-Volume=/var/log:/var/log:ro
+Volume=/var/log/journal:/var/log/journal:ro
-Volume=/var/lib/containers:/var/lib/containers:ro
+Volume=/run/log/journal:/run/log/journal:ro
-HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:12345/-/healthy
+HealthCmd=/usr/local/bin/busybox wget --spider -q http://127.0.0.1:12345/-/healthy
 HealthInterval=30s
 HealthTimeout=10s
 HealthRetries=3
@@ -60,4 +56,4 @@ Restart=always
 RestartSec=10
 
 [Install]
-WantedBy=default.target
+WantedBy=multi-user.target

scripts/authentik-set-login-session-duration.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Set the authentik login-session TTL on Dankclaw.
+#
+# The login-session lifetime is governed by the `default-authentication-login`
+# stage's `session_duration`, NOT a global setting and NOT SESSION_COOKIE_AGE
+# (that is the pre-login session, sessions.unauthenticated_age). Authentik does
+# not expose SESSION_SAVE_EVERY_REQUEST at all, so the session is an absolute
+# window from login, not a sliding one.
+#
+# The field is a STRING in authentik's own idiom ("days=7", "seconds=30",
+# "hours=12") — not a Python timedelta. Writing str(timedelta(days=7)) stores
+# "7 days, 0:00:00", which authentik's timedelta_from_string parser REJECTS
+# (ValueError), and login then crashes computing the session expiry. Always pass
+# the "key=value" form; this script validates it before writing.
+#
+# Existing sessions keep their old expiry; the new TTL applies from the next
+# login. This is DB state — it is NOT repo-managed and a blueprint re-import or
+# rebuild-from-scratch reverts it. Run this again after such a rebuild.
+#
+# Usage: authentik-set-login-session-duration.sh ["days=7"]   (default days=7)
+set -euo pipefail
+
+DUR="${1:-days=7}"
+STAGE="default-authentication-login"
+HOST="${AUTHENTIK_HOST:-dankclaw}"
+
+# Refuse anything that is not the safe key=value duration idiom.
+if ! printf '%s' "$DUR" | grep -Eq '^[A-Za-z]+=[0-9]+([ ]+[A-Za-z]+=[0-9]+)*$'; then
+    echo "invalid duration '$DUR' — use authentik form, e.g. 'days=7' or 'seconds=30'" >&2
+    exit 2
+fi
+
+ssh "$HOST" 'podman exec -i authentik-server ak shell' <<PYEOF
+from django.apps import apps
+from authentik.lib.utils.time import timedelta_from_string
+dur = "$DUR"
+timedelta_from_string(dur)  # validate the form before touching the row
+ULS = apps.get_model("authentik_stages_user_login", "UserLoginStage")
+s = ULS.objects.get(name="$STAGE")
+s.session_duration = dur
+s.save(update_fields=["session_duration"])
+back = ULS.objects.get(name="$STAGE").session_duration
+assert back == dur, "persisted value mismatch: %r" % back
+print("session_duration set to", repr(back), "->", timedelta_from_string(back))
+PYEOF
+
+echo "Login-session TTL for '$STAGE' is now $DUR (applies on next login)."

scripts/deploy-remote.sh

@@ -487,13 +487,15 @@ seed_grafana_env() {
         || { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; }
 }
 
-# The five observability containers left compose on 2026-09-11 (they were
+# The observability containers left compose on 2026-09-11 (they were started by
-# started by hand with podman-compose and never came back after a reboot).
+# hand with podman-compose and never came back after a reboot). Five are ordinary
-# They are ordinary rootless Quadlet units now: same images, same mounts, same
+# rootless Quadlet units: same images, same mounts, same arguments, same
-# arguments, same container names, so nothing that addresses them by name —
+# container names, so nothing that addresses them by name — Caddy, Prometheus'
-# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
+# own scrape config — changed. Only the published ports did: the three HTTP UIs
-# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
+# bind 127.0.0.1, because the compose services published on 0.0.0.0 and none of
-# compose services published on 0.0.0.0 and none of them authenticates.
+# them authenticates. Alloy is the exception: a ROOT system service on
+# Network=host, because reading the host journal needs privileges a rootless
+# container cannot get. It is handled by deploy_alloy, not the loop below.
 deploy_observability() {
     local service="$1"
     # Observability units live under <target>/observability/quadlet: Dankclaw
@@ -512,10 +514,6 @@ deploy_observability() {
             copy_loki_config
             build_compose_image loki
             ;;
-        alloy)
-            copy_alloy_config
-            build_compose_image alloy
-            ;;
         grafana)
             copy_grafana_provisioning
             seed_grafana_env || return 1
@@ -534,11 +532,71 @@ deploy_observability() {
         alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
         grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
         loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
-        alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
         node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
     esac
 }
 
+# Mirror an image the deployer built rootless into the ROOT image store. A root
+# quadlet reads root storage; a root build (build_compose_image is rootless) is
+# not possible on this host — the build container's resolver fails DNS — so the
+# only way to get a root-readable copy is save-and-load. Compares image ids so a
+# rebuilt alloy actually re-lands in root storage instead of leaving a stale tag.
+mirror_image_to_root() {
+    local ref="$1" rid
+    rid="$(podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" || {
+        echo "ERROR: rootless image $ref is not built; cannot mirror to root" >&2
+        return 1
+    }
+    if [[ "$(run_privileged podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" == "$rid" ]]; then
+        return 0
+    fi
+    local tar; tar="$(mktemp /var/tmp/alloy-mirror.XXXXXX.tar)"
+    podman save -o "$tar" "$ref"
+    run_privileged podman load -i "$tar"
+    rm -f "$tar"
+}
+
+# Install a quadlet into the SYSTEM generator dir (root manager) instead of the
+# user dir. Boot persistence is the unit's own [Install] section (multi-user
+# target); the v1 system generator creates the .wants on daemon-reload, so —
+# like the user units — no `systemctl enable` (generated units cannot be enabled
+# here).
+install_system_quadlet() {
+    local source_dir="$1" unit="$2"
+    run_privileged mkdir -p /etc/containers/systemd
+    run_privileged cp "$source_dir/$unit.container" /etc/containers/systemd/
+    run_privileged systemctl daemon-reload
+}
+
+# Alloy as a ROOT system service on Network=host. See the unit file: a rootless
+# container cannot read the 0640 journal files, and it cannot reach Loki's
+# 127.0.0.1:3100 publish from a bridge netns. Root + host network fixes both;
+# the price is that this one container runs as root and on the host network.
+deploy_alloy() {
+    copy_alloy_config
+    build_compose_image alloy                      # rootless build (resolver works)
+    mirror_image_to_root localhost/franksdank/alloy:latest
+
+    # Retire the rootless unit BEFORE the system one starts: both managers
+    # generate alloy.service, and two containers named "alloy" both binding
+    # 127.0.0.1:12345 cannot coexist. Move the user quadlet out of the generator
+    # dir (it regenerates from there) and stop the user unit.
+    if [[ -f "$quadlet_unit_dir/alloy.container" ]] ||
+       systemctl --user is-active --quiet alloy.service 2>/dev/null; then
+        systemctl --user stop alloy.service 2>/dev/null || true
+        if [[ -f "$quadlet_unit_dir/alloy.container" ]]; then
+            local trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-alloy-root-cutover"
+            mkdir -p "$trash"
+            mv -- "$quadlet_unit_dir/alloy.container" "$trash/alloy.container"
+            echo "retired rootless alloy.container -> $trash"
+        fi
+        systemctl --user daemon-reload
+    fi
+
+    install_system_quadlet dankclaw/observability/quadlet alloy
+    run_privileged systemctl restart alloy.service
+    wait_for_health alloy http://127.0.0.1:12345/-/healthy 60
+}
 build_compose_image() {
     local service="$1" image cfhash oldhash containerfile
     # Every file the image COPIES IN, beside the Containerfile. The skip below
@@ -1941,7 +1999,8 @@ deploy_service() {
         copyparty) deploy_copyparty ;;
         plex) deploy_plex ;;
         tautulli) deploy_tautulli ;;
-        prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
+        alloy) deploy_alloy ;;
+        prometheus|grafana|loki|node-exporter|alertmanager) deploy_observability "$service" ;;
         comfyui) deploy_comfyui ;;
         *) deploy_compose_service "$service" ;;
     esac

scripts/frank-tts.env

@@ -7,7 +7,7 @@
 # whichever TTS engine wins the engine decision (dankclaw/voiceservice/app.py
 # is a single stdlib-only file in a locally built image). Consumers — omp's
 # TTS device, OpenWebUI, open-notebook — address a VOICE BY NAME
-# (voice=narrator1, or the harness' voice_id=narrator1) and never see a path:
+# (voice=wilwheaton, or the harness' voice_id=wilwheaton) and never see a path:
 # a name is real exactly when /voices/<name>/reference.wav exists, and
 # anything else is a 400 that lists the names that are.
 #
@@ -65,7 +65,7 @@ TTS_UPSTREAM_MODEL=voxcpm2
 # name still 400s with the list that exists — this covers the absent case only —
 # and the journal marks such a line `(default)` so an unrequested voice never
 # reads as a requested one. Any pack directory in /home/dank/voices is legal.
-DEFAULT_VOICE=narrator1
+DEFAULT_VOICE=wilwheaton
 
 # CLI-backend knobs, inert while TTS_BACKEND=omni-http. Empty means "not
 # configured": the endpoint answers a 500 that says so instead of running

serverAppConfigs-docs/Dankclaw/Frank TTS.md

@@ -21,7 +21,7 @@ never learn a path.
 **Deployed 2026-10-18** — `./scripts/deploy.sh dankclaw --service frank-tts`
 both units `active`, no drop-ins, and the homepage card's `wip` gate came off in
 the same change. Probes on the deployed endpoint: `/health` answers
-`{"status":"ok","backend":"omni-http","voices":["narrator1","narrator2","narrator3"]}`
+`{"status":"ok","backend":"omni-http","voices":["dr.kern","garrickhagon","jeffhays","vaati","wilwheaton"]}`
 and two different voices rendered the same sentence into **different** bytes
 (24 kHz mono PCM16, 4.24 s, healthy RMS) — so name resolution is real, not a
 pass-through of one default. The operator harness routes its `speech` role here;
@@ -42,8 +42,8 @@ render needs the app installed, not this endpoint.
 
 ## Naming a voice
 
-A voice **is** a directory name. `voice=narrator1` resolves to
+A voice **is** a directory name. `voice=wilwheaton` resolves to
-`/voices/narrator1/reference.wav`, and that file — not a manifest, not a config
+`/voices/wilwheaton/reference.wav`, and that file — not a manifest, not a config
 entry — is what makes the name real. omp's TTS device may send `voice_id`; the
 shim accepts either. A call that sends **neither** — the harness's normal shape,
 because no harness setting carries a voice for this transport — speaks
@@ -59,10 +59,11 @@ loudly instead of letting it half-work.
 
 Names are user-visible: an unknown voice comes back as
 `400 unknown voice 'x'; available voices: …`, and that list is the honest
-census. The podcast speakers are `narrator1`, `narrator2`, `narrator3` —
+census. As of 2026-10-20 the live packs are `dr.kern`, `garrickhagon`,
-[[Open Notebook]] is configured with exactly those three, so renaming them
+`jeffhays`, `vaati`, `wilwheaton`. [[Open Notebook]]'s `dank-duo` profile speaks
-breaks episode rendering, not just speech. A personal voice gets the person's
+`wilwheaton` (Frank) and `vaati` (Comrade), and `DEFAULT_VOICE` is `wilwheaton`.
-name, lowercase.
+Renaming a pack that a speaker profile names breaks episode rendering, not just
+speech. A personal voice gets the person's name, lowercase.
 
 ## Voice packs
 
@@ -221,5 +222,5 @@ harness bundle and measured the day of the flip:
 | Endpoint answers, synthesis fails | the engine: `podman healthcheck run frank-tts-engine`, then `systemctl --user status frank-tts-engine.service` |
 | First request of the day is ~0.6 s slower than the rest — or every name returns 400 | the warm pass did not run; it carries `-`, so it cannot fail the unit. `journalctl --user -u frank-tts-engine` for the warmer's lines, then `podman exec frank-tts-engine /app/warm-voices.sh` — the registry is memory-only (law 1 of [[Frank TTS Engine]]) |
 | The unit never appears after a deploy | the generator dropped it: `frank-tts-engine.network` is missing from `~/.config/containers/systemd/` |
-| A consumer rejects `narrator1` by listing 29 names (`af_heart … bm_fable`) and no request ever reaches this endpoint | that consumer is synthesizing **in the browser** with Kokoro.js, whose voice pack is fixed — OpenWebUI 0.11.4 ships it as the `browser-kokoro` TTS engine; see [[OpenWebUI]] |
+| A consumer rejects `wilwheaton` by listing 29 names (`af_heart … bm_fable`) and no request ever reaches this endpoint | that consumer is synthesizing **in the browser** with Kokoro.js, whose voice pack is fixed — OpenWebUI 0.11.4 ships it as the `browser-kokoro` TTS engine; see [[OpenWebUI]] |
 ---

serverAppConfigs-docs/Dankclaw/Open Notebook.md

@@ -45,6 +45,8 @@ open-notebook ([lfnovo/open-notebook](https://github.com/lfnovo/open-notebook))
 
 The app's own login stays on behind the chain. It is not redundant: publishing the API on loopback does **not** hide it from this host's other containers, which can dial `open-notebook:5055` by name on `dankclaw_default`. The chain decides *who from outside* gets in; `OPEN_NOTEBOOK_PASSWORD` decides what anything that already reached the port may do.
 
+**Single tenant — there are no user accounts, so this is an admin-side tool.** Upstream ships one global password and nothing else: `PasswordAuthMiddleware` compares `Bearer <OPEN_NOTEBOOK_PASSWORD>`, and the SurrealDB schema has no user/account/owner table (verified in the running image, 2026-10-20). One shared password therefore means **one shared workspace** — the guard chain can gate the door but cannot partition the data, and Authentik identity is never consumed. Multi-user is *approved but unbuilt* upstream (issue **#712**, "design phase, targeted post-v2"); OIDC/SSO was declined for now (**#607** closed). Real per-person isolation means **one deployment per person**, not a flag to flip. For that reason the homepage lists open-notebook under **admin**, not as a user-facing app.
+
 ## What the image actually runs
 
 One container, three processes under supervisord (`supervisord.conf` upstream): `uvicorn api.main:app --port 5055` (`:12`), the job worker (`:25`), and the Next.js frontend (`:39`). Two consequences the config depends on:
@@ -68,7 +70,7 @@ Credentials live in the database, encrypted with `OPEN_NOTEBOOK_ENCRYPTION_KEY`,
 | Model row `voxcpm2` | type `text_to_speech` on the frank-tts credential. The name is decorative: frank-tts ignores the request's `model` field and synthesizes whatever `voice` names ([[Frank TTS]]) |
 | Model row `frank-archon` | type `language`; `PUT /api/models/defaults` points chat, transformation and large-context at it |
 | Model row `qwen3-embedding-0.6b` | type `embedding` on the NPU-lane credential; `default_embedding_model` points at it, so every note save embeds through the XDNA2 lane ([[frank-dragoon]]) |
-| Speaker profile `dank-duo` | two speakers, Frank → `voice_id: narrator1`, Comrade → `narrator2`, `voice_model` = the TTS row |
+| Speaker profile `dank-duo` | two speakers, Frank → `voice_id: wilwheaton`, Comrade → `vaati`, `voice_model` = the TTS row |
 | Episode profile `dank-daily` | `num_segments: 4`, `speaker_config` = `dank-duo`, outline and transcript LLM = the archon row, briefing that ends "Never invent numbers." |
 
 Order that works over REST: credentials → model rows (`POST /api/models` with `credential: <credential id>`) → `PUT /api/models/defaults` → speaker profile (`voice_model: <model id>`) → episode profile (`speaker_config` / `outline_llm` / `transcript_llm`). `POST /api/models/{id}/test` proves each row: the language row answered with a real completion from frank-archon; the TTS row reached frank-tts and came back `unknown voice 'alloy'; available voices: narrator1, narrator2, narrator3` — the app's own test path hard-codes an OpenAI voice, so that message *is* the proof the leg works.
@@ -106,13 +108,13 @@ A **speaker profile** carries one to four speakers (`docs/3-USER-GUIDE/creating-
 
 | Speaker | Voice ID to type |
 |---------|------------------|
-| Speaker 1 | `narrator1` |
+| Speaker 1 | `wilwheaton` |
-| Speaker 2 | `narrator2` |
+| Speaker 2 | `vaati` |
-| Speaker 3 | `narrator3` |
+| Speaker 3 | `dr.kern` |
 
 Those are the pack names frank-tts resolves from `/home/dank/voices` (`scripts/frank-tts.env` header: a voice is real exactly when `/voices/<name>/reference.wav` exists). Two traps, both from upstream's shipped defaults:
 
-- The three seeded speaker profiles carry **OpenAI voice names** (`nova`, `alloy`, …) and **no voice model at all** — they show a "Setup required" badge, and generation fails with a voice error until they are edited (`creating-podcasts.md:13`, `:33`; `local-tts.md:89`). Add a `narrator1`/`narrator2` pair (or a third/fourth) rather than editing the seeded ones in place.
+- The three seeded speaker profiles carry **OpenAI voice names** (`nova`, `alloy`, …) and **no voice model at all** — they show a "Setup required" badge, and generation fails with a voice error until they are edited (`creating-podcasts.md:13`, `:33`; `local-tts.md:89`). Add a `wilwheaton`/`vaati` pair (or a third/fourth) rather than editing the seeded ones in place.
 - Two speakers with the same voice are hard to follow (`creating-podcasts.md:89`). Distinct packs per speaker is the point of the 1–4 shape.
 
 ## Two buttons that fail against frank-tts, and why that is cosmetic

serverAppConfigs-docs/Dankclaw/OpenWebUI.md

@@ -290,9 +290,12 @@ on large sizes ⇒ check ComfyUI resources; 401 ⇒ the token is bad.
 
 ## Text-to-speech (frank-tts)
 
-TTS is the fleet's one OpenAI-compatible speech endpoint —
+TTS is the fleet's one OpenAI-compatible speech endpoint. The config DB
-`http://host.containers.internal:7772/v1` (`audio.tts.openai.api_base_url`
+`audio.tts.*` is set to engine `openai`, `audio.tts.openai.api_base_url`
-in the config DB, 2026-10-18), model `frank-tts`, voice `narrator1`. That
+`http://192.168.0.143:7772/v1`, `audio.tts.voice` `wilwheaton` (re-keyed
+2026-10-20 from the retired `narrator1`; 7772 is a published port so
+`host.containers.internal` also reaches it, but the LAN IP is the canonical
+on-host form). That
 port is the rootless quadlet `frank-tts` (this repo:
 `dankclaw/voiceservice/quadlet/frank-tts.container`
 `scripts/frank-tts.env`), and its shim resolves `voice=<name>` against
@@ -306,8 +309,8 @@ ships an in-page Kokoro implementation (chunk carrying a frozen voice table
 `onnx-community/Kokoro-82M-v1.0-ONNX`, plus a personal *Kokoro.js dtype*
 setting). Its engine select takes `""` (Default) or `browser-kokoro`; when
 `browser-kokoro` is chosen the page synthesizes locally and rejects any
-voice outside its fixed pack — so `narrator1` fails with
+voice outside its fixed pack — so `wilwheaton` fails with
-`Voice: "narrator1" not found. Should be one of: af_heart, …` **without a
+`Voice: "wilwheaton" not found. Should be one of: af_heart, …` **without a
 single request reaching the server**. Personal audio settings are
 browser: no `user.settings` row and no container env carries an audio
 override, yet the toast still appears ⇒ the choice lives in that browser's

serverAppConfigs-docs/Infrastructure/Alloy.md

@@ -3,70 +3,111 @@ type: reference
 description: "Alloy is Grafana's log and telemetry shipper."
 tags:
   - infrastructure
-timestamp: 2026-09-11
+timestamp: 2026-10-20
 ---
 
 # Alloy
 
-Alloy is Grafana's log and telemetry shipper. It replaces Promtail (end of life as of 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/). It is wired to forward container and journal logs to Loki, and every unit in the path is healthy — but as of 2026-10-18 it is delivering **zero** log lines. See "Delivery state" below.
+Alloy is Grafana's log and telemetry shipper. It replaces Promtail (end of life
+as of 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/). It
+forwards the host journal — including the rootless Quadlet containers' logs —
+to Loki. Unlike its observability siblings it runs as a **root system service on
+the host network**, because a rootless container cannot read the journal (see
+*Why root* below).
 
-**Host:** Dankclaw (one instance per host) — **Port:** 12345 (loopback) — **Docs:** https://grafana.com/docs/alloy/latest
+**Host:** Dankclaw (one instance per host) — **UI/health:** 127.0.0.1:12345
+(loopback) — **Docs:** https://grafana.com/docs/alloy/latest
 
 ## Facts
 
 | Item | Value |
 |------|-------|
-| Image | `franksdank/alloy:latest` (local build: upstream + a static busybox for the healthcheck) |
+| Image | `localhost/franksdank/alloy:latest` — built rootless, then mirrored into the root image store by the deployer (a root build fails on DNS in this host's build container) |
-| Published | `127.0.0.1:12345` → 12345 (its own UI/health port) |
+| Unit | `dankclaw/observability/quadlet/alloy.container` → system `alloy.service`, installed at `/etc/containers/systemd/` (root manager) |
-| Unit | `dankclaw/observability/quadlet/alloy.container` → `alloy.service` |
+| Config | `shared/alloy/config.alloy` → `/opt/alloy/config/config.alloy` on every deploy |
-| Config | `shared/alloy/config.alloy`, installed to `/opt/alloy/config/config.alloy` on every deploy (`scripts/deploy-remote.sh:281-284`) |
+| Network | `host` — required so it can reach Loki's `127.0.0.1:3100` publish |
-| Inputs | host journald socket, podman container logs, `/var/log` |
+| UI | `--server.http.listen-addr=127.0.0.1:12345` (loopback; nothing off-host reaches it) |
-| Output | `http://loki:3100/loki/api/v1/push` (`shared/alloy/config.alloy:13-17`) |
+| Inputs | host journal: `system.journal` + `user-*.journal` (root reads both) |
-| Successor to | Promtail (EOL 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/) |
+| Output | `http://127.0.0.1:3100/loki/api/v1/push` |
-| Boot persistence | `Restart=always`/`RestartSec=10` + `WantedBy=default.target` (`dankclaw/observability/quadlet/alloy.container:58-63`); installed at `~/.config/containers/systemd/alloy.container`, reporting `generated` + `active` (`systemctl --user is-enabled|is-active alloy` 2026-10-18) |
+| Boot persistence | `Restart=always`/`RestartSec=10` + `WantedBy=multi-user.target`; boot persistence is the `[Install]` section (generated system units cannot be `enable`d here) |
-| Delivery | **Configured and healthy, currently delivering zero data** — see "Delivery state" |
+| Successor to | Promtail (EOL 2026-03-02) |
+| Deploy status | root cutover built and tested in-repo (2026-10-20); the **live** instance is still the old rootless unit until `./scripts/deploy.sh dankclaw --service alloy` runs |
 
-## Why Alloy and not Promtail
+## Why root (the load-bearing decision)
 
-Promtail reached end-of-life on 2026-03-02 and Grafana Labs directs new installs to Alloy. This stack uses Alloy from day one: no Promtail config or unit has ever existed in this repository (`git log -S promtail` over the whole history returns no commit that added one; the compose-era service of the same name was already Alloy — `dankclaw/observability/quadlet/alloy.container:6-7`).
+A rootless Alloy delivered **zero** lines, silently and with a green health
+check. Two independent causes, both proven on the host 2026-10-20:
+
+1. **The journal is not readable by a rootless container.** The files are
+   `root:systemd-journal 0640` (`group:adm`, `other::---`). Even bind-mounted, a
+   rootless container has no supplementary groups, so opening them returns
+   `DENIED`. `loki.source.journal` then sees an empty view and the counter just
+   sits at 0 — no error logged.
+2. **Container logs live in the *user* journal, not `ctr.log`.** The rootless
+   Quadlet containers log via the journald driver into `user-1000.journal`; the
+   old file globs (`/var/lib/containers/.../ctr.log`, `/var/log/containers/*.log`)
+   point at a root-storage layout that does not exist here.
+
+**Root fixes both at once.** As root the container opens every journal file
+under the machine-id directory — `system.journal` *and* `user-1000.journal` —
+so a single `loki.source.journal` carries host services and the rootless
+containers' logs together. Validated with a throwaway root probe before writing
+the unit: **31,825 journal lines read, 26,966 pushed to Loki in 20 s, zero
+drops**, and Loki's label set went from empty to populated.
+
+### The two mounts that make it work
+
+- **`/etc/machine-id` (ro) — load-bearing.** `sd_journal` locates the journal
+  directory by the *local* machine id. Without it the container opens nothing and
+  the journal source reads exactly 0 lines — silently, health green. This single
+  mount is the whole difference between a working and an empty pipeline; do not
+  remove it. (A root probe without it reads 0; with it, 31,825.)
+- **`Network=host`.** Loki publishes `127.0.0.1:3100` (loopback only). No bridge
+  container — rootless or root — can reach the host's loopback, so Alloy must
+  share the host network to push to `127.0.0.1:3100`. Consequence: the UI binds
+  `127.0.0.1` directly (no `PublishPort`), so it stays loopback-only as before.
 
 ## Configuration
 
-`shared/alloy/config.alloy` is the source of truth:
+`shared/alloy/config.alloy` is the source of truth — one `loki.source.journal`
-
+(labelled `job="systemd-journal"`) forwarding to the loopback `loki.write`
-- Reads journald logs from the host's `/run/systemd/journal/socket`, labelled `job="systemd-journal"` (`shared/alloy/config.alloy:20-26`).
+endpoint. There is deliberately **no** container-file source: container stdout is
-- Reads container logs from two globs: `/var/lib/containers/storage/overlay-containers/*/userdata/ctr.log` (`job="podman"`) and `/var/log/containers/*.log` (`job="containers"`) (`config.alloy:29-35`).
+in the journal.
-- Forwards everything to `http://loki:3100/loki/api/v1/push` by container name on `dankclaw_default` (`config.alloy:13-17`).
 
 ## Volumes
 
 | Container path | Host path |
 |----------------|-----------|
-| `/etc/alloy` | `/opt/alloy/config` |
+| `/etc/alloy` | `/opt/alloy/config` (ro) |
-| `/run/systemd/journal/socket` | host journald socket (read-only) |
+| `/etc/machine-id` | host machine id (ro) — **required** |
-| `/var/log` | host logs (read-only) |
+| `/run/systemd/journal/socket` | host journald socket (ro) |
-| `/var/lib/containers` | podman storage (read-only) |
+| `/var/log/journal` | persistent journal files (ro) |
+| `/run/log/journal` | runtime journal files (ro) |
 
-## The UI port is published on purpose
+## The UI stays on loopback
 
-Alloy's UI listens on the container's own `127.0.0.1:12345` by default, and Alloy 1.x refuses to configure the `ui` service from a config block ("service ui does not support being configured", `shared/alloy/config.alloy:8-10`). Under compose that meant **no reachable health endpoint at all**: the homepage status card for alloy read `down` continuously and a deploy had nothing to wait on. The unit therefore passes `--server.http.listen-addr=0.0.0.0:12345` (`dankclaw/observability/quadlet/alloy.container:46`) and publishes `127.0.0.1:12345` (`alloy.container:47`, confirmed by `ss -ltnp` 2026-10-18) — same posture as the other three: reachable from the host and from Caddy's network, and from nothing off-host. Alloy has no authentication, which is exactly why it is not published wider. The homepage card probes that same loopback port (`shared/homepage/services.json:233`).
+Alloy 1.x refuses to configure the `ui` service from a config block, so the UI
-
+address is set on the command line instead. Under the old rootless unit that
-## Delivery state (2026-10-18): healthy, shipping nothing
+meant `0.0.0.0` inside the container plus a published `127.0.0.1:12345`; under
-
+`Network=host` the listen-addr is `127.0.0.1:12345` directly — the host's
-The unit is `active`, the container reports `healthy`, the log says `Alloy is ready`, and the pipeline carries no data at all. Measured on 2026-10-18:
+loopback, reachable from the host and from nothing off-host. Alloy has no
-
+authentication, which is exactly why it is never published wider. The homepage
-- `curl -s http://127.0.0.1:12345/metrics` → `loki_source_journal_target_lines_total{component_id="loki.source.journal.journal"} 0` and `loki_source_file_files_active_total{component_id="loki.source.file.containers"} 0`. The journal source logs no error; the counter simply never moves. The journal files it would have to read through the `/var/log` bind are `root:systemd-journal 0640` with an ACL (`ls -l /var/log/journal/64a287d7…/` 2026-10-18).
+card and the deploy health gate both dial `http://127.0.0.1:12345/-/healthy`.
-- Both file globs were refused at start (`journalctl --user -u alloy`, 2026-10-18T08:25:27Z), verbatim: `failed to create source, skipping … component_id=loki.source.file.containers error="failed to tail file, stat failed: stat /var/lib/containers/storage/overlay-containers/*/userdata/ctr.log: permission denied"` and the same message for `/var/log/containers/*.log: no such file or directory`.
-- Downstream that means Loki holds zero streams — `GET /loki/api/v1/labels` returns `{"status":"success"}` with no `data` — see [[Loki]].
-
-No fix is claimed here: the unit, the config and the push endpoint are all correct as written, and the gap is source-side access (and a Kubernetes-style glob that does not exist on this host). Anyone closing it should re-check the two counters above, not the health status — health is exactly what hides this failure mode.
 
 ## Gotchas
 
-- The journald socket mount is required for journal scraping.
+- **No `/etc/machine-id` mount ⇒ silent zero.** The health check still passes.
-- Container log paths differ between docker (`/var/lib/docker/containers`) and podman (`/var/lib/containers/storage/...`); point the config at the right one per host.
+  This is the failure mode that hid the broken pipeline; check
-- One Alloy per host; do not run a single instance that tries to reach both hosts' logs.
+  `loki_source_journal_target_lines_total` and the Loki label query, never
-- The unit `Wants=`/`After=` `loki.service`: without a reachable Loki the write client fails and the pipeline backs up.
+  `/healthy`.
+- **`After=loki.service` is gone.** Alloy is on the system manager; Loki is a
+  rootless user unit — a system unit cannot order against a user unit. Alloy
+  retries its push until Loki answers, so ordering is unnecessary.
+- **The image must be in the ROOT store.** The deployer builds rootless and
+  mirrors (`mirror_image_to_root`); a `Pull=never` root quadlet with only a
+  rootless tag fails at create.
+- One Alloy per host; do not run a single instance that tries to reach both
+  hosts' logs.
 
 ## See Also
 

serverAppConfigs-docs/Infrastructure/Authentik.md

@@ -267,6 +267,12 @@ The silent re-auth path was proven end-to-end: after deleting
 `qa-e2e-admin`'s rows, the e2e admin spec ran against the stale storage
 state and passed with no login prompt (2 tests, 2.9 s).
 
+## Login-session lifetime (how often you re-authenticate)
+
+How long a login stays valid is the **`default-authentication-login` stage's `session_duration`** (`UserLoginStage`) — not a global setting, and not `SESSION_COOKIE_AGE` (that is the *pre-login* session). Raised to **`days=7`** on 2026-10-20 (was stage value `0` = the 24 h framework default). It is an **absolute** window, never sliding: authentik implements `SESSION_SAVE_EVERY_REQUEST` nowhere, so activity does not extend it and no supported knob makes it one.
+
+**`session_duration` is a STRING** in authentik's idiom (`"days=7"`, `"seconds=30"`, `"hours=12"`). A Python `timedelta` stores `"7 days, 0:00:00"`, which `authentik.lib.utils.time.timedelta_from_string` rejects with `ValueError` and login then crashes computing the expiry — so set it with **`scripts/authentik-set-login-session-duration.sh`** (validates the form first). Any non-zero value applies to *every* login (the "remember me" box only adds `remember_me_offset` on top) and a positive `set_expiry` also overrides `SESSION_EXPIRE_AT_BROWSER_CLOSE`, so closing the browser no longer logs you out; **existing sessions keep their old expiry** — delete the user's `authentik_core_session`/`…proxysession` rows and re-login to force it now. This is **DB state, not repo-managed**: a blueprint re-import or rebuild reverts the stage to `0`, so re-run the script after one. The other two login stages (`default-source-*`) drive external-source/enrollment logins and stay at the framework default.
+
 ## Guard-chain flip recipe (adding a guarded vhost)
 
 Move a public vhost from `reverse_proxy <app>:<port>` onto the chain:

serverAppConfigs-docs/Infrastructure/Observability.md

@@ -3,7 +3,7 @@ type: reference
 title: "Observability on Dankclaw"
 description: "The observability stack: what runs, how it is deployed, how it is exposed, and what is still unconfigured."
 tags: [observability, prometheus, grafana, loki, alloy, node-exporter, quadlet, dankclaw]
-timestamp: 2026-09-11
+timestamp: 2026-10-20
 ---
 
 # Observability on Dankclaw
@@ -83,27 +83,37 @@ health endpoint (`/prometheus`: `/-/healthy`, `/alertmanager`: `/-/healthy`
 [[Observability Provisioning]] for what is provisioned as code and how to
 verify it through the APIs.
 
-## Current state (2026-10-18)
+## Current state (2026-10-20)
 
 * **Metrics: working.** `curl -s http://127.0.0.1:9090/api/v1/targets` reports
-  eight up targets; the four downs are structural (the `comfyui` and `services`
+  eight up targets — the three LLM endpoints (frank-archon on `vllm:*`,
-  HTML probes, and Gitea's disabled metrics endpoint), not outages. Rule set:
+  frank-dragoon and frank-zealot on `llamacpp:*`), grafana, loki, prometheus
-  4 alert rules + 2 recording rules (`/api/v1/rules`, matching
+  and both node exporters. The four downs are structural (the `comfyui` and
-  `shared/prometheus/rules/llm-fleet.yml`).
+  `services` HTML probes, and Gitea's disabled metrics endpoint), not outages.
+  Two rule groups in `shared/prometheus/rules/llm-fleet.yml`: `llm-fleet`
+  (2 recording + 4 alert, `llamacpp:*`) and `vllm-fleet` (1 recording + 4
+  alert, `vllm:*` — archon queueing, prefix collapse, preemption, p95
+  time-to-first-token), each keyed only to the namespace its runtime exports.
 * **Delivery: working.** Alertmanager's single `signal` receiver webhooks the
   relay, and `journalctl --user -u signal-relay` records successful
   `sent to +12062476336:` lines for every rule in the set.
-* **Dashboards: working.** One provisioned Prometheus datasource (uid
+* **Dashboards: working.** Provisioned datasources — Prometheus (uid
-  `prometheus`) and the `llm-fleet` dashboard, both installed from
+  `prometheus`) and Loki (uid `loki`) — and the `llm-fleet` dashboard, all
-  `shared/grafana/provisioning/`.
+  installed from `shared/grafana/provisioning/`. The Loki datasource is how
-* **Logs: configured, empty.** Alloy ships zero lines (journal counter 0, both
+  you read logs: Loki has no web UI of its own, so Grafana Explore is the
-  file globs refused) and Loki therefore holds zero streams — `/opt/loki/data`
+  human surface (the guarded `loki.franksdank.cloud` vhost serves only the API).
-  is a few hundred KB of WAL/compactor bookkeeping with **no chunk files**
+* **Logs: fixed in code, awaiting deploy.** The pipeline shipped nothing because
-  (8 files total by `find`, 2026-10-18) and `GET /loki/api/v1/labels` returns
+  a rootless Alloy cannot read the `0640` journal files, and the container-log
-  no label names.
+  globs point at a root-storage layout that does not exist here. The fix is built
-  See [[Alloy]] → Delivery state and [[Loki]] → Contents.
+  and tested in-repo (2026-10-20): Alloy is now a **root system service on
-* **Not configured:** no Loki datasource in Grafana, no blackbox exporter (the
+  `Network=host`**, reading `system.journal` + `user-*.journal` and pushing to
-  structural downs have no real uptime signal), no Alertmanager vhost.
+  Loki's `127.0.0.1:3100` loopback publish — a throwaway root probe read 31,825
+  lines and pushed 26,966 with zero drops. The **live** instance is still the old
+  rootless unit until `deploy.sh dankclaw --service alloy` runs. Detail in
+  [[Alloy]] → Why root; Loki's store state in [[Loki]] → Contents.
+* **Not configured:** no blackbox exporter (the structural downs have no real
+  uptime signal) and no Alertmanager vhost. The log pipeline's fix is in code but
+  not yet deployed (see *Logs* above).
 
 ## Citations
 

shared/alloy/config.alloy

@@ -1,22 +1,34 @@
 // Alloy configuration — ships host and container logs to Loki.
 // One instance per host. Deployed to /opt/alloy/config/config.alloy.
+//
+// Runs as a ROOT system service on Network=host (see
+// dankclaw/observability/quadlet/alloy.container), so it reaches Loki on the
+// host's loopback and reads every journal — host services and rootless-quadlet
+// containers alike.
 
 logging {
   level = "info"
 }
 
-// The alloy UI service listens on 127.0.0.1:12345 by default; the
+// The alloy UI listens on 127.0.0.1:12345; the in-container healthcheck probes
-// in-container healthcheck probes it there. alloy 1.x refuses to
+// it there. alloy 1.x refuses to configure the ui service from a config block,
-// configure the ui service from a config block, so there is no block.
+// so there is no ui block.
 
-// Ship logs to Loki (container name on the podman network).
+// Loki is published on the host's loopback (127.0.0.1:3100). Alloy shares the
+// host network, so it pushes by loopback, never by container name — a root
+// container is not on the rootless dankclaw_default network.
 loki.write "default" {
   endpoint {
-    url = "http://loki:3100/loki/api/v1/push"
+    url = "http://127.0.0.1:3100/loki/api/v1/push"
   }
 }
 
-// Systemd journal from the host socket.
+// The systemd journal. As root with /etc/machine-id + the journal dirs mounted,
+// this one source reads system.journal AND user-*.journal, so it carries both
+// host services and the rootless Quadlet containers' logs. There is deliberately
+// no container-file source: podman container stdout lands in the journal, and
+// the old /var/lib/containers/.../ctr.log and /var/log/containers globs pointed
+// at a root-storage layout that does not exist on this host.
 loki.source.journal "journal" {
   format_as_json = true
   labels = {
@@ -24,12 +36,3 @@ loki.source.journal "journal" {
   }
   forward_to = [loki.write.default.receiver]
 }
-
-// Container logs (podman storage path; adjust for docker).
-loki.source.file "containers" {
-  targets = [
-    {__path__ = "/var/lib/containers/storage/overlay-containers/*/userdata/ctr.log", job = "podman"},
-    {__path__ = "/var/log/containers/*.log", job = "containers"},
-  ]
-  forward_to = [loki.write.default.receiver]
-}

shared/grafana/provisioning/datasources/loki.yml

@@ -0,0 +1,17 @@
+# Grafana datasource for Loki, declared in git rather than clicked into
+# grafana.db — same reason as the Prometheus datasource: a UI-configured
+# datasource is invisible to review, lost with the data volume, and unknown
+# to the repository. Loki OSS ships no web UI of its own, so Explore through
+# this datasource is the only way a human reads the log store.
+#
+# URL is the container name on dankclaw_default (the same address Caddy
+# proxies to), not the published loopback port.
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    uid: loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false

shared/homepage/services.json

@@ -30,9 +30,9 @@
   },
   {
     "name": "open-notebook",
-    "category": "user",
+    "category": "admin",
     "section": "AI",
-    "description": "Podcast generator — notes in, spoken episode out: script written by frank-zealot, voices rendered by frank-tts. Guarded twice: Authentik at the edge, then its own login.",
+    "description": "Podcast generator — notes in, spoken episode out: script written by frank-archon, voices rendered by frank-tts. Guarded twice: Authentik at the edge, then its own login. Single shared password, no per-user accounts — admin-only.",
     "url": "http://notebook.franksdank.cloud",
     "docs": "",
     "probe": "127.0.0.1:8502",

shared/prometheus/rules/llm-fleet.yml

@@ -1,10 +1,15 @@
 # Alerting and recording rules for the LLM fleet.
 #
-# One metric namespace: llamacpp:* — frank-zealot and frank-dragoon, both
+# Two metric namespaces, because two runtime families are scraped:
-# llama.cpp with /metrics (the family verified from /metrics on 2026-09-16
+#
-# and 2026-09-20). frank-archon LEFT the family 2026-10-05: FreeToken (the
+#   * llamacpp:* — frank-zealot (llama.cpp) and frank-dragoon (halogen, which
-# PR587 ROCm engine) has no /metrics endpoint, so its scrape job was
+#     re-exports the llama.cpp metric names). Verified from /metrics on
-# deleted rather than left never-green.
+#     2026-09-16 / 2026-09-20.
+#   * vllm:* — frank-archon, the radiance engine on Dankserver since
+#     2026-10-05. It exports vLLM-compatible names, a DIFFERENT namespace, so
+#     it is a separate scrape job in prometheus.yml and the rules below are a
+#     separate group. (The old claim that its scrape job was deleted is false:
+#     it re-joined the day it left the FreeToken engine and is `up` today.)
 # (The netavark published-port quirk that forces the Dankclaw endpoints
 # onto the gateway is same-host only.)
 #
@@ -22,7 +27,8 @@
 #     cached by cached + prompt.
 #   * Speculative decoding: drafts and accepted tokens are both counters,
 #     so LlmSpecDecodeOff watches the pair.
-#   * No latency histograms exist, so there is no p95 alert.
+#   * No latency histograms in llama.cpp, so no p95 alert there; radiance's
+#     vllm:* namespace DOES export one, so the vllm-fleet group alerts on it.
 #
 # IMPORTANT: delivery. Since 2026-09-17 a firing alert reaches Frank on
 # Signal: Prometheus pushes to Alertmanager (dankclaw/observability/quadlet/
@@ -112,6 +118,84 @@ groups:
           description: >-
             Prometheus cannot reach the LLM endpoint. frank-zealot and
             frank-dragoon are root units on Dankclaw: `systemctl is-active
-            frank-zealot` / `frank-dragoon`. Both scraped endpoints export
+            frank-zealot` / `frank-dragoon` (both export llamacpp:* —
-            the llamacpp:* family. (frank-archon left the scrape set
+            dragoon via halogen's llama.cpp-compatible names). frank-archon
-            2026-10-05 — FreeToken exports no /metrics.)
+            is a radiance endpoint on Dankserver (`systemctl show
+            frank-archon` there) exporting the vllm:* family; the
+            `frank-.*` selector covers all three regardless of namespace.
+
+  # frank-archon (radiance) exports the vLLM-compatible namespace, NOT
+  # llamacpp:*. Every rule here is keyed to a series the exporter actually
+  # emits (verified from /metrics 2026-10-20: num_requests_waiting,
+  # gpu_prefix_cache_hit_rate, num_preemptions_total, prompt_tokens_total,
+  # time_to_first_token_seconds). The llamacpp rules above evaluate nothing for
+  # this job and these evaluate nothing for llama.cpp/halogen — two clean
+  # namespaces, each keyed only to what its runtime exports.
+  - name: vllm-fleet
+    interval: 30s
+    rules:
+      # p95 time-to-first-token, shared with the dashboard. When the endpoint
+      # is idle every bucket rate is zero, histogram_quantile yields NaN, and
+      # neither this nor the alert below can fire on an idle engine.
+      - record: job:vllm_ttft_p95:rate5m
+        expr: |
+          histogram_quantile(0.95, sum by (job, le)
+              (rate(vllm:time_to_first_token_seconds_bucket[5m])))
+
+      # Radiance exports a real queue-depth gauge (llama.cpp only shows the
+      # wall as requests_deferred). --max-num-seqs is small, so a sustained
+      # wait is either real load or this endpoint carrying the fleet alone.
+      - alert: VllmRequestsQueueing
+        expr: vllm:num_requests_waiting > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.job }}: requests waiting for 10 minutes"
+          description: >-
+            The radiance scheduler has requests queued. Either the fleet is
+            genuinely busy or this endpoint is carrying it alone.
+
+      # Radiance reports prefix reuse directly as a hit-rate gauge — no
+      # counter division needed. The token-rate floor keeps an idle engine
+      # (hit rate 0 with no traffic) from firing.
+      - alert: VllmPrefixCacheCollapse
+        expr: >-
+          vllm:gpu_prefix_cache_hit_rate < 0.20
+            and rate(vllm:prompt_tokens_total[5m]) > 0.05
+        for: 15m
+        labels:
+          severity: info
+        annotations:
+          summary: "{{ $labels.job }}: prefix hit rate below 20 % for 15 minutes"
+          description: >-
+            Nearly every request prefilling from scratch. With several large
+            contexts in flight this is the capacity wall, not a routing fault.
+
+      # Preemption is the KV-thrash signature radiance reports that llama.cpp
+      # cannot: the engine evicts a running request to admit another. Any
+      # sustained rate means the KV cache cannot hold the concurrency in flight.
+      - alert: VllmPreempting
+        expr: rate(vllm:num_preemptions_total[5m]) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.job }}: engine preempting requests"
+          description: >-
+            The engine is evicting running requests to admit new ones — the KV
+            cache is undersized for the concurrency in flight.
+
+      # p95 first-token latency, kept HIGH on purpose: a large-prompt prefill
+      # legitimately spends many seconds before token 1, so this fires only on
+      # a wedged scheduler or a queue backed up behind one request.
+      - alert: VllmTtftSlow
+        expr: job:vllm_ttft_p95:rate5m > 120
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.job }}: p95 time-to-first-token over 120 s"
+          description: >-
+            First-token latency is pathological, not an honest big prefill.
+            Likely a wedged scheduler or one request stalling the queue.

tests/caddyfile.bats

@@ -240,14 +240,12 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
 }
 
 @test "alloy's UI address comes from the unit, not the config file" {
-    # alloy 1.x refuses to configure the ui service from a config block
+    # alloy 1.x refuses to configure the ui service from a config block, so the
-    # ("service ui does not support being configured"), so the unit moves the
+    # unit sets the UI address on the command line. As a Network=host root unit
-    # UI off the container's loopback with a command-line flag. Without that
+    # that address is the host loopback directly (no PublishPort) — reachable
-    # flag the port cannot be published and the service has no reachable
+    # from the host, from nothing off-host.
-    # health endpoint at all — which is why the homepage status card for
-    # alloy read "down" while it ran under compose.
     ! grep -qE '^(ui|server) \{' "$REPO_ROOT/shared/alloy/config.alloy"
     local unit="$REPO_ROOT/dankclaw/observability/quadlet/alloy.container"
-    grep -q -- '--server.http.listen-addr=0.0.0.0:12345' "$unit"
+    grep -q -- '--server.http.listen-addr=127.0.0.1:12345' "$unit"
-    grep -q '^PublishPort=127.0.0.1:12345:12345$' "$unit"
+    ! grep -q '^PublishPort=' "$unit"
 }

tests/observability_quadlet.bats

@@ -41,13 +41,17 @@ service_section() {
 }
 
 @test "every observability unit is boot-persistent" {
-    # Generated units cannot be enabled on this host's podman 5.x v1 quadlet
+    # Generated units cannot be enabled on this host's podman quadlet generator;
-    # generator; [Install] + linger is the entire boot-persistence story.
+    # [Install] + linger is the boot-persistence story for the rootless five.
+    # Alloy is a ROOT system unit, so its [Install] targets multi-user.target.
     local svc
     for svc in "${UNITS[@]}"; do
+        [[ "$svc" == alloy ]] && continue
         grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \
             || { echo "$svc is not boot-persistent"; return 1; }
     done
+    grep -q '^WantedBy=multi-user.target$' "$UNIT_DIR/alloy.container" \
+        || { echo "alloy (root unit) is not boot-persistent"; return 1; }
 }
 
 @test "every observability unit is self-healing" {
@@ -140,15 +144,20 @@ service_section() {
     grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile"
     grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile"
     local svc
-    for svc in prometheus grafana loki alloy; do
+    for svc in prometheus grafana loki; do
         grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \
             || { echo "$svc unit renames its container"; return 1; }
         grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \
             || { echo "$svc is not on the network Caddy and its peers use"; return 1; }
     done
+    # Alloy is not on that network: it is a root container on Network=host, the
+    # only way to reach Loki's 127.0.0.1:3100 publish from outside a bridge.
+    grep -q '^ContainerName=alloy$' "$UNIT_DIR/alloy.container"
+    grep -q '^Network=host$' "$UNIT_DIR/alloy.container" \
+        || { echo "alloy must be Network=host to reach Loki on the host loopback"; return 1; }
     grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
     grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
-    grep -q 'url = "http://loki:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
+    grep -q 'url = "http://127.0.0.1:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
 }
 
 @test "every observability unit has a health check" {
@@ -239,12 +248,14 @@ service_section() {
 }
 
 @test "startup order follows the data dependencies" {
-    # Grafana is useless without Prometheus and Alloy drops logs without Loki;
+    # Grafana is useless without Prometheus (both rootless, same manager). Alloy
-    # both pairs are expressed as systemd ordering, not launch luck.
+    # used to order on loki.service; as a ROOT unit it lives on the SYSTEM
+    # manager and cannot order against a rootless user unit, so it depends only
+    # on the network and retries its push until Loki answers.
     grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
     grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
-    grep -q 'After=network-online.target loki.service' "$UNIT_DIR/alloy.container"
+    grep -q '^After=network-online.target$' "$UNIT_DIR/alloy.container"
-    grep -q 'Wants=network-online.target loki.service' "$UNIT_DIR/alloy.container"
+    ! grep -q 'loki.service' "$UNIT_DIR/alloy.container"
 }
 
 @test "the deployer installs, restarts and health-checks each unit" {
@@ -281,6 +292,24 @@ PAIRS
     grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY"
 }
 
+@test "alloy is a root host-network unit that reads every journal" {
+    # The whole point of the root cutover: a rootless container cannot open the
+    # 0640 journal files. Pin the properties that make the pipeline carry data.
+    local u="$UNIT_DIR/alloy.container"
+    grep -q '^Network=host$' "$u"
+    grep -q '^Volume=/etc/machine-id:/etc/machine-id:ro$' "$u" \
+        || { echo "alloy missing the load-bearing machine-id mount"; return 1; }
+    grep -q '^Volume=/var/log/journal:/var/log/journal:ro$' "$u"
+    grep -q '^Volume=/run/log/journal:/run/log/journal:ro$' "$u"
+    grep -q -- '--server.http.listen-addr=127.0.0.1:12345' "$u"
+    # Deployer: system install + root image mirror, not the user quadlet loop.
+    grep -q '^deploy_alloy() {' "$REMOTE_DEPLOY"
+    grep -q 'install_system_quadlet dankclaw/observability/quadlet alloy' "$REMOTE_DEPLOY"
+    grep -q '/etc/containers/systemd/' "$REMOTE_DEPLOY"
+    grep -q 'mirror_image_to_root localhost/franksdank/alloy:latest' "$REMOTE_DEPLOY"
+    grep -q 'alloy) deploy_alloy ;;' "$REMOTE_DEPLOY"
+}
+
 @test "no dankclaw compose file competes with the quadlet units" {
     # Two owners for one container name is how a compose run resurrects a
     # stale duplicate next to the quadlet-managed one. This file was retired on
@@ -313,33 +342,33 @@ PAIRS
     done
 }
 
-@test "prometheus scrapes every LLM endpoint that exports metrics" {
+@test "prometheus scrapes every LLM endpoint, in its real metric namespace" {
-    # The LLM endpoints are the services whose behaviour this fleet argues about
+    # Every LLM endpoint that exports /metrics must have a scrape job. As of
-    # most (KV pressure, prefix-cache reuse, queue depth). Both Dankclaw
+    # 2026-10-05 there are TWO runtime namespaces scraped:
-    # endpoints export the llamacpp:* family. frank-archon LEFT the set
+    #   * llamacpp:* — frank-zealot (llama.cpp) and frank-dragoon (halogen,
-    # 2026-10-05: the FreeToken PR587 ROCm engine has NO /metrics endpoint —
+    #     re-exporting the llama.cpp names), both root host processes on
-    # scraping it was a structural never-green (404), so the job was deleted
+    #     Dankclaw reached via host.containers.internal — this Prometheus is a
-    # rather than left vacuous (the Lucebox DS4.1 precedent, 2026-09-26).
+    #     bridge container and a host-process bind is refused at the LAN IP,
-    # The two Dankclaw endpoints are HOST processes reached through
+    #     which is why frank-zealot read red by construction for days.
-    # host.containers.internal: this Prometheus is a bridge-networked
+    #   * vllm:* — frank-archon (radiance) on Dankserver, a different machine,
-    # container, and on this netavark host a container can only reach the
+    #     so plain LAN routing reaches 192.168.0.235:7770. It RE-JOINED the
-    # host's LAN IP at a PUBLISHED port — a host-process bind like
+    #     scrape set with radiance; the old "job deleted, exports no /metrics"
-    # llama-server's 0.0.0.0:7770 is refused at 192.168.0.143:7770 while it
+    #     note belonged to the retired FreeToken engine and is false today, so
-    # answers at 169.254.1.2:7770 (why frank-zealot stayed red by construction
+    #     this seam now asserts archon IS scraped (a bare `! grep` here was
-    # for days until the 2026-09-20 /targets audit).
+    #     set -e-exempt and never enforced the opposite).
     local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
     grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-zealot not scraped"; return 1; }
     grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-dragoon not scraped"; return 1; }
-    ! grep -q '"192.168.0.235:7770"' "$cfg"
+    grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-archon not scraped"; return 1; }
-    grep -q '"host.containers.internal:7770,192.168.0.235:7770"' "$cfg" \
+    for job in frank-zealot frank-dragoon frank-archon; do
-        && { echo "frank-archon folded into a Dankclaw gateway target"; return 1; }
-    for job in frank-zealot frank-dragoon; do
         grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; }
     done
-    ! grep -q 'job_name: "frank-archon"' "$cfg"
+    # No LLM endpoint is reached through a guarded vhost: Prometheus cannot
-    # None of them is reached through a guarded vhost: Prometheus cannot solve
+    # solve an Anubis challenge, so a franksdank.cloud target is red by
-    # an Anubis challenge, so a franksdank.cloud target is red by construction.
+    # construction.
-    ! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg"
+    grep -qE '^[[:space:]]*-[[:space:]]*"?https?://[^"]*franksdank' "$cfg" \
+        && { echo "a scrape target points at a guarded vhost"; return 1; }
+    true
 }
 
 @test "grafana provisioning is declared in git and installed by the deployer" {
@@ -378,28 +407,44 @@ names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rule
 for want in ('job:llm_prefix_cache_reuse:rate5m',
              'job:llm_spec_accept_length:rate5m',
              'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
-             'LlmSpecDecodeOff', 'LlmEndpointDown'):
+             'LlmSpecDecodeOff', 'LlmEndpointDown',
+             'job:vllm_ttft_p95:rate5m', 'VllmRequestsQueueing',
+             'VllmPrefixCacheCollapse', 'VllmPreempting', 'VllmTtftSlow'):
     assert want in names, want
 # Every rule must be keyed to a series that actually exists. A rule naming a
 # series nobody exports evaluates nothing forever and errors never — the
-# failure this seam exists for. Since the 2026-09-26 cutover the only scraped
+# failure this seam exists for. Two runtimes are scraped in two namespaces:
-# LLM runtime is llama.cpp (frank-zealot, frank-dragoon); the vllm: family left
+# llama.cpp/halogen (llamacpp:*) for frank-zealot and frank-dragoon, and
-# with frank-medium's radiance runtime and must not survive in any expr.
+# radiance (vllm:*) for frank-archon. Both families were read straight off the
+# endpoints' /metrics; anything else is a guess.
 exprs = [r.get('expr', '') for g in d['groups'] for r in g['rules']]
-# llama.cpp families measured from /metrics on 2026-09-16 and 2026-09-20
+live = {
-# (frank-zealot, frank-dragoon). Anything else is a guess.
+    # llama.cpp / halogen, measured from /metrics 2026-09-16 and 2026-09-20.
-live = {'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
+    'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
-        'llamacpp:requests_deferred', 'llamacpp:requests_processing',
+    'llamacpp:requests_deferred', 'llamacpp:requests_processing',
-        'llamacpp:spec_decode_num_drafts_total',
+    'llamacpp:spec_decode_num_drafts_total',
-        'llamacpp:spec_decode_num_accepted_tokens_total',
+    'llamacpp:spec_decode_num_accepted_tokens_total',
-        'llamacpp:spec_decode_num_draft_tokens_total',
+    'llamacpp:spec_decode_num_draft_tokens_total',
-        'llamacpp:tokens_predicted_total',
+    'llamacpp:tokens_predicted_total',
-        'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
+    'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
-        'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode'}
+    'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode',
+    # radiance (frank-archon), measured from /metrics 2026-10-20.
+    'vllm:num_requests_waiting', 'vllm:gpu_prefix_cache_hit_rate',
+    'vllm:num_preemptions_total', 'vllm:prompt_tokens_total',
+    'vllm:time_to_first_token_seconds_bucket',
+}
 used = {m for e in exprs
         for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', e)}
 assert used <= live, used - live
-assert not any(':vllm' in e for e in exprs), 'vllm: series in rules but no vllm endpoint is scraped'
+# The two namespaces must not bleed across groups: a llamacpp: selector
+# evaluates nothing for the radiance job and vice versa, so each group may
+# reference only its own family.
+per = {g['name']: {m.split(':')[0] for r in g['rules']
+                   for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', r.get('expr', ''))}
+       for g in d['groups']}
+for name, fams in per.items():
+    allowed = {'llamacpp'} if name == 'llm-fleet' else {'vllm'}
+    assert fams <= allowed, (name, fams)
 print('rules ok:', len(names))
 "
     # A rule file that nothing loads evaluates nothing, and Prometheus reports

tests/open_notebook.bats

@@ -290,7 +290,9 @@ vhost = sys.argv[2]
 cards = [s for s in data if s["name"] == "open-notebook"]
 assert len(cards) == 1, f"expected exactly one open-notebook card, got {len(cards)}"
 card = cards[0]
-assert card["category"] == "user" and card["section"] == "AI", card
+# category moved to "admin": single shared password, no per-user accounts, so it
+# is positioned as an admin-side tool, not a user-facing app (see the vault page).
+assert card["category"] == "admin" and card["section"] == "AI", card
 assert card["probe"] == "127.0.0.1:8502", card["probe"]
 assert card["method"] == "http"
 # Deployed and guarded: the card is live and its link is the vhost, so the