Merge pull request 'Observability: vllm rule group for frank-archon + Loki datasource; correct stale log-pipeline facts' (#52 ) from fix/observability-vllm-loki into main

2026-10-20 19:39:38 -07:00
13 changed files with 195 additions and 468 deletions
@@ -1,51 +1,55 @@
-# Quadlet unit: alloy (ROOT system service, boot-persistent).
+# Quadlet unit: alloy (rootless, boot-persistent).
 #
-# Install path on dankclaw: /etc/containers/systemd/alloy.container
+# Install path on dankclaw: ~/.config/containers/systemd/alloy.container
-# Unit name: alloy.service (container name: alloy) — SYSTEM manager, root.
+# Unit name: alloy.service (container name: alloy)
 #
-# WHY ROOT (not rootless, unlike its observability siblings). Alloy reads the
+# Log shipper. This replaces the compose service of the same name in
-# host journal. The journal files are root:systemd-journal 0640 (other::---);
+# dankclaw/compose.yaml ("Observability" block).
 # a rootless container carries no supplementary groups and cannot open them —
 # a rootless read of system.journal returns DENIED (see the Alloy page in the
 # vault). Running as root, one loki.source.journal reads system.journal AND
 # user-1000.journal, so it carries both host logs and the rootless Quadlet
 # containers' logs. Validated 2026-10-20: a root probe read 31,825 journal
 # lines and pushed 26,966 into Loki in 20 s, zero drops.
 #
-# NETWORK=host. Loki publishes 127.0.0.1:3100 (loopback only) on the host, and
+# IMAGE. franksdank/alloy is built locally by deploy-remote.sh
-# no bridge container — rootless or root — can reach the host's loopback, so
+# (build_compose_image) from dankclaw/alloy/Containerfile: upstream
-# this unit runs on the host network and pushes to 127.0.0.1:3100. The side
+# grafana/alloy plus a static busybox for the healthcheck. Pull=never keeps a
-# effect: the UI must bind loopback too, so Exec sets the listen-addr to
+# boot from reaching for that local tag at a registry.
 # 127.0.0.1 (the old rootless unit used 0.0.0.0 only because container loopback
 # was not host loopback; under Network=host they are the same address, so the
 # unauthenticated UI stays loopback-only — nothing off-host reaches it).
 #
-# THE /etc/machine-id MOUNT IS LOAD-BEARING. sd_journal locates the journal
+# NETWORK. It joins dankclaw_default so its config's loki.write endpoint
-# directory by the local machine id; without it the container opens nothing and
+# resolves the container name loki:3100.
 # the journal source reads exactly 0 lines — silently, no error, health green.
 # This mount is the entire reason the pipeline carries data. Never remove it.
 #
-# IMAGE. Built rootless by build_compose_image alloy, then mirrored into the
+# THE UI PORT EXISTS ON PURPOSE. Alloy's UI listens on the container's own
-# root image store by the deployer (a root build runs in the build container's
+# 127.0.0.1 by default and cannot be moved from the config file (alloy 1.x
-# own resolver and fails on DNS here). Pull=never: boot must not reach for the
+# refuses the ui block), so the compose service had no reachable health
-# local tag at a registry.
+# endpoint at all: the homepage status card for alloy reported "down"
 # continuously and there was nothing for a deploy to wait on. The command
 # below therefore moves the UI to the container's wildcard address and the
 # port is published on host loopback only, which puts alloy in the same class
 # as the other three: reachable from the host, by Caddy over the private
 # network, and by nothing off-host. Alloy has no authentication, which is
 # exactly why it is not published wider.
 #
 # SOURCES. The host journald socket and the host log/container-storage trees
 # are bind-mounted read-only; Alloy tails them and pushes to Loki. The config
 # lives at /opt/alloy/config/config.alloy, installed from
 # shared/alloy/config.alloy on every deploy.
 #
 # ORDERING. Wants/After loki.service: without a reachable Loki, Alloy's write
 # client fails and its log pipeline backs up.
 [Unit]
 Description=Alloy log shipper (Dankclaw)
-After=network-online.target
+After=network-online.target loki.service
 Wants=network-online.target loki.service
 [Container]
 ContainerName=alloy
 Image=localhost/franksdank/alloy:latest
 Pull=never
-Network=host
+Network=dankclaw_default
-Exec=run --server.http.listen-addr=127.0.0.1:12345 /etc/alloy/config.alloy
+Exec=run --server.http.listen-addr=0.0.0.0:12345 /etc/alloy/config.alloy
-Volume=/opt/alloy/config:/etc/alloy:ro
+PublishPort=127.0.0.1:12345:12345
-Volume=/etc/machine-id:/etc/machine-id:ro
+Volume=/opt/alloy/config:/etc/alloy
 Volume=/run/systemd/journal/socket:/run/systemd/journal/socket:ro
-Volume=/var/log/journal:/var/log/journal:ro
+Volume=/var/log:/var/log:ro
-Volume=/run/log/journal:/run/log/journal:ro
+Volume=/var/lib/containers:/var/lib/containers:ro
-HealthCmd=/usr/local/bin/busybox wget --spider -q http://127.0.0.1:12345/-/healthy
+HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:12345/-/healthy
 HealthInterval=30s
 HealthTimeout=10s
 HealthRetries=3
@@ -56,4 +60,4 @@ Restart=always
 RestartSec=10
 [Install]
-WantedBy=multi-user.target
+WantedBy=default.target
@@ -47,24 +47,9 @@ Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${GRAFANA_OAUTH_CLIENT_SECRET}
 # client_id query parameter — /application/o/<slug>/authorize/ does not exist.
 Environment=GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://auth.franksdank.cloud/application/o/authorize/
 Environment=GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://auth.franksdank.cloud/application/o/token/
-# Standard OIDC userinfo. Grafana v13 fetches <api_url>/emails for the address
+# ?api=1 = introspection-style userinfo: returns custom claims (groups) too.
-# ONLY when the token carries no `email` claim (authentik has no /emails path and
+Environment=GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.franksdank.cloud/application/o/userinfo/?api=1
-# 404s it, failing the login) — so the claim must actually be there: the fix is
+Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=false
 # the grafana-email-scope mapping (see scripts/authentik-grafana-sso-fix.sh),
 # not a Grafana flag. Do NOT append ?api=1 to this URL: Grafana then GETs
 # <...>/userinfo/?api=1/emails, a 403.
 Environment=GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.franksdank.cloud/application/o/userinfo/
 # authentik's scopes are openid/email/profile. `user:email` (a GitLab-ism) is
 # not a known scope here; requesting it makes the authorize view crash.
 # Quoted: quadlet word-splits Environment=, so an unquoted spaced list
 # truncates SCOPES to its first token.
 Environment="GF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile"
 Environment=GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email
 Environment=GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=sub
 Environment=GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=preferred_username
 # First SSO login auto-provisions the user. Safe: unauthenticated traffic never
 # reaches this callback — the Anubis -> Authentik guard fronts every request.
 Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true
 Environment=GF_AUTH_GENERIC_OAUTH_AUTO_UPSERT=true
 Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_ORG=true
 Environment=GF_AUTH_GENERIC_OAUTH_ORG_ID=1
@@ -1,81 +0,0 @@
 #!/usr/bin/env bash
 # Repair the Grafana OAuth2 relying-party SSO on Dankclaw's authentik.
 #
 # grafana.franksdank.cloud logs in through authentik as an OAuth2 relying party.
 # Two DB-side facts (the provider was created by hand via `ak shell`, never
 # repo-managed) silently broke that login, and BOTH had previously been
 # mis-recorded here as a single "the property-mapping evaluator is broken, so
 # userinfo returns only `sub`" myth. They are neither, and the evaluator is fine:
 #
 #   1. provider.authorization_flow was NULL. The authorize view builds a flow
 #      planner from it and crashes with
 #      `AttributeError: 'NoneType' object has no attribute 'slug'` — an HTTP 500
 #      "Server Error — Powered by authentik", before any claim is touched.
 #      Fix: point it at default-provider-authorization-implicit-consent, the
 #      flow every other provider in this instance already uses.
 #   2. The `grafana-email-scope` ScopeMapping's expression was the bare string
 #      `user.email`. authentik merges a scope mapping's result into the token as
 #      a MAPPING: a returned dict adds its keys as claims; a returned string is
 #      keyed under the mapping's NAME ("grafana-email-scope"), so no `email`
 #      claim is ever emitted. Grafana, finding no email, falls back to
 #      GET <api_url>/emails, authentik 404s that path, and the login dies with
 #      "Login failed: InternalError". Fix: return
 #      {"email": user.email, "email_verified": True} — the shape the stock
 #      OpenID 'email' mapping uses. The dict-valued `grafana-profile-scope`
 #      mapping has always worked, which is the proof the evaluator is fine.
 #
 # This is authentik DB state. It is NOT in any deployable file: a blueprint
 # re-import, or an ak-shell re-creation of the grafana provider, reverts BOTH
 # facts and the symptom returns verbatim. Re-run this after any such rebuild.
 # Every step is idempotent.
 #
 # Grafana's own half — SCOPES="openid email profile", a standard userinfo API_URL
 # with no `?api=1`, and ALLOW_SIGN_UP=true — lives in
 # dankclaw/observability/quadlet/grafana.container and deploys normally.
 #
 # Usage: authentik-grafana-sso-fix.sh          (override host: AUTHENTIK_HOST=…)
 set -euo pipefail
 HOST="${AUTHENTIK_HOST:-dankclaw}"
 ssh "$HOST" 'podman exec -i authentik-server ak shell' <<'PYEOF'
 from authentik.flows.models import Flow
 from authentik.providers.oauth2.models import OAuth2Provider
 p = OAuth2Provider.objects.filter(name="grafana").first()
 if not p:
    raise SystemExit("no OAuth2Provider named 'grafana'")
 # 1. The authorization flow every other provider uses.
 want = Flow.objects.filter(slug="default-provider-authorization-implicit-consent").first()
 if not want:
    raise SystemExit("default-provider-authorization-implicit-consent flow is missing")
 if p.authorization_flow_id != want.pk:
    p.authorization_flow = want
    p.save()
    print(f"set grafana.authorization_flow -> {want.slug}")
 else:
    print(f"grafana.authorization_flow already {want.slug}")
 # 2. The email scope mapping must RETURN A DICT, not a bare string.
 m = p.property_mappings.filter(name="grafana-email-scope").first()
 if not m:
    raise SystemExit("grafana-email-scope mapping is missing")
 fixed = 'return {"email": user.email, "email_verified": True}'
 if m.expression.strip() != fixed:
    m.expression = fixed
    m.save()
    print("repaired grafana-email-scope -> returns {email, email_verified}")
 else:
    print("grafana-email-scope expression already correct")
 # 3. The id_token must carry the scope claims; Grafana reads identity from it.
 if not p.include_claims_in_id_token:
    p.include_claims_in_id_token = True
    p.save()
    print("enabled include_claims_in_id_token")
 else:
    print("include_claims_in_id_token already on")
 PYEOF
 echo "Done. Verify: sign in at https://grafana.franksdank.cloud via Authentik;"
 echo "the round trip must land on / and /api/user must answer 200 with an email."
@@ -487,15 +487,13 @@ seed_grafana_env() {
        || { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; }
 }
-# The observability containers left compose on 2026-09-11 (they were started by
+# The five observability containers left compose on 2026-09-11 (they were
-# hand with podman-compose and never came back after a reboot). Five are ordinary
+# started by hand with podman-compose and never came back after a reboot).
-# rootless Quadlet units: same images, same mounts, same arguments, same
+# They are ordinary rootless Quadlet units now: same images, same mounts, same
-# container names, so nothing that addresses them by name — Caddy, Prometheus'
+# arguments, same container names, so nothing that addresses them by name —
-# own scrape config — changed. Only the published ports did: the three HTTP UIs
+# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
-# bind 127.0.0.1, because the compose services published on 0.0.0.0 and none of
+# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
-# them authenticates. Alloy is the exception: a ROOT system service on
+# compose services published on 0.0.0.0 and none of them authenticates.
 # Network=host, because reading the host journal needs privileges a rootless
 # container cannot get. It is handled by deploy_alloy, not the loop below.
 deploy_observability() {
    local service="$1"
    # Observability units live under <target>/observability/quadlet: Dankclaw
@@ -514,6 +512,10 @@ deploy_observability() {
            copy_loki_config
            build_compose_image loki
            ;;
        alloy)
            copy_alloy_config
            build_compose_image alloy
            ;;
        grafana)
            copy_grafana_provisioning
            seed_grafana_env || return 1
@@ -532,71 +534,11 @@ deploy_observability() {
        alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
        grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
        loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
        alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
        node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
    esac
 }
 # Mirror an image the deployer built rootless into the ROOT image store. A root
 # quadlet reads root storage; a root build (build_compose_image is rootless) is
 # not possible on this host — the build container's resolver fails DNS — so the
 # only way to get a root-readable copy is save-and-load. Compares image ids so a
 # rebuilt alloy actually re-lands in root storage instead of leaving a stale tag.
 mirror_image_to_root() {
    local ref="$1" rid
    rid="$(podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" || {
        echo "ERROR: rootless image $ref is not built; cannot mirror to root" >&2
        return 1
    }
    if [[ "$(run_privileged podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" == "$rid" ]]; then
        return 0
    fi
    local tar; tar="$(mktemp /var/tmp/alloy-mirror.XXXXXX.tar)"
    podman save -o "$tar" "$ref"
    run_privileged podman load -i "$tar"
    rm -f "$tar"
 }
 # Install a quadlet into the SYSTEM generator dir (root manager) instead of the
 # user dir. Boot persistence is the unit's own [Install] section (multi-user
 # target); the v1 system generator creates the .wants on daemon-reload, so —
 # like the user units — no `systemctl enable` (generated units cannot be enabled
 # here).
 install_system_quadlet() {
    local source_dir="$1" unit="$2"
    run_privileged mkdir -p /etc/containers/systemd
    run_privileged cp "$source_dir/$unit.container" /etc/containers/systemd/
    run_privileged systemctl daemon-reload
 }
 # Alloy as a ROOT system service on Network=host. See the unit file: a rootless
 # container cannot read the 0640 journal files, and it cannot reach Loki's
 # 127.0.0.1:3100 publish from a bridge netns. Root + host network fixes both;
 # the price is that this one container runs as root and on the host network.
 deploy_alloy() {
    copy_alloy_config
    build_compose_image alloy                      # rootless build (resolver works)
    mirror_image_to_root localhost/franksdank/alloy:latest
    # Retire the rootless unit BEFORE the system one starts: both managers
    # generate alloy.service, and two containers named "alloy" both binding
    # 127.0.0.1:12345 cannot coexist. Move the user quadlet out of the generator
    # dir (it regenerates from there) and stop the user unit.
    if [[ -f "$quadlet_unit_dir/alloy.container" ]] ||
       systemctl --user is-active --quiet alloy.service 2>/dev/null; then
        systemctl --user stop alloy.service 2>/dev/null || true
        if [[ -f "$quadlet_unit_dir/alloy.container" ]]; then
            local trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-alloy-root-cutover"
            mkdir -p "$trash"
            mv -- "$quadlet_unit_dir/alloy.container" "$trash/alloy.container"
            echo "retired rootless alloy.container -> $trash"
        fi
        systemctl --user daemon-reload
    fi
    install_system_quadlet dankclaw/observability/quadlet alloy
    run_privileged systemctl restart alloy.service
    wait_for_health alloy http://127.0.0.1:12345/-/healthy 60
 }
 build_compose_image() {
    local service="$1" image cfhash oldhash containerfile
    # Every file the image COPIES IN, beside the Containerfile. The skip below
@@ -1999,8 +1941,7 @@ deploy_service() {
        copyparty) deploy_copyparty ;;
        plex) deploy_plex ;;
        tautulli) deploy_tautulli ;;
-        alloy) deploy_alloy ;;
+        prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
        prometheus|grafana|loki|node-exporter|alertmanager) deploy_observability "$service" ;;
        comfyui) deploy_comfyui ;;
        *) deploy_compose_service "$service" ;;
    esac
@@ -3,111 +3,96 @@ type: reference
 description: "Alloy is Grafana's log and telemetry shipper."
 tags:
  - infrastructure
-timestamp: 2026-10-20
+timestamp: 2026-09-11
 ---
 # Alloy
-Alloy is Grafana's log and telemetry shipper. It replaces Promtail (end of life
+Alloy is Grafana's log and telemetry shipper. It replaces Promtail (end of life as of 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/). It is wired to forward container and journal logs to Loki, and every unit in the path is healthy — but as of 2026-10-18 it is delivering **zero** log lines. See "Delivery state" below.
 as of 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/). It
 forwards the host journal — including the rootless Quadlet containers' logs —
 to Loki. Unlike its observability siblings it runs as a **root system service on
 the host network**, because a rootless container cannot read the journal (see
 *Why root* below).
-**Host:** Dankclaw (one instance per host) — **UI/health:** 127.0.0.1:12345
+**Host:** Dankclaw (one instance per host) — **Port:** 12345 (loopback) — **Docs:** https://grafana.com/docs/alloy/latest
 (loopback) — **Docs:** https://grafana.com/docs/alloy/latest
 ## Facts
 | Item | Value |
 |------|-------|
-| Image | `localhost/franksdank/alloy:latest` — built rootless, then mirrored into the root image store by the deployer (a root build fails on DNS in this host's build container) |
+| Image | `franksdank/alloy:latest` (local build: upstream + a static busybox for the healthcheck) |
-| Unit | `dankclaw/observability/quadlet/alloy.container` → system `alloy.service`, installed at `/etc/containers/systemd/` (root manager) |
+| Published | `127.0.0.1:12345` → 12345 (its own UI/health port) |
-| Config | `shared/alloy/config.alloy` → `/opt/alloy/config/config.alloy` on every deploy |
+| Unit | `dankclaw/observability/quadlet/alloy.container` → `alloy.service` |
-| Network | `host` — required so it can reach Loki's `127.0.0.1:3100` publish |
+| Config | `shared/alloy/config.alloy`, installed to `/opt/alloy/config/config.alloy` on every deploy (`scripts/deploy-remote.sh:281-284`) |
-| UI | `--server.http.listen-addr=127.0.0.1:12345` (loopback; nothing off-host reaches it) |
+| Inputs | host journald socket, podman container logs, `/var/log` |
-| Inputs | host journal: `system.journal` + `user-*.journal` (root reads both) |
+| Output | `http://loki:3100/loki/api/v1/push` (`shared/alloy/config.alloy:13-17`) |
-| Output | `http://127.0.0.1:3100/loki/api/v1/push` |
+| Successor to | Promtail (EOL 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/) |
-| Boot persistence | `Restart=always`/`RestartSec=10` + `WantedBy=multi-user.target`; boot persistence is the `[Install]` section (generated system units cannot be `enable`d here) |
+| Boot persistence | `Restart=always`/`RestartSec=10` + `WantedBy=default.target` (`dankclaw/observability/quadlet/alloy.container:58-63`); installed at `~/.config/containers/systemd/alloy.container`, reporting `generated` + `active` (`systemctl --user is-enabled|is-active alloy` 2026-10-18) |
-| Successor to | Promtail (EOL 2026-03-02) |
+| Delivery | **Configured and healthy, currently delivering zero data** — see "Delivery state" |
 | Deploy status | root cutover built and tested in-repo (2026-10-20); the **live** instance is still the old rootless unit until `./scripts/deploy.sh dankclaw --service alloy` runs |
-## Why root (the load-bearing decision)
+## Why Alloy and not Promtail
-A rootless Alloy delivered **zero** lines, silently and with a green health
+Promtail reached end-of-life on 2026-03-02 and Grafana Labs directs new installs to Alloy. This stack uses Alloy from day one: no Promtail config or unit has ever existed in this repository (`git log -S promtail` over the whole history returns no commit that added one; the compose-era service of the same name was already Alloy — `dankclaw/observability/quadlet/alloy.container:6-7`).
 check. Two independent causes, both proven on the host 2026-10-20:
 1. **The journal is not readable by a rootless container.** The files are
   `root:systemd-journal 0640` (`group:adm`, `other::---`). Even bind-mounted, a
   rootless container has no supplementary groups, so opening them returns
   `DENIED`. `loki.source.journal` then sees an empty view and the counter just
   sits at 0 — no error logged.
 2. **Container logs live in the *user* journal, not `ctr.log`.** The rootless
   Quadlet containers log via the journald driver into `user-1000.journal`; the
   old file globs (`/var/lib/containers/.../ctr.log`, `/var/log/containers/*.log`)
   point at a root-storage layout that does not exist here.
 **Root fixes both at once.** As root the container opens every journal file
 under the machine-id directory — `system.journal` *and* `user-1000.journal` —
 so a single `loki.source.journal` carries host services and the rootless
 containers' logs together. Validated with a throwaway root probe before writing
 the unit: **31,825 journal lines read, 26,966 pushed to Loki in 20 s, zero
 drops**, and Loki's label set went from empty to populated.
 ### The two mounts that make it work
 - **`/etc/machine-id` (ro) — load-bearing.** `sd_journal` locates the journal
  directory by the *local* machine id. Without it the container opens nothing and
  the journal source reads exactly 0 lines — silently, health green. This single
  mount is the whole difference between a working and an empty pipeline; do not
  remove it. (A root probe without it reads 0; with it, 31,825.)
 - **`Network=host`.** Loki publishes `127.0.0.1:3100` (loopback only). No bridge
  container — rootless or root — can reach the host's loopback, so Alloy must
  share the host network to push to `127.0.0.1:3100`. Consequence: the UI binds
  `127.0.0.1` directly (no `PublishPort`), so it stays loopback-only as before.
 ## Configuration
-`shared/alloy/config.alloy` is the source of truth — one `loki.source.journal`
+`shared/alloy/config.alloy` is the source of truth:
-(labelled `job="systemd-journal"`) forwarding to the loopback `loki.write`
+
-endpoint. There is deliberately **no** container-file source: container stdout is
+- Reads journald logs from the host's `/run/systemd/journal/socket`, labelled `job="systemd-journal"` (`shared/alloy/config.alloy:20-26`).
-in the journal.
+- Reads container logs from two globs: `/var/lib/containers/storage/overlay-containers/*/userdata/ctr.log` (`job="podman"`) and `/var/log/containers/*.log` (`job="containers"`) (`config.alloy:29-35`).
 - Forwards everything to `http://loki:3100/loki/api/v1/push` by container name on `dankclaw_default` (`config.alloy:13-17`).
 ## Volumes
 | Container path | Host path |
 |----------------|-----------|
-| `/etc/alloy` | `/opt/alloy/config` (ro) |
+| `/etc/alloy` | `/opt/alloy/config` |
-| `/etc/machine-id` | host machine id (ro) — **required** |
+| `/run/systemd/journal/socket` | host journald socket (read-only) |
-| `/run/systemd/journal/socket` | host journald socket (ro) |
+| `/var/log` | host logs (read-only) |
-| `/var/log/journal` | persistent journal files (ro) |
+| `/var/lib/containers` | podman storage (read-only) |
 | `/run/log/journal` | runtime journal files (ro) |
-## The UI stays on loopback
+## The UI port is published on purpose
-Alloy 1.x refuses to configure the `ui` service from a config block, so the UI
+Alloy's UI listens on the container's own `127.0.0.1:12345` by default, and Alloy 1.x refuses to configure the `ui` service from a config block ("service ui does not support being configured", `shared/alloy/config.alloy:8-10`). Under compose that meant **no reachable health endpoint at all**: the homepage status card for alloy read `down` continuously and a deploy had nothing to wait on. The unit therefore passes `--server.http.listen-addr=0.0.0.0:12345` (`dankclaw/observability/quadlet/alloy.container:46`) and publishes `127.0.0.1:12345` (`alloy.container:47`, confirmed by `ss -ltnp` 2026-10-18) — same posture as the other three: reachable from the host and from Caddy's network, and from nothing off-host. Alloy has no authentication, which is exactly why it is not published wider. The homepage card probes that same loopback port (`shared/homepage/services.json:233`).
-address is set on the command line instead. Under the old rootless unit that
+
-meant `0.0.0.0` inside the container plus a published `127.0.0.1:12345`; under
+## Delivery state (2026-10-20): healthy, shipping nothing
-`Network=host` the listen-addr is `127.0.0.1:12345` directly — the host's
+
-loopback, reachable from the host and from nothing off-host. Alloy has no
+The unit is `active`, the container reports `healthy`, the log says `Alloy is
-authentication, which is exactly why it is never published wider. The homepage
+ready`, and the pipeline carries no data at all.
-card and the deploy health gate both dial `http://127.0.0.1:12345/-/healthy`.
+
 **Root cause, proven: a rootless container cannot read the host journal.** The
 journal files are `root:systemd-journal 0640` (with `group:adm`, `other::---`).
 `/var/log` is bind-mounted so the files are *visible*, but a rootless container
 spawns with no supplementary groups, so it is denied. Proven directly:
 `podman run --rm -v /var/log/journal:/var/log/journal:ro busybox cat
 /var/log/journal/*/system.journal` → `DENIED`. `loki.source.journal` opens via
 `sd_journal_open`, so it silently sees an empty view — no error logged, counter
 frozen at `loki_source_journal_target_lines_total 0` (`system.journal` and
 `user-1000.journal` are both present and non-trivial, 2026-10-20).
 **Second, independent gap:** the rootless Quadlet containers log to the *user*
 journal (`user-1000.journal`), and `loki.source.journal` reads the *system*
 journal only — a fixed system read would still miss them. Container stdout is
 not in a `ctr.log` here either: the file globs point at root storage
 (`/var/lib/containers/storage/.../ctr.log`, `/var/log/containers/*.log`) that
 rootless does not use, refused at start with `stat …: permission denied` / `no
 such file or directory`, `loki_source_file_files_active_total 0`.
 Downstream: Loki holds zero streams (`GET /loki/api/v1/labels` → no `data`) —
 see [[Loki]].
 **Closing it is a permission/architecture decision, not a config edit.** The
 unit, config and push endpoint are correct as written; rootless Podman simply
 cannot grant the journal group. The viable shapes: run Alloy as a *system*
 service (root, in `adm`/`systemd-journal`) instead of a rootless Quadlet; grant
 the container's mapped uid journal read via a tmpfiles/ACL rule; or have
 journald forward to a path the container can read. Each trades rootless posture
 for reach — decide deliberately. Re-check `loki_source_journal_target_lines_total`
 and the Loki label query to confirm a fix, never `/healthy`: health is exactly
 what hides this failure.
 ## Gotchas
- **No `/etc/machine-id` mount ⇒ silent zero.** The health check still passes.
+- The journald socket mount is required for journal scraping.
-  This is the failure mode that hid the broken pipeline; check
+- Container log paths differ between docker (`/var/lib/docker/containers`) and podman (`/var/lib/containers/storage/...`); point the config at the right one per host.
-  `loki_source_journal_target_lines_total` and the Loki label query, never
+- One Alloy per host; do not run a single instance that tries to reach both hosts' logs.
-  `/healthy`.
+- The unit `Wants=`/`After=` `loki.service`: without a reachable Loki the write client fails and the pipeline backs up.
 - **`After=loki.service` is gone.** Alloy is on the system manager; Loki is a
  rootless user unit — a system unit cannot order against a user unit. Alloy
  retries its push until Loki answers, so ordering is unnecessary.
 - **The image must be in the ROOT store.** The deployer builds rootless and
  mirrors (`mirror_image_to_root`); a `Pull=never` root quadlet with only a
  rootless tag fails at create.
 - One Alloy per host; do not run a single instance that tries to reach both
  hosts' logs.
 ## See Also
@@ -81,7 +81,7 @@ Two provisioning rules, each verified live by an outage:
   appears. Create the Application with `provider_id` set (the UI does this
   automatically; raw API/ORM drives must do it explicitly).
-## OAuth2 relying-party quirks (this fork; corrected live 2026-10-20)
+## OAuth2 relying-party quirks (this fork, verified live 2026-08-31)
 The Grafana RP provider (`grafana`, pk 9, confidential client) was created and
 verified end-to-end via `ak shell` ORM drives. This fork's OAuth2 provider
@@ -100,32 +100,23 @@ implementation diverges from upstream in ways that bit us; all verified live:
 - **Provider resolution is by `client_id` query param** on the global
  `/application/o/authorize/` route; per-slug authorize paths do not exist in
  this build. Token/userinfo live at `/application/o/token/` and
-  `/application/o/userinfo/` — do **not** append `?api=1` to a Grafana API URL.
+  `/application/o/userinfo/?api=1`.
- **NULL `authorization_flow` → authorize 500, `AttributeError: 'NoneType'
+- **Property-mapping evaluator is broken → userinfo returns `sub` only.**
-  object has no attribute 'slug'`.** A provider created via raw `ak shell`
+  `PropertyMappingEvaluator.evaluate(*args, **kwargs)` forwards kwargs to
-  `create()` leaves `authorization_flow` unset; the authorize view then crashes
+  `BaseEvaluator.evaluate(expression_source)`, so the first context kwarg
-  building the flow planner (`flows/planner.py`) and returns "Server Error —
+  becomes the *expression source* instead of the context; every call raises a
-  Powered by authentik" before any claim is built. This was previously
+  swallowed `PropertyMappingExpressionException` and the claim is skipped.
-  mis-attributed to a broken evaluator; the crash is the missing flow. Fix: set
+  Scope-grant filtering has the same class of bug: `ScopeMapping` has no
-  `authorization_flow = default-provider-authorization-implicit-consent` (the
+  provider FK (linkage lives in the `ProviderPropertyMapping` through model)
-  flow every other provider in this instance already uses).
+  so `filter(provider=...)` always yields an empty queryset and tokens asked
- **A scope mapping that returns a non-dict emits no claim.** authentik merges a
+  to carry scopes get zero. Net effect: userinfo/introspection answers with
-  granted scope mapping's evaluated result into the token **as a mapping**: a
+  `{sub}` only — email/groups/entitlements claims cannot be delivered by this
-  dict contributes its keys as claims; a bare string is keyed under the
+  build until the evaluator is patched or upgraded. Clients consuming this
-  mapping's NAME, so nothing usable appears. `grafana-email-scope` returned the
+  provider must key identity off `sub`.
-  bare string `user.email`, so no `email` claim landed — the evaluator was
+- Scope mappings for this provider exist — `grafana-email-scope` and
-  never broken (the dict-valued `grafana-profile-scope` mapping has always
+  `grafana-profile-scope` are both rows in `authentik_core_propertymapping`
-  delivered, and the stock OpenID mappings return dicts). Fix:
+  (read-only `SELECT`, 2026-10-18) — and will start emitting claims the moment
-  `return {"email": user.email, "email_verified": True}`. With no email, Grafana
+  the evaluator bug is fixed; no config change is needed then.
  v13 falls back to GET `<api_url>/emails`; authentik 404s it → "Login failed:
  InternalError".
 - **Grafana reads identity from the standard userinfo endpoint** (`API_URL` with
  no `?api=1`); no `USE_ID_TOKEN` is needed once the `email` claim exists.
 - **Net (2026-10-20):** the evaluator is fine; email/profile claims reach RP
  clients. Both IdP fixes are idempotent in
  `scripts/authentik-grafana-sso-fix.sh` (re-run after an ak-shell re-creation of
  the provider or a blueprint import, which revert them). Grafana's half plus the
  verified outcome (Viewer, `login = <sub hex>`, email present) are in [[Grafana]].
 ## LDAP provider objects (provisioned via `ak shell` ORM)
@@ -4,7 +4,7 @@ description: "Grafana visualizes the metrics that Prometheus collects from every
 tags:
  - infrastructure
  - grafana
-timestamp: 2026-10-20
+timestamp: 2026-09-11
 ---
 # Grafana
@@ -64,28 +64,19 @@ Grafana responds on `/api/health` with 200 and `"database":"ok"`; measured 2026-
 - Change the default admin credentials at first boot (env or UI).
 - Dashboards provisioned as code (dashboard JSON plus a YAML provider, `shared/grafana/provisioning/dashboards/`) survive container recreation; UI-created ones live in the data volume — back it up.
 - **Auth = OAuth2 relying-party, not auth_proxy.** The embedded outpost carries
-  no provider token (`token=None`), so no identity header reaches Grafana; the
+  no provider token (`token=None`), so no identity header ever reaches Grafana;
-  login is Grafana's own "Sign in with Authentik" button against the `grafana`
+  RP mode is topology-independent. The full verified contract (provider pk 9,
-  confidential-client provider. **Verified end-to-end 2026-10-20** after four
+  client_id `grafana-21ca190cde75d60f`, global authorize/token/userinfo paths,
-  independent breakages had made it fail. The two Authentik-side causes (a NULL
+  Basic client auth, form-encoded token body, and the fork's userinfo-claims
-  `authorization_flow` → authorize 500; an email scope mapping that returned a
+  bug that leaves identity at the `sub` hash only) is in
-  bare string so no `email` claim was emitted) and the full wire contract live in
+  [[Authentik Outpost Reference]] → "OAuth2 relying-party quirks". Consequences for Grafana:
-  [[Authentik Outpost Reference]] → "OAuth2 relying-party quirks"; the IdP half is
+  The unit sets `ALLOW_SIGN_UP=false`, `AUTO_UPSERT=true`, `ALLOW_ASSIGN_ORG=true`
-  re-applied by `scripts/authentik-grafana-sso-fix.sh` (idempotent — re-run after
+  `ORG_ID=1` and **no** role-attribute mapping (`grafana.container:52-55`), so
-  an ak-shell re-creation of the provider or a blueprint import, both of which
+  users land as Viewer with `login = <sub hex>`; no email/groups/role claims
-  revert it silently). The Grafana half is the unit's own env (`grafana.container`):
+  reach Grafana until the evaluator bug is fixed — do not write role mappings
-  `SCOPES="openid email profile"` (a GitLab-style `user:email` scope crashes the
+  that depend on those claims. The local admin account stays the only Admin.
-  authorize view), the standard userinfo `API_URL` with **no `?api=1`**, the
+  (The Viewer-default outcome and the `<sub hex>` login are not readable from
-  attribute paths `email`/`sub`/`preferred_username`, and `ALLOW_SIGN_UP=true`
+  the host without the database or an admin password. *(unverified 2026-10-18)*)
  (safe: the guard fronts the callback). Grafana v13 only GETs `<api_url>/emails`
  when the token carries no `email` claim — so it is the email claim itself (an
  IdP fix, above) that makes the login land, not any Grafana toggle.
 - **What a new login becomes:** a Viewer in org 1 (`ALLOW_ASSIGN_ORG=true`,
  `ORG_ID=1`, no role-attribute mapping), with `login = <sub hex>` but the real
  `email`. The local admin account stays the only Admin. The landing that the
  edge-only 13-sso spec never checked is now asserted by
  `tests/e2e/specs/20-grafana-sso.spec.ts` (RP round-trip → `/api/user` 200 with
  an email).
 - **Never point this unit's `EnvironmentFile` at the repo-root `.env`.** That
  file is a gluetun env file and it carries `HOME=/root`; systemd imports it
  for the service, podman then looks for its configuration under `/root`
@@ -102,18 +102,17 @@ verify it through the APIs.
  installed from `shared/grafana/provisioning/`. The Loki datasource is how
  you read logs: Loki has no web UI of its own, so Grafana Explore is the
  human surface (the guarded `loki.franksdank.cloud` vhost serves only the API).
-* **Logs: fixed in code, awaiting deploy.** The pipeline shipped nothing because
+* **Logs: configured, empty.** Alloy ships zero lines and Loki holds zero
-  a rootless Alloy cannot read the `0640` journal files, and the container-log
+  streams (`GET /loki/api/v1/labels` returns no label names; no chunk files on
-  globs point at a root-storage layout that does not exist here. The fix is built
+  disk). The cause is source-side access, now proven, not a config bug: a
-  and tested in-repo (2026-10-20): Alloy is now a **root system service on
+  rootless container cannot read the journal files (`system.journal` is
-  `Network=host`**, reading `system.journal` + `user-*.journal` and pushing to
+  `root:systemd-journal 0640`/`adm` and a rootless read of it returns DENIED),
-  Loki's `127.0.0.1:3100` loopback publish — a throwaway root probe read 31,825
+  and the container-log globs point at a root-storage path that does not exist
-  lines and pushed 26,966 with zero drops. The **live** instance is still the old
+  under rootless. Closing it is a permission/architecture decision, not an
-  rootless unit until `deploy.sh dankclaw --service alloy` runs. Detail in
+  edit — see [[Alloy]] → Delivery state and [[Loki]] → Contents.
  [[Alloy]] → Why root; Loki's store state in [[Loki]] → Contents.
 * **Not configured:** no blackbox exporter (the structural downs have no real
-  uptime signal) and no Alertmanager vhost. The log pipeline's fix is in code but
+  uptime signal) and no Alertmanager vhost. The log pipeline is provisioned
-  not yet deployed (see *Logs* above).
+  but carries no data (see *Logs* above).
 ## Citations
@@ -1,34 +1,22 @@
 // Alloy configuration — ships host and container logs to Loki.
 // One instance per host. Deployed to /opt/alloy/config/config.alloy.
 //
 // Runs as a ROOT system service on Network=host (see
 // dankclaw/observability/quadlet/alloy.container), so it reaches Loki on the
 // host's loopback and reads every journal — host services and rootless-quadlet
 // containers alike.
 logging {
  level = "info"
 }
-// The alloy UI listens on 127.0.0.1:12345; the in-container healthcheck probes
+// The alloy UI service listens on 127.0.0.1:12345 by default; the
-// it there. alloy 1.x refuses to configure the ui service from a config block,
+// in-container healthcheck probes it there. alloy 1.x refuses to
-// so there is no ui block.
+// configure the ui service from a config block, so there is no block.
-// Loki is published on the host's loopback (127.0.0.1:3100). Alloy shares the
+// Ship logs to Loki (container name on the podman network).
 // host network, so it pushes by loopback, never by container name — a root
 // container is not on the rootless dankclaw_default network.
 loki.write "default" {
  endpoint {
-    url = "http://127.0.0.1:3100/loki/api/v1/push"
+    url = "http://loki:3100/loki/api/v1/push"
  }
 }
-// The systemd journal. As root with /etc/machine-id + the journal dirs mounted,
+// Systemd journal from the host socket.
 // this one source reads system.journal AND user-*.journal, so it carries both
 // host services and the rootless Quadlet containers' logs. There is deliberately
 // no container-file source: podman container stdout lands in the journal, and
 // the old /var/lib/containers/.../ctr.log and /var/log/containers globs pointed
 // at a root-storage layout that does not exist on this host.
 loki.source.journal "journal" {
  format_as_json = true
  labels = {
@@ -36,3 +24,12 @@ loki.source.journal "journal" {
  }
  forward_to = [loki.write.default.receiver]
 }
 // Container logs (podman storage path; adjust for docker).
 loki.source.file "containers" {
  targets = [
    {__path__ = "/var/lib/containers/storage/overlay-containers/*/userdata/ctr.log", job = "podman"},
    {__path__ = "/var/log/containers/*.log", job = "containers"},
  ]
  forward_to = [loki.write.default.receiver]
 }
@@ -240,12 +240,14 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
 }
@test "alloy's UI address comes from the unit, not the config file" {
-    # alloy 1.x refuses to configure the ui service from a config block, so the
+    # alloy 1.x refuses to configure the ui service from a config block
-    # unit sets the UI address on the command line. As a Network=host root unit
+    # ("service ui does not support being configured"), so the unit moves the
-    # that address is the host loopback directly (no PublishPort) — reachable
+    # UI off the container's loopback with a command-line flag. Without that
-    # from the host, from nothing off-host.
+    # flag the port cannot be published and the service has no reachable
    # health endpoint at all — which is why the homepage status card for
    # alloy read "down" while it ran under compose.
    ! grep -qE '^(ui|server) \{' "$REPO_ROOT/shared/alloy/config.alloy"
    local unit="$REPO_ROOT/dankclaw/observability/quadlet/alloy.container"
-    grep -q -- '--server.http.listen-addr=127.0.0.1:12345' "$unit"
+    grep -q -- '--server.http.listen-addr=0.0.0.0:12345' "$unit"
-    ! grep -q '^PublishPort=' "$unit"
+    grep -q '^PublishPort=127.0.0.1:12345:12345$' "$unit"
 }
@@ -29,10 +29,8 @@ export default defineConfig({
      // green suite with a test that does not exist. 17- joined on 2026-09-16
      // (the hermes vhost, whose upstream is a host address rather than a
      // container name, so it needs its own pass condition). 18- joined on
-      // 2026-09-24 (OpenWebUI SSO through authentik). 20- joined on 2026-10-21
+      // 2026-09-24 (OpenWebUI SSO through authentik).
-      // (Grafana's OAuth2 relying-party login, which the edge-only 13- never
+      testMatch: /(10|12|13|16|17|18)-.*\.spec\.ts/,
      // follows).
      testMatch: /(10|12|13|16|17|18|20)-.*\.spec\.ts/,
      use: { storageState: ".auth/qa-e2e.json" },
      dependencies: ["setup"],
    },
@@ -1,56 +0,0 @@
 import { expect, test } from "@playwright/test";
 // Grafana (grafana.franksdank.cloud) is an OAuth2 RELYING PARTY, not an
 // outpost-fronted proxy: the guard chain (13-sso) only proves an authenticated
 // session reaches Grafana's login page. The login itself happens through
 // Grafana's own "Sign in with Authentik" button, which the edge test never
 // follows — that is exactly how the RP login could be dead (500 on authorize,
 // or an email-claim gap bouncing back to /login) while 13- still passed green.
 // This spec drives the real RP round-trip and asserts a live Grafana session.
 const HOST = "grafana.franksdank.cloud";
 const CHALLENGE = "Making sure you're not a bot";
 test("SSO: Grafana relying-party login lands a real session", async ({ page, request }) => {
  test.setTimeout(180_000);
  // Warm the guard: the vhost runs its own Anubis proof-of-work on first visit.
  await page.goto(`https://${HOST}/`);
  await expect
    .poll(
      async () => !(await page.locator("body").innerText()).includes(CHALLENGE),
      { timeout: 90_000 },
    )
    .toBe(true);
  // The "Sign in with Authentik" button targets this RP start URL exactly.
  // Going straight there is the same code path without depending on the SPA's
  // button markup.
  await page.goto(`https://${HOST}/login/generic_oauth`);
  // The chain: grafana -> auth (implicit consent, the session is already
  // there) -> back to grafana with the code -> grafana provisions the user and
  // redirects to "/". A dead provider/claims config bounces back to /login.
  await expect
    .poll(
      async () => {
        const u = new URL(page.url());
        return u.hostname === HOST && u.pathname !== "/login";
      },
      { timeout: 90_000 },
    )
    .toBe(true);
  const body = await page.locator("body").innerText();
  expect(body).not.toContain("Login failed");
  expect(body.toLowerCase()).not.toContain("powered by authentik"); // IdP flow shell
  expect(body).not.toContain(CHALLENGE);
  // A genuine logged-in session exists: /api/user answers 200 with an email,
  // not 401. This is the assertion the edge-only spec could never make.
  const resp = await page.request.get(`https://${HOST}/api/user`);
  expect(resp.status()).toBe(200);
  const user = await resp.json();
  expect(typeof user.email).toBe("string");
  expect(user.email.length).toBeGreaterThan(0);
 });
@@ -41,17 +41,13 @@ service_section() {
 }
@test "every observability unit is boot-persistent" {
-    # Generated units cannot be enabled on this host's podman quadlet generator;
+    # Generated units cannot be enabled on this host's podman 5.x v1 quadlet
-    # [Install] + linger is the boot-persistence story for the rootless five.
+    # generator; [Install] + linger is the entire boot-persistence story.
    # Alloy is a ROOT system unit, so its [Install] targets multi-user.target.
    local svc
    for svc in "${UNITS[@]}"; do
        [[ "$svc" == alloy ]] && continue
        grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \
            || { echo "$svc is not boot-persistent"; return 1; }
    done
    grep -q '^WantedBy=multi-user.target$' "$UNIT_DIR/alloy.container" \
        || { echo "alloy (root unit) is not boot-persistent"; return 1; }
 }
@test "every observability unit is self-healing" {
@@ -144,20 +140,15 @@ service_section() {
    grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile"
    grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile"
    local svc
-    for svc in prometheus grafana loki; do
+    for svc in prometheus grafana loki alloy; do
        grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \
            || { echo "$svc unit renames its container"; return 1; }
        grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \
            || { echo "$svc is not on the network Caddy and its peers use"; return 1; }
    done
    # Alloy is not on that network: it is a root container on Network=host, the
    # only way to reach Loki's 127.0.0.1:3100 publish from outside a bridge.
    grep -q '^ContainerName=alloy$' "$UNIT_DIR/alloy.container"
    grep -q '^Network=host$' "$UNIT_DIR/alloy.container" \
        || { echo "alloy must be Network=host to reach Loki on the host loopback"; return 1; }
    grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
    grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
-    grep -q 'url = "http://127.0.0.1:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
+    grep -q 'url = "http://loki:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
 }
@test "every observability unit has a health check" {
@@ -248,14 +239,12 @@ service_section() {
 }
@test "startup order follows the data dependencies" {
-    # Grafana is useless without Prometheus (both rootless, same manager). Alloy
+    # Grafana is useless without Prometheus and Alloy drops logs without Loki;
-    # used to order on loki.service; as a ROOT unit it lives on the SYSTEM
+    # both pairs are expressed as systemd ordering, not launch luck.
    # manager and cannot order against a rootless user unit, so it depends only
    # on the network and retries its push until Loki answers.
    grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
    grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
-    grep -q '^After=network-online.target$' "$UNIT_DIR/alloy.container"
+    grep -q 'After=network-online.target loki.service' "$UNIT_DIR/alloy.container"
-    ! grep -q 'loki.service' "$UNIT_DIR/alloy.container"
+    grep -q 'Wants=network-online.target loki.service' "$UNIT_DIR/alloy.container"
 }
@test "the deployer installs, restarts and health-checks each unit" {
@@ -292,24 +281,6 @@ PAIRS
    grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY"
 }
@test "alloy is a root host-network unit that reads every journal" {
    # The whole point of the root cutover: a rootless container cannot open the
    # 0640 journal files. Pin the properties that make the pipeline carry data.
    local u="$UNIT_DIR/alloy.container"
    grep -q '^Network=host$' "$u"
    grep -q '^Volume=/etc/machine-id:/etc/machine-id:ro$' "$u" \
        || { echo "alloy missing the load-bearing machine-id mount"; return 1; }
    grep -q '^Volume=/var/log/journal:/var/log/journal:ro$' "$u"
    grep -q '^Volume=/run/log/journal:/run/log/journal:ro$' "$u"
    grep -q -- '--server.http.listen-addr=127.0.0.1:12345' "$u"
    # Deployer: system install + root image mirror, not the user quadlet loop.
    grep -q '^deploy_alloy() {' "$REMOTE_DEPLOY"
    grep -q 'install_system_quadlet dankclaw/observability/quadlet alloy' "$REMOTE_DEPLOY"
    grep -q '/etc/containers/systemd/' "$REMOTE_DEPLOY"
    grep -q 'mirror_image_to_root localhost/franksdank/alloy:latest' "$REMOTE_DEPLOY"
    grep -q 'alloy) deploy_alloy ;;' "$REMOTE_DEPLOY"
 }
@test "no dankclaw compose file competes with the quadlet units" {
    # Two owners for one container name is how a compose run resurrects a
    # stale duplicate next to the quadlet-managed one. This file was retired on