feat(alloy): run it as a root host-network service so it reads the journal

A rootless Alloy shipped zero lines: the journal files are
root:systemd-journal 0640 (other::---) and a rootless container carries no
supplementary groups, so sd_journal_open sees an empty view silently; the
rootless quadlet container logs also live in user-1000.journal, not the
ctr.log paths the old config tailed.

As root, one loki.source.journal reads system.journal + user-*.journal
and carries both. Network=host because Loki publishes 127.0.0.1:3100 and
no bridge container can reach the host loopback; the UI binds loopback
directly (no PublishPort). The /etc/machine-id mount is load-bearing —
sd_journal finds the journal dir by the local machine id; without it the
source reads 0 lines, health green.

deploy_alloy routes around the rootless loop: build rootless, mirror the
image into the root store (a root build fails on DNS in the build
container), retire the old rootless alloy unit, install to
/etc/containers/systemd. Validated with a throwaway root probe: 31,825
lines read, 26,966 pushed to Loki in 20s, zero drops.

dankclaw/observability/quadlet/alloy.container

@@ -1,55 +1,51 @@
-# Quadlet unit: alloy (rootless, boot-persistent).
+# Quadlet unit: alloy (ROOT system service, boot-persistent).
 #
-# Install path on dankclaw: ~/.config/containers/systemd/alloy.container
+# Install path on dankclaw: /etc/containers/systemd/alloy.container
-# Unit name: alloy.service (container name: alloy)
+# Unit name: alloy.service (container name: alloy) — SYSTEM manager, root.
 #
-# Log shipper. This replaces the compose service of the same name in
+# WHY ROOT (not rootless, unlike its observability siblings). Alloy reads the
-# dankclaw/compose.yaml ("Observability" block).
+# host journal. The journal files are root:systemd-journal 0640 (other::---);
+# a rootless container carries no supplementary groups and cannot open them —
+# a rootless read of system.journal returns DENIED (see the Alloy page in the
+# vault). Running as root, one loki.source.journal reads system.journal AND
+# user-1000.journal, so it carries both host logs and the rootless Quadlet
+# containers' logs. Validated 2026-10-20: a root probe read 31,825 journal
+# lines and pushed 26,966 into Loki in 20 s, zero drops.
 #
-# IMAGE. franksdank/alloy is built locally by deploy-remote.sh
+# NETWORK=host. Loki publishes 127.0.0.1:3100 (loopback only) on the host, and
-# (build_compose_image) from dankclaw/alloy/Containerfile: upstream
+# no bridge container — rootless or root — can reach the host's loopback, so
-# grafana/alloy plus a static busybox for the healthcheck. Pull=never keeps a
+# this unit runs on the host network and pushes to 127.0.0.1:3100. The side
-# boot from reaching for that local tag at a registry.
+# effect: the UI must bind loopback too, so Exec sets the listen-addr to
+# 127.0.0.1 (the old rootless unit used 0.0.0.0 only because container loopback
+# was not host loopback; under Network=host they are the same address, so the
+# unauthenticated UI stays loopback-only — nothing off-host reaches it).
 #
-# NETWORK. It joins dankclaw_default so its config's loki.write endpoint
+# THE /etc/machine-id MOUNT IS LOAD-BEARING. sd_journal locates the journal
-# resolves the container name loki:3100.
+# directory by the local machine id; without it the container opens nothing and
+# the journal source reads exactly 0 lines — silently, no error, health green.
+# This mount is the entire reason the pipeline carries data. Never remove it.
 #
-# THE UI PORT EXISTS ON PURPOSE. Alloy's UI listens on the container's own
+# IMAGE. Built rootless by build_compose_image alloy, then mirrored into the
-# 127.0.0.1 by default and cannot be moved from the config file (alloy 1.x
+# root image store by the deployer (a root build runs in the build container's
-# refuses the ui block), so the compose service had no reachable health
+# own resolver and fails on DNS here). Pull=never: boot must not reach for the
-# endpoint at all: the homepage status card for alloy reported "down"
+# local tag at a registry.
-# continuously and there was nothing for a deploy to wait on. The command
-# below therefore moves the UI to the container's wildcard address and the
-# port is published on host loopback only, which puts alloy in the same class
-# as the other three: reachable from the host, by Caddy over the private
-# network, and by nothing off-host. Alloy has no authentication, which is
-# exactly why it is not published wider.
-#
-# SOURCES. The host journald socket and the host log/container-storage trees
-# are bind-mounted read-only; Alloy tails them and pushes to Loki. The config
-# lives at /opt/alloy/config/config.alloy, installed from
-# shared/alloy/config.alloy on every deploy.
-#
-# ORDERING. Wants/After loki.service: without a reachable Loki, Alloy's write
-# client fails and its log pipeline backs up.
 
 [Unit]
 Description=Alloy log shipper (Dankclaw)
-After=network-online.target loki.service
+After=network-online.target
-Wants=network-online.target loki.service
 
 [Container]
 ContainerName=alloy
 Image=localhost/franksdank/alloy:latest
 Pull=never
-Network=dankclaw_default
+Network=host
-Exec=run --server.http.listen-addr=0.0.0.0:12345 /etc/alloy/config.alloy
+Exec=run --server.http.listen-addr=127.0.0.1:12345 /etc/alloy/config.alloy
-PublishPort=127.0.0.1:12345:12345
+Volume=/opt/alloy/config:/etc/alloy:ro
-Volume=/opt/alloy/config:/etc/alloy
+Volume=/etc/machine-id:/etc/machine-id:ro
 Volume=/run/systemd/journal/socket:/run/systemd/journal/socket:ro
-Volume=/var/log:/var/log:ro
+Volume=/var/log/journal:/var/log/journal:ro
-Volume=/var/lib/containers:/var/lib/containers:ro
+Volume=/run/log/journal:/run/log/journal:ro
-HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:12345/-/healthy
+HealthCmd=/usr/local/bin/busybox wget --spider -q http://127.0.0.1:12345/-/healthy
 HealthInterval=30s
 HealthTimeout=10s
 HealthRetries=3
@@ -60,4 +56,4 @@ Restart=always
 RestartSec=10
 
 [Install]
-WantedBy=default.target
+WantedBy=multi-user.target

scripts/deploy-remote.sh

@@ -487,13 +487,15 @@ seed_grafana_env() {
         || { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; }
 }
 
-# The five observability containers left compose on 2026-09-11 (they were
+# The observability containers left compose on 2026-09-11 (they were started by
-# started by hand with podman-compose and never came back after a reboot).
+# hand with podman-compose and never came back after a reboot). Five are ordinary
-# They are ordinary rootless Quadlet units now: same images, same mounts, same
+# rootless Quadlet units: same images, same mounts, same arguments, same
-# arguments, same container names, so nothing that addresses them by name —
+# container names, so nothing that addresses them by name — Caddy, Prometheus'
-# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
+# own scrape config — changed. Only the published ports did: the three HTTP UIs
-# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
+# bind 127.0.0.1, because the compose services published on 0.0.0.0 and none of
-# compose services published on 0.0.0.0 and none of them authenticates.
+# them authenticates. Alloy is the exception: a ROOT system service on
+# Network=host, because reading the host journal needs privileges a rootless
+# container cannot get. It is handled by deploy_alloy, not the loop below.
 deploy_observability() {
     local service="$1"
     # Observability units live under <target>/observability/quadlet: Dankclaw
@@ -512,10 +514,6 @@ deploy_observability() {
             copy_loki_config
             build_compose_image loki
             ;;
-        alloy)
-            copy_alloy_config
-            build_compose_image alloy
-            ;;
         grafana)
             copy_grafana_provisioning
             seed_grafana_env || return 1
@@ -534,11 +532,71 @@ deploy_observability() {
         alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
         grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
         loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
-        alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
         node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
     esac
 }
 
+# Mirror an image the deployer built rootless into the ROOT image store. A root
+# quadlet reads root storage; a root build (build_compose_image is rootless) is
+# not possible on this host — the build container's resolver fails DNS — so the
+# only way to get a root-readable copy is save-and-load. Compares image ids so a
+# rebuilt alloy actually re-lands in root storage instead of leaving a stale tag.
+mirror_image_to_root() {
+    local ref="$1" rid
+    rid="$(podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" || {
+        echo "ERROR: rootless image $ref is not built; cannot mirror to root" >&2
+        return 1
+    }
+    if [[ "$(run_privileged podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" == "$rid" ]]; then
+        return 0
+    fi
+    local tar; tar="$(mktemp /var/tmp/alloy-mirror.XXXXXX.tar)"
+    podman save -o "$tar" "$ref"
+    run_privileged podman load -i "$tar"
+    rm -f "$tar"
+}
+
+# Install a quadlet into the SYSTEM generator dir (root manager) instead of the
+# user dir. Boot persistence is the unit's own [Install] section (multi-user
+# target); the v1 system generator creates the .wants on daemon-reload, so —
+# like the user units — no `systemctl enable` (generated units cannot be enabled
+# here).
+install_system_quadlet() {
+    local source_dir="$1" unit="$2"
+    run_privileged mkdir -p /etc/containers/systemd
+    run_privileged cp "$source_dir/$unit.container" /etc/containers/systemd/
+    run_privileged systemctl daemon-reload
+}
+
+# Alloy as a ROOT system service on Network=host. See the unit file: a rootless
+# container cannot read the 0640 journal files, and it cannot reach Loki's
+# 127.0.0.1:3100 publish from a bridge netns. Root + host network fixes both;
+# the price is that this one container runs as root and on the host network.
+deploy_alloy() {
+    copy_alloy_config
+    build_compose_image alloy                      # rootless build (resolver works)
+    mirror_image_to_root localhost/franksdank/alloy:latest
+
+    # Retire the rootless unit BEFORE the system one starts: both managers
+    # generate alloy.service, and two containers named "alloy" both binding
+    # 127.0.0.1:12345 cannot coexist. Move the user quadlet out of the generator
+    # dir (it regenerates from there) and stop the user unit.
+    if [[ -f "$quadlet_unit_dir/alloy.container" ]] ||
+       systemctl --user is-active --quiet alloy.service 2>/dev/null; then
+        systemctl --user stop alloy.service 2>/dev/null || true
+        if [[ -f "$quadlet_unit_dir/alloy.container" ]]; then
+            local trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-alloy-root-cutover"
+            mkdir -p "$trash"
+            mv -- "$quadlet_unit_dir/alloy.container" "$trash/alloy.container"
+            echo "retired rootless alloy.container -> $trash"
+        fi
+        systemctl --user daemon-reload
+    fi
+
+    install_system_quadlet dankclaw/observability/quadlet alloy
+    run_privileged systemctl restart alloy.service
+    wait_for_health alloy http://127.0.0.1:12345/-/healthy 60
+}
 build_compose_image() {
     local service="$1" image cfhash oldhash containerfile
     # Every file the image COPIES IN, beside the Containerfile. The skip below
@@ -1941,7 +1999,8 @@ deploy_service() {
         copyparty) deploy_copyparty ;;
         plex) deploy_plex ;;
         tautulli) deploy_tautulli ;;
-        prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
+        alloy) deploy_alloy ;;
+        prometheus|grafana|loki|node-exporter|alertmanager) deploy_observability "$service" ;;
         comfyui) deploy_comfyui ;;
         *) deploy_compose_service "$service" ;;
     esac

shared/alloy/config.alloy

@@ -1,22 +1,34 @@
 // Alloy configuration — ships host and container logs to Loki.
 // One instance per host. Deployed to /opt/alloy/config/config.alloy.
+//
+// Runs as a ROOT system service on Network=host (see
+// dankclaw/observability/quadlet/alloy.container), so it reaches Loki on the
+// host's loopback and reads every journal — host services and rootless-quadlet
+// containers alike.
 
 logging {
   level = "info"
 }
 
-// The alloy UI service listens on 127.0.0.1:12345 by default; the
+// The alloy UI listens on 127.0.0.1:12345; the in-container healthcheck probes
-// in-container healthcheck probes it there. alloy 1.x refuses to
+// it there. alloy 1.x refuses to configure the ui service from a config block,
-// configure the ui service from a config block, so there is no block.
+// so there is no ui block.
 
-// Ship logs to Loki (container name on the podman network).
+// Loki is published on the host's loopback (127.0.0.1:3100). Alloy shares the
+// host network, so it pushes by loopback, never by container name — a root
+// container is not on the rootless dankclaw_default network.
 loki.write "default" {
   endpoint {
-    url = "http://loki:3100/loki/api/v1/push"
+    url = "http://127.0.0.1:3100/loki/api/v1/push"
   }
 }
 
-// Systemd journal from the host socket.
+// The systemd journal. As root with /etc/machine-id + the journal dirs mounted,
+// this one source reads system.journal AND user-*.journal, so it carries both
+// host services and the rootless Quadlet containers' logs. There is deliberately
+// no container-file source: podman container stdout lands in the journal, and
+// the old /var/lib/containers/.../ctr.log and /var/log/containers globs pointed
+// at a root-storage layout that does not exist on this host.
 loki.source.journal "journal" {
   format_as_json = true
   labels = {
@@ -24,12 +36,3 @@ loki.source.journal "journal" {
   }
   forward_to = [loki.write.default.receiver]
 }
-
-// Container logs (podman storage path; adjust for docker).
-loki.source.file "containers" {
-  targets = [
-    {__path__ = "/var/lib/containers/storage/overlay-containers/*/userdata/ctr.log", job = "podman"},
-    {__path__ = "/var/log/containers/*.log", job = "containers"},
-  ]
-  forward_to = [loki.write.default.receiver]
-}