From 634fef242b9476be677475f3b247580d4d961fa6 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 20 Oct 2026 19:50:15 -0700 Subject: [PATCH] feat(alloy): run it as a root host-network service so it reads the journal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A rootless Alloy shipped zero lines: the journal files are root:systemd-journal 0640 (other::---) and a rootless container carries no supplementary groups, so sd_journal_open sees an empty view silently; the rootless quadlet container logs also live in user-1000.journal, not the ctr.log paths the old config tailed. As root, one loki.source.journal reads system.journal + user-*.journal and carries both. Network=host because Loki publishes 127.0.0.1:3100 and no bridge container can reach the host loopback; the UI binds loopback directly (no PublishPort). The /etc/machine-id mount is load-bearing — sd_journal finds the journal dir by the local machine id; without it the source reads 0 lines, health green. deploy_alloy routes around the rootless loop: build rootless, mirror the image into the root store (a root build fails on DNS in the build container), retire the old rootless alloy unit, install to /etc/containers/systemd. Validated with a throwaway root probe: 31,825 lines read, 26,966 pushed to Loki in 20s, zero drops. --- .../observability/quadlet/alloy.container | 74 ++++++++-------- scripts/deploy-remote.sh | 85 ++++++++++++++++--- shared/alloy/config.alloy | 33 +++---- 3 files changed, 125 insertions(+), 67 deletions(-) diff --git a/dankclaw/observability/quadlet/alloy.container b/dankclaw/observability/quadlet/alloy.container index b7815cc..336a7a5 100644 --- a/dankclaw/observability/quadlet/alloy.container +++ b/dankclaw/observability/quadlet/alloy.container @@ -1,55 +1,51 @@ -# Quadlet unit: alloy (rootless, boot-persistent). +# Quadlet unit: alloy (ROOT system service, boot-persistent). # -# Install path on dankclaw: ~/.config/containers/systemd/alloy.container -# Unit name: alloy.service (container name: alloy) +# Install path on dankclaw: /etc/containers/systemd/alloy.container +# Unit name: alloy.service (container name: alloy) — SYSTEM manager, root. # -# Log shipper. This replaces the compose service of the same name in -# dankclaw/compose.yaml ("Observability" block). +# WHY ROOT (not rootless, unlike its observability siblings). Alloy reads the +# host journal. The journal files are root:systemd-journal 0640 (other::---); +# a rootless container carries no supplementary groups and cannot open them — +# a rootless read of system.journal returns DENIED (see the Alloy page in the +# vault). Running as root, one loki.source.journal reads system.journal AND +# user-1000.journal, so it carries both host logs and the rootless Quadlet +# containers' logs. Validated 2026-10-20: a root probe read 31,825 journal +# lines and pushed 26,966 into Loki in 20 s, zero drops. # -# IMAGE. franksdank/alloy is built locally by deploy-remote.sh -# (build_compose_image) from dankclaw/alloy/Containerfile: upstream -# grafana/alloy plus a static busybox for the healthcheck. Pull=never keeps a -# boot from reaching for that local tag at a registry. +# NETWORK=host. Loki publishes 127.0.0.1:3100 (loopback only) on the host, and +# no bridge container — rootless or root — can reach the host's loopback, so +# this unit runs on the host network and pushes to 127.0.0.1:3100. The side +# effect: the UI must bind loopback too, so Exec sets the listen-addr to +# 127.0.0.1 (the old rootless unit used 0.0.0.0 only because container loopback +# was not host loopback; under Network=host they are the same address, so the +# unauthenticated UI stays loopback-only — nothing off-host reaches it). # -# NETWORK. It joins dankclaw_default so its config's loki.write endpoint -# resolves the container name loki:3100. +# THE /etc/machine-id MOUNT IS LOAD-BEARING. sd_journal locates the journal +# directory by the local machine id; without it the container opens nothing and +# the journal source reads exactly 0 lines — silently, no error, health green. +# This mount is the entire reason the pipeline carries data. Never remove it. # -# THE UI PORT EXISTS ON PURPOSE. Alloy's UI listens on the container's own -# 127.0.0.1 by default and cannot be moved from the config file (alloy 1.x -# refuses the ui block), so the compose service had no reachable health -# endpoint at all: the homepage status card for alloy reported "down" -# continuously and there was nothing for a deploy to wait on. The command -# below therefore moves the UI to the container's wildcard address and the -# port is published on host loopback only, which puts alloy in the same class -# as the other three: reachable from the host, by Caddy over the private -# network, and by nothing off-host. Alloy has no authentication, which is -# exactly why it is not published wider. -# -# SOURCES. The host journald socket and the host log/container-storage trees -# are bind-mounted read-only; Alloy tails them and pushes to Loki. The config -# lives at /opt/alloy/config/config.alloy, installed from -# shared/alloy/config.alloy on every deploy. -# -# ORDERING. Wants/After loki.service: without a reachable Loki, Alloy's write -# client fails and its log pipeline backs up. +# IMAGE. Built rootless by build_compose_image alloy, then mirrored into the +# root image store by the deployer (a root build runs in the build container's +# own resolver and fails on DNS here). Pull=never: boot must not reach for the +# local tag at a registry. [Unit] Description=Alloy log shipper (Dankclaw) -After=network-online.target loki.service -Wants=network-online.target loki.service +After=network-online.target [Container] ContainerName=alloy Image=localhost/franksdank/alloy:latest Pull=never -Network=dankclaw_default -Exec=run --server.http.listen-addr=0.0.0.0:12345 /etc/alloy/config.alloy -PublishPort=127.0.0.1:12345:12345 -Volume=/opt/alloy/config:/etc/alloy +Network=host +Exec=run --server.http.listen-addr=127.0.0.1:12345 /etc/alloy/config.alloy +Volume=/opt/alloy/config:/etc/alloy:ro +Volume=/etc/machine-id:/etc/machine-id:ro Volume=/run/systemd/journal/socket:/run/systemd/journal/socket:ro -Volume=/var/log:/var/log:ro -Volume=/var/lib/containers:/var/lib/containers:ro -HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:12345/-/healthy +Volume=/var/log/journal:/var/log/journal:ro +Volume=/run/log/journal:/run/log/journal:ro +HealthCmd=/usr/local/bin/busybox wget --spider -q http://127.0.0.1:12345/-/healthy HealthInterval=30s HealthTimeout=10s HealthRetries=3 @@ -60,4 +56,4 @@ Restart=always RestartSec=10 [Install] -WantedBy=default.target +WantedBy=multi-user.target diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 10a0709..fc4f39b 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -487,13 +487,15 @@ seed_grafana_env() { || { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; } } -# The five observability containers left compose on 2026-09-11 (they were -# started by hand with podman-compose and never came back after a reboot). -# They are ordinary rootless Quadlet units now: same images, same mounts, same -# arguments, same container names, so nothing that addresses them by name — -# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed. -# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the -# compose services published on 0.0.0.0 and none of them authenticates. +# The observability containers left compose on 2026-09-11 (they were started by +# hand with podman-compose and never came back after a reboot). Five are ordinary +# rootless Quadlet units: same images, same mounts, same arguments, same +# container names, so nothing that addresses them by name — Caddy, Prometheus' +# own scrape config — changed. Only the published ports did: the three HTTP UIs +# bind 127.0.0.1, because the compose services published on 0.0.0.0 and none of +# them authenticates. Alloy is the exception: a ROOT system service on +# Network=host, because reading the host journal needs privileges a rootless +# container cannot get. It is handled by deploy_alloy, not the loop below. deploy_observability() { local service="$1" # Observability units live under /observability/quadlet: Dankclaw @@ -512,10 +514,6 @@ deploy_observability() { copy_loki_config build_compose_image loki ;; - alloy) - copy_alloy_config - build_compose_image alloy - ;; grafana) copy_grafana_provisioning seed_grafana_env || return 1 @@ -534,11 +532,71 @@ deploy_observability() { alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;; grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;; loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;; - alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;; node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;; esac } +# Mirror an image the deployer built rootless into the ROOT image store. A root +# quadlet reads root storage; a root build (build_compose_image is rootless) is +# not possible on this host — the build container's resolver fails DNS — so the +# only way to get a root-readable copy is save-and-load. Compares image ids so a +# rebuilt alloy actually re-lands in root storage instead of leaving a stale tag. +mirror_image_to_root() { + local ref="$1" rid + rid="$(podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" || { + echo "ERROR: rootless image $ref is not built; cannot mirror to root" >&2 + return 1 + } + if [[ "$(run_privileged podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" == "$rid" ]]; then + return 0 + fi + local tar; tar="$(mktemp /var/tmp/alloy-mirror.XXXXXX.tar)" + podman save -o "$tar" "$ref" + run_privileged podman load -i "$tar" + rm -f "$tar" +} + +# Install a quadlet into the SYSTEM generator dir (root manager) instead of the +# user dir. Boot persistence is the unit's own [Install] section (multi-user +# target); the v1 system generator creates the .wants on daemon-reload, so — +# like the user units — no `systemctl enable` (generated units cannot be enabled +# here). +install_system_quadlet() { + local source_dir="$1" unit="$2" + run_privileged mkdir -p /etc/containers/systemd + run_privileged cp "$source_dir/$unit.container" /etc/containers/systemd/ + run_privileged systemctl daemon-reload +} + +# Alloy as a ROOT system service on Network=host. See the unit file: a rootless +# container cannot read the 0640 journal files, and it cannot reach Loki's +# 127.0.0.1:3100 publish from a bridge netns. Root + host network fixes both; +# the price is that this one container runs as root and on the host network. +deploy_alloy() { + copy_alloy_config + build_compose_image alloy # rootless build (resolver works) + mirror_image_to_root localhost/franksdank/alloy:latest + + # Retire the rootless unit BEFORE the system one starts: both managers + # generate alloy.service, and two containers named "alloy" both binding + # 127.0.0.1:12345 cannot coexist. Move the user quadlet out of the generator + # dir (it regenerates from there) and stop the user unit. + if [[ -f "$quadlet_unit_dir/alloy.container" ]] || + systemctl --user is-active --quiet alloy.service 2>/dev/null; then + systemctl --user stop alloy.service 2>/dev/null || true + if [[ -f "$quadlet_unit_dir/alloy.container" ]]; then + local trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-alloy-root-cutover" + mkdir -p "$trash" + mv -- "$quadlet_unit_dir/alloy.container" "$trash/alloy.container" + echo "retired rootless alloy.container -> $trash" + fi + systemctl --user daemon-reload + fi + + install_system_quadlet dankclaw/observability/quadlet alloy + run_privileged systemctl restart alloy.service + wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 +} build_compose_image() { local service="$1" image cfhash oldhash containerfile # Every file the image COPIES IN, beside the Containerfile. The skip below @@ -1941,7 +1999,8 @@ deploy_service() { copyparty) deploy_copyparty ;; plex) deploy_plex ;; tautulli) deploy_tautulli ;; - prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;; + alloy) deploy_alloy ;; + prometheus|grafana|loki|node-exporter|alertmanager) deploy_observability "$service" ;; comfyui) deploy_comfyui ;; *) deploy_compose_service "$service" ;; esac diff --git a/shared/alloy/config.alloy b/shared/alloy/config.alloy index 6f26b61..8727b58 100644 --- a/shared/alloy/config.alloy +++ b/shared/alloy/config.alloy @@ -1,22 +1,34 @@ // Alloy configuration — ships host and container logs to Loki. // One instance per host. Deployed to /opt/alloy/config/config.alloy. +// +// Runs as a ROOT system service on Network=host (see +// dankclaw/observability/quadlet/alloy.container), so it reaches Loki on the +// host's loopback and reads every journal — host services and rootless-quadlet +// containers alike. logging { level = "info" } -// The alloy UI service listens on 127.0.0.1:12345 by default; the -// in-container healthcheck probes it there. alloy 1.x refuses to -// configure the ui service from a config block, so there is no block. +// The alloy UI listens on 127.0.0.1:12345; the in-container healthcheck probes +// it there. alloy 1.x refuses to configure the ui service from a config block, +// so there is no ui block. -// Ship logs to Loki (container name on the podman network). +// Loki is published on the host's loopback (127.0.0.1:3100). Alloy shares the +// host network, so it pushes by loopback, never by container name — a root +// container is not on the rootless dankclaw_default network. loki.write "default" { endpoint { - url = "http://loki:3100/loki/api/v1/push" + url = "http://127.0.0.1:3100/loki/api/v1/push" } } -// Systemd journal from the host socket. +// The systemd journal. As root with /etc/machine-id + the journal dirs mounted, +// this one source reads system.journal AND user-*.journal, so it carries both +// host services and the rootless Quadlet containers' logs. There is deliberately +// no container-file source: podman container stdout lands in the journal, and +// the old /var/lib/containers/.../ctr.log and /var/log/containers globs pointed +// at a root-storage layout that does not exist on this host. loki.source.journal "journal" { format_as_json = true labels = { @@ -24,12 +36,3 @@ loki.source.journal "journal" { } forward_to = [loki.write.default.receiver] } - -// Container logs (podman storage path; adjust for docker). -loki.source.file "containers" { - targets = [ - {__path__ = "/var/lib/containers/storage/overlay-containers/*/userdata/ctr.log", job = "podman"}, - {__path__ = "/var/log/containers/*.log", job = "containers"}, - ] - forward_to = [loki.write.default.receiver] -}