Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Frank
|
e494617599 | ||
|
Frank
|
9809ed9c74 | ||
|
Frank
|
634fef242b |
@@ -1,55 +1,51 @@
|
||||
# Quadlet unit: alloy (rootless, boot-persistent).
|
||||
# Quadlet unit: alloy (ROOT system service, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/alloy.container
|
||||
# Unit name: alloy.service (container name: alloy)
|
||||
# Install path on dankclaw: /etc/containers/systemd/alloy.container
|
||||
# Unit name: alloy.service (container name: alloy) — SYSTEM manager, root.
|
||||
#
|
||||
# Log shipper. This replaces the compose service of the same name in
|
||||
# dankclaw/compose.yaml ("Observability" block).
|
||||
# WHY ROOT (not rootless, unlike its observability siblings). Alloy reads the
|
||||
# host journal. The journal files are root:systemd-journal 0640 (other::---);
|
||||
# a rootless container carries no supplementary groups and cannot open them —
|
||||
# a rootless read of system.journal returns DENIED (see the Alloy page in the
|
||||
# vault). Running as root, one loki.source.journal reads system.journal AND
|
||||
# user-1000.journal, so it carries both host logs and the rootless Quadlet
|
||||
# containers' logs. Validated 2026-10-20: a root probe read 31,825 journal
|
||||
# lines and pushed 26,966 into Loki in 20 s, zero drops.
|
||||
#
|
||||
# IMAGE. franksdank/alloy is built locally by deploy-remote.sh
|
||||
# (build_compose_image) from dankclaw/alloy/Containerfile: upstream
|
||||
# grafana/alloy plus a static busybox for the healthcheck. Pull=never keeps a
|
||||
# boot from reaching for that local tag at a registry.
|
||||
# NETWORK=host. Loki publishes 127.0.0.1:3100 (loopback only) on the host, and
|
||||
# no bridge container — rootless or root — can reach the host's loopback, so
|
||||
# this unit runs on the host network and pushes to 127.0.0.1:3100. The side
|
||||
# effect: the UI must bind loopback too, so Exec sets the listen-addr to
|
||||
# 127.0.0.1 (the old rootless unit used 0.0.0.0 only because container loopback
|
||||
# was not host loopback; under Network=host they are the same address, so the
|
||||
# unauthenticated UI stays loopback-only — nothing off-host reaches it).
|
||||
#
|
||||
# NETWORK. It joins dankclaw_default so its config's loki.write endpoint
|
||||
# resolves the container name loki:3100.
|
||||
# THE /etc/machine-id MOUNT IS LOAD-BEARING. sd_journal locates the journal
|
||||
# directory by the local machine id; without it the container opens nothing and
|
||||
# the journal source reads exactly 0 lines — silently, no error, health green.
|
||||
# This mount is the entire reason the pipeline carries data. Never remove it.
|
||||
#
|
||||
# THE UI PORT EXISTS ON PURPOSE. Alloy's UI listens on the container's own
|
||||
# 127.0.0.1 by default and cannot be moved from the config file (alloy 1.x
|
||||
# refuses the ui block), so the compose service had no reachable health
|
||||
# endpoint at all: the homepage status card for alloy reported "down"
|
||||
# continuously and there was nothing for a deploy to wait on. The command
|
||||
# below therefore moves the UI to the container's wildcard address and the
|
||||
# port is published on host loopback only, which puts alloy in the same class
|
||||
# as the other three: reachable from the host, by Caddy over the private
|
||||
# network, and by nothing off-host. Alloy has no authentication, which is
|
||||
# exactly why it is not published wider.
|
||||
#
|
||||
# SOURCES. The host journald socket and the host log/container-storage trees
|
||||
# are bind-mounted read-only; Alloy tails them and pushes to Loki. The config
|
||||
# lives at /opt/alloy/config/config.alloy, installed from
|
||||
# shared/alloy/config.alloy on every deploy.
|
||||
#
|
||||
# ORDERING. Wants/After loki.service: without a reachable Loki, Alloy's write
|
||||
# client fails and its log pipeline backs up.
|
||||
# IMAGE. Built rootless by build_compose_image alloy, then mirrored into the
|
||||
# root image store by the deployer (a root build runs in the build container's
|
||||
# own resolver and fails on DNS here). Pull=never: boot must not reach for the
|
||||
# local tag at a registry.
|
||||
|
||||
[Unit]
|
||||
Description=Alloy log shipper (Dankclaw)
|
||||
After=network-online.target loki.service
|
||||
Wants=network-online.target loki.service
|
||||
After=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=alloy
|
||||
Image=localhost/franksdank/alloy:latest
|
||||
Pull=never
|
||||
Network=dankclaw_default
|
||||
Exec=run --server.http.listen-addr=0.0.0.0:12345 /etc/alloy/config.alloy
|
||||
PublishPort=127.0.0.1:12345:12345
|
||||
Volume=/opt/alloy/config:/etc/alloy
|
||||
Network=host
|
||||
Exec=run --server.http.listen-addr=127.0.0.1:12345 /etc/alloy/config.alloy
|
||||
Volume=/opt/alloy/config:/etc/alloy:ro
|
||||
Volume=/etc/machine-id:/etc/machine-id:ro
|
||||
Volume=/run/systemd/journal/socket:/run/systemd/journal/socket:ro
|
||||
Volume=/var/log:/var/log:ro
|
||||
Volume=/var/lib/containers:/var/lib/containers:ro
|
||||
HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:12345/-/healthy
|
||||
Volume=/var/log/journal:/var/log/journal:ro
|
||||
Volume=/run/log/journal:/run/log/journal:ro
|
||||
HealthCmd=/usr/local/bin/busybox wget --spider -q http://127.0.0.1:12345/-/healthy
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
@@ -60,4 +56,4 @@ Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
WantedBy=multi-user.target
|
||||
|
||||
+72
-13
@@ -487,13 +487,15 @@ seed_grafana_env() {
|
||||
|| { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; }
|
||||
}
|
||||
|
||||
# The five observability containers left compose on 2026-09-11 (they were
|
||||
# started by hand with podman-compose and never came back after a reboot).
|
||||
# They are ordinary rootless Quadlet units now: same images, same mounts, same
|
||||
# arguments, same container names, so nothing that addresses them by name —
|
||||
# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
|
||||
# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
|
||||
# compose services published on 0.0.0.0 and none of them authenticates.
|
||||
# The observability containers left compose on 2026-09-11 (they were started by
|
||||
# hand with podman-compose and never came back after a reboot). Five are ordinary
|
||||
# rootless Quadlet units: same images, same mounts, same arguments, same
|
||||
# container names, so nothing that addresses them by name — Caddy, Prometheus'
|
||||
# own scrape config — changed. Only the published ports did: the three HTTP UIs
|
||||
# bind 127.0.0.1, because the compose services published on 0.0.0.0 and none of
|
||||
# them authenticates. Alloy is the exception: a ROOT system service on
|
||||
# Network=host, because reading the host journal needs privileges a rootless
|
||||
# container cannot get. It is handled by deploy_alloy, not the loop below.
|
||||
deploy_observability() {
|
||||
local service="$1"
|
||||
# Observability units live under <target>/observability/quadlet: Dankclaw
|
||||
@@ -512,10 +514,6 @@ deploy_observability() {
|
||||
copy_loki_config
|
||||
build_compose_image loki
|
||||
;;
|
||||
alloy)
|
||||
copy_alloy_config
|
||||
build_compose_image alloy
|
||||
;;
|
||||
grafana)
|
||||
copy_grafana_provisioning
|
||||
seed_grafana_env || return 1
|
||||
@@ -534,11 +532,71 @@ deploy_observability() {
|
||||
alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
|
||||
grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
|
||||
loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
|
||||
alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
|
||||
node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Mirror an image the deployer built rootless into the ROOT image store. A root
|
||||
# quadlet reads root storage; a root build (build_compose_image is rootless) is
|
||||
# not possible on this host — the build container's resolver fails DNS — so the
|
||||
# only way to get a root-readable copy is save-and-load. Compares image ids so a
|
||||
# rebuilt alloy actually re-lands in root storage instead of leaving a stale tag.
|
||||
mirror_image_to_root() {
|
||||
local ref="$1" rid
|
||||
rid="$(podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" || {
|
||||
echo "ERROR: rootless image $ref is not built; cannot mirror to root" >&2
|
||||
return 1
|
||||
}
|
||||
if [[ "$(run_privileged podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" == "$rid" ]]; then
|
||||
return 0
|
||||
fi
|
||||
local tar; tar="$(mktemp /var/tmp/alloy-mirror.XXXXXX.tar)"
|
||||
podman save -o "$tar" "$ref"
|
||||
run_privileged podman load -i "$tar"
|
||||
rm -f "$tar"
|
||||
}
|
||||
|
||||
# Install a quadlet into the SYSTEM generator dir (root manager) instead of the
|
||||
# user dir. Boot persistence is the unit's own [Install] section (multi-user
|
||||
# target); the v1 system generator creates the .wants on daemon-reload, so —
|
||||
# like the user units — no `systemctl enable` (generated units cannot be enabled
|
||||
# here).
|
||||
install_system_quadlet() {
|
||||
local source_dir="$1" unit="$2"
|
||||
run_privileged mkdir -p /etc/containers/systemd
|
||||
run_privileged cp "$source_dir/$unit.container" /etc/containers/systemd/
|
||||
run_privileged systemctl daemon-reload
|
||||
}
|
||||
|
||||
# Alloy as a ROOT system service on Network=host. See the unit file: a rootless
|
||||
# container cannot read the 0640 journal files, and it cannot reach Loki's
|
||||
# 127.0.0.1:3100 publish from a bridge netns. Root + host network fixes both;
|
||||
# the price is that this one container runs as root and on the host network.
|
||||
deploy_alloy() {
|
||||
copy_alloy_config
|
||||
build_compose_image alloy # rootless build (resolver works)
|
||||
mirror_image_to_root localhost/franksdank/alloy:latest
|
||||
|
||||
# Retire the rootless unit BEFORE the system one starts: both managers
|
||||
# generate alloy.service, and two containers named "alloy" both binding
|
||||
# 127.0.0.1:12345 cannot coexist. Move the user quadlet out of the generator
|
||||
# dir (it regenerates from there) and stop the user unit.
|
||||
if [[ -f "$quadlet_unit_dir/alloy.container" ]] ||
|
||||
systemctl --user is-active --quiet alloy.service 2>/dev/null; then
|
||||
systemctl --user stop alloy.service 2>/dev/null || true
|
||||
if [[ -f "$quadlet_unit_dir/alloy.container" ]]; then
|
||||
local trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-alloy-root-cutover"
|
||||
mkdir -p "$trash"
|
||||
mv -- "$quadlet_unit_dir/alloy.container" "$trash/alloy.container"
|
||||
echo "retired rootless alloy.container -> $trash"
|
||||
fi
|
||||
systemctl --user daemon-reload
|
||||
fi
|
||||
|
||||
install_system_quadlet dankclaw/observability/quadlet alloy
|
||||
run_privileged systemctl restart alloy.service
|
||||
wait_for_health alloy http://127.0.0.1:12345/-/healthy 60
|
||||
}
|
||||
build_compose_image() {
|
||||
local service="$1" image cfhash oldhash containerfile
|
||||
# Every file the image COPIES IN, beside the Containerfile. The skip below
|
||||
@@ -1941,7 +1999,8 @@ deploy_service() {
|
||||
copyparty) deploy_copyparty ;;
|
||||
plex) deploy_plex ;;
|
||||
tautulli) deploy_tautulli ;;
|
||||
prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
|
||||
alloy) deploy_alloy ;;
|
||||
prometheus|grafana|loki|node-exporter|alertmanager) deploy_observability "$service" ;;
|
||||
comfyui) deploy_comfyui ;;
|
||||
*) deploy_compose_service "$service" ;;
|
||||
esac
|
||||
|
||||
@@ -3,96 +3,111 @@ type: reference
|
||||
description: "Alloy is Grafana's log and telemetry shipper."
|
||||
tags:
|
||||
- infrastructure
|
||||
timestamp: 2026-09-11
|
||||
timestamp: 2026-10-20
|
||||
---
|
||||
|
||||
# Alloy
|
||||
|
||||
Alloy is Grafana's log and telemetry shipper. It replaces Promtail (end of life as of 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/). It is wired to forward container and journal logs to Loki, and every unit in the path is healthy — but as of 2026-10-18 it is delivering **zero** log lines. See "Delivery state" below.
|
||||
Alloy is Grafana's log and telemetry shipper. It replaces Promtail (end of life
|
||||
as of 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/). It
|
||||
forwards the host journal — including the rootless Quadlet containers' logs —
|
||||
to Loki. Unlike its observability siblings it runs as a **root system service on
|
||||
the host network**, because a rootless container cannot read the journal (see
|
||||
*Why root* below).
|
||||
|
||||
**Host:** Dankclaw (one instance per host) — **Port:** 12345 (loopback) — **Docs:** https://grafana.com/docs/alloy/latest
|
||||
**Host:** Dankclaw (one instance per host) — **UI/health:** 127.0.0.1:12345
|
||||
(loopback) — **Docs:** https://grafana.com/docs/alloy/latest
|
||||
|
||||
## Facts
|
||||
|
||||
| Item | Value |
|
||||
|------|-------|
|
||||
| Image | `franksdank/alloy:latest` (local build: upstream + a static busybox for the healthcheck) |
|
||||
| Published | `127.0.0.1:12345` → 12345 (its own UI/health port) |
|
||||
| Unit | `dankclaw/observability/quadlet/alloy.container` → `alloy.service` |
|
||||
| Config | `shared/alloy/config.alloy`, installed to `/opt/alloy/config/config.alloy` on every deploy (`scripts/deploy-remote.sh:281-284`) |
|
||||
| Inputs | host journald socket, podman container logs, `/var/log` |
|
||||
| Output | `http://loki:3100/loki/api/v1/push` (`shared/alloy/config.alloy:13-17`) |
|
||||
| Successor to | Promtail (EOL 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/) |
|
||||
| Boot persistence | `Restart=always`/`RestartSec=10` + `WantedBy=default.target` (`dankclaw/observability/quadlet/alloy.container:58-63`); installed at `~/.config/containers/systemd/alloy.container`, reporting `generated` + `active` (`systemctl --user is-enabled|is-active alloy` 2026-10-18) |
|
||||
| Delivery | **Configured and healthy, currently delivering zero data** — see "Delivery state" |
|
||||
| Image | `localhost/franksdank/alloy:latest` — built rootless, then mirrored into the root image store by the deployer (a root build fails on DNS in this host's build container) |
|
||||
| Unit | `dankclaw/observability/quadlet/alloy.container` → system `alloy.service`, installed at `/etc/containers/systemd/` (root manager) |
|
||||
| Config | `shared/alloy/config.alloy` → `/opt/alloy/config/config.alloy` on every deploy |
|
||||
| Network | `host` — required so it can reach Loki's `127.0.0.1:3100` publish |
|
||||
| UI | `--server.http.listen-addr=127.0.0.1:12345` (loopback; nothing off-host reaches it) |
|
||||
| Inputs | host journal: `system.journal` + `user-*.journal` (root reads both) |
|
||||
| Output | `http://127.0.0.1:3100/loki/api/v1/push` |
|
||||
| Boot persistence | `Restart=always`/`RestartSec=10` + `WantedBy=multi-user.target`; boot persistence is the `[Install]` section (generated system units cannot be `enable`d here) |
|
||||
| Successor to | Promtail (EOL 2026-03-02) |
|
||||
| Deploy status | root cutover built and tested in-repo (2026-10-20); the **live** instance is still the old rootless unit until `./scripts/deploy.sh dankclaw --service alloy` runs |
|
||||
|
||||
## Why Alloy and not Promtail
|
||||
## Why root (the load-bearing decision)
|
||||
|
||||
Promtail reached end-of-life on 2026-03-02 and Grafana Labs directs new installs to Alloy. This stack uses Alloy from day one: no Promtail config or unit has ever existed in this repository (`git log -S promtail` over the whole history returns no commit that added one; the compose-era service of the same name was already Alloy — `dankclaw/observability/quadlet/alloy.container:6-7`).
|
||||
A rootless Alloy delivered **zero** lines, silently and with a green health
|
||||
check. Two independent causes, both proven on the host 2026-10-20:
|
||||
|
||||
1. **The journal is not readable by a rootless container.** The files are
|
||||
`root:systemd-journal 0640` (`group:adm`, `other::---`). Even bind-mounted, a
|
||||
rootless container has no supplementary groups, so opening them returns
|
||||
`DENIED`. `loki.source.journal` then sees an empty view and the counter just
|
||||
sits at 0 — no error logged.
|
||||
2. **Container logs live in the *user* journal, not `ctr.log`.** The rootless
|
||||
Quadlet containers log via the journald driver into `user-1000.journal`; the
|
||||
old file globs (`/var/lib/containers/.../ctr.log`, `/var/log/containers/*.log`)
|
||||
point at a root-storage layout that does not exist here.
|
||||
|
||||
**Root fixes both at once.** As root the container opens every journal file
|
||||
under the machine-id directory — `system.journal` *and* `user-1000.journal` —
|
||||
so a single `loki.source.journal` carries host services and the rootless
|
||||
containers' logs together. Validated with a throwaway root probe before writing
|
||||
the unit: **31,825 journal lines read, 26,966 pushed to Loki in 20 s, zero
|
||||
drops**, and Loki's label set went from empty to populated.
|
||||
|
||||
### The two mounts that make it work
|
||||
|
||||
- **`/etc/machine-id` (ro) — load-bearing.** `sd_journal` locates the journal
|
||||
directory by the *local* machine id. Without it the container opens nothing and
|
||||
the journal source reads exactly 0 lines — silently, health green. This single
|
||||
mount is the whole difference between a working and an empty pipeline; do not
|
||||
remove it. (A root probe without it reads 0; with it, 31,825.)
|
||||
- **`Network=host`.** Loki publishes `127.0.0.1:3100` (loopback only). No bridge
|
||||
container — rootless or root — can reach the host's loopback, so Alloy must
|
||||
share the host network to push to `127.0.0.1:3100`. Consequence: the UI binds
|
||||
`127.0.0.1` directly (no `PublishPort`), so it stays loopback-only as before.
|
||||
|
||||
## Configuration
|
||||
|
||||
`shared/alloy/config.alloy` is the source of truth:
|
||||
|
||||
- Reads journald logs from the host's `/run/systemd/journal/socket`, labelled `job="systemd-journal"` (`shared/alloy/config.alloy:20-26`).
|
||||
- Reads container logs from two globs: `/var/lib/containers/storage/overlay-containers/*/userdata/ctr.log` (`job="podman"`) and `/var/log/containers/*.log` (`job="containers"`) (`config.alloy:29-35`).
|
||||
- Forwards everything to `http://loki:3100/loki/api/v1/push` by container name on `dankclaw_default` (`config.alloy:13-17`).
|
||||
`shared/alloy/config.alloy` is the source of truth — one `loki.source.journal`
|
||||
(labelled `job="systemd-journal"`) forwarding to the loopback `loki.write`
|
||||
endpoint. There is deliberately **no** container-file source: container stdout is
|
||||
in the journal.
|
||||
|
||||
## Volumes
|
||||
|
||||
| Container path | Host path |
|
||||
|----------------|-----------|
|
||||
| `/etc/alloy` | `/opt/alloy/config` |
|
||||
| `/run/systemd/journal/socket` | host journald socket (read-only) |
|
||||
| `/var/log` | host logs (read-only) |
|
||||
| `/var/lib/containers` | podman storage (read-only) |
|
||||
| `/etc/alloy` | `/opt/alloy/config` (ro) |
|
||||
| `/etc/machine-id` | host machine id (ro) — **required** |
|
||||
| `/run/systemd/journal/socket` | host journald socket (ro) |
|
||||
| `/var/log/journal` | persistent journal files (ro) |
|
||||
| `/run/log/journal` | runtime journal files (ro) |
|
||||
|
||||
## The UI port is published on purpose
|
||||
## The UI stays on loopback
|
||||
|
||||
Alloy's UI listens on the container's own `127.0.0.1:12345` by default, and Alloy 1.x refuses to configure the `ui` service from a config block ("service ui does not support being configured", `shared/alloy/config.alloy:8-10`). Under compose that meant **no reachable health endpoint at all**: the homepage status card for alloy read `down` continuously and a deploy had nothing to wait on. The unit therefore passes `--server.http.listen-addr=0.0.0.0:12345` (`dankclaw/observability/quadlet/alloy.container:46`) and publishes `127.0.0.1:12345` (`alloy.container:47`, confirmed by `ss -ltnp` 2026-10-18) — same posture as the other three: reachable from the host and from Caddy's network, and from nothing off-host. Alloy has no authentication, which is exactly why it is not published wider. The homepage card probes that same loopback port (`shared/homepage/services.json:233`).
|
||||
|
||||
## Delivery state (2026-10-20): healthy, shipping nothing
|
||||
|
||||
The unit is `active`, the container reports `healthy`, the log says `Alloy is
|
||||
ready`, and the pipeline carries no data at all.
|
||||
|
||||
**Root cause, proven: a rootless container cannot read the host journal.** The
|
||||
journal files are `root:systemd-journal 0640` (with `group:adm`, `other::---`).
|
||||
`/var/log` is bind-mounted so the files are *visible*, but a rootless container
|
||||
spawns with no supplementary groups, so it is denied. Proven directly:
|
||||
`podman run --rm -v /var/log/journal:/var/log/journal:ro busybox cat
|
||||
/var/log/journal/*/system.journal` → `DENIED`. `loki.source.journal` opens via
|
||||
`sd_journal_open`, so it silently sees an empty view — no error logged, counter
|
||||
frozen at `loki_source_journal_target_lines_total 0` (`system.journal` and
|
||||
`user-1000.journal` are both present and non-trivial, 2026-10-20).
|
||||
|
||||
**Second, independent gap:** the rootless Quadlet containers log to the *user*
|
||||
journal (`user-1000.journal`), and `loki.source.journal` reads the *system*
|
||||
journal only — a fixed system read would still miss them. Container stdout is
|
||||
not in a `ctr.log` here either: the file globs point at root storage
|
||||
(`/var/lib/containers/storage/.../ctr.log`, `/var/log/containers/*.log`) that
|
||||
rootless does not use, refused at start with `stat …: permission denied` / `no
|
||||
such file or directory`, `loki_source_file_files_active_total 0`.
|
||||
|
||||
Downstream: Loki holds zero streams (`GET /loki/api/v1/labels` → no `data`) —
|
||||
see [[Loki]].
|
||||
|
||||
**Closing it is a permission/architecture decision, not a config edit.** The
|
||||
unit, config and push endpoint are correct as written; rootless Podman simply
|
||||
cannot grant the journal group. The viable shapes: run Alloy as a *system*
|
||||
service (root, in `adm`/`systemd-journal`) instead of a rootless Quadlet; grant
|
||||
the container's mapped uid journal read via a tmpfiles/ACL rule; or have
|
||||
journald forward to a path the container can read. Each trades rootless posture
|
||||
for reach — decide deliberately. Re-check `loki_source_journal_target_lines_total`
|
||||
and the Loki label query to confirm a fix, never `/healthy`: health is exactly
|
||||
what hides this failure.
|
||||
Alloy 1.x refuses to configure the `ui` service from a config block, so the UI
|
||||
address is set on the command line instead. Under the old rootless unit that
|
||||
meant `0.0.0.0` inside the container plus a published `127.0.0.1:12345`; under
|
||||
`Network=host` the listen-addr is `127.0.0.1:12345` directly — the host's
|
||||
loopback, reachable from the host and from nothing off-host. Alloy has no
|
||||
authentication, which is exactly why it is never published wider. The homepage
|
||||
card and the deploy health gate both dial `http://127.0.0.1:12345/-/healthy`.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- The journald socket mount is required for journal scraping.
|
||||
- Container log paths differ between docker (`/var/lib/docker/containers`) and podman (`/var/lib/containers/storage/...`); point the config at the right one per host.
|
||||
- One Alloy per host; do not run a single instance that tries to reach both hosts' logs.
|
||||
- The unit `Wants=`/`After=` `loki.service`: without a reachable Loki the write client fails and the pipeline backs up.
|
||||
- **No `/etc/machine-id` mount ⇒ silent zero.** The health check still passes.
|
||||
This is the failure mode that hid the broken pipeline; check
|
||||
`loki_source_journal_target_lines_total` and the Loki label query, never
|
||||
`/healthy`.
|
||||
- **`After=loki.service` is gone.** Alloy is on the system manager; Loki is a
|
||||
rootless user unit — a system unit cannot order against a user unit. Alloy
|
||||
retries its push until Loki answers, so ordering is unnecessary.
|
||||
- **The image must be in the ROOT store.** The deployer builds rootless and
|
||||
mirrors (`mirror_image_to_root`); a `Pull=never` root quadlet with only a
|
||||
rootless tag fails at create.
|
||||
- One Alloy per host; do not run a single instance that tries to reach both
|
||||
hosts' logs.
|
||||
|
||||
## See Also
|
||||
|
||||
|
||||
@@ -102,17 +102,18 @@ verify it through the APIs.
|
||||
installed from `shared/grafana/provisioning/`. The Loki datasource is how
|
||||
you read logs: Loki has no web UI of its own, so Grafana Explore is the
|
||||
human surface (the guarded `loki.franksdank.cloud` vhost serves only the API).
|
||||
* **Logs: configured, empty.** Alloy ships zero lines and Loki holds zero
|
||||
streams (`GET /loki/api/v1/labels` returns no label names; no chunk files on
|
||||
disk). The cause is source-side access, now proven, not a config bug: a
|
||||
rootless container cannot read the journal files (`system.journal` is
|
||||
`root:systemd-journal 0640`/`adm` and a rootless read of it returns DENIED),
|
||||
and the container-log globs point at a root-storage path that does not exist
|
||||
under rootless. Closing it is a permission/architecture decision, not an
|
||||
edit — see [[Alloy]] → Delivery state and [[Loki]] → Contents.
|
||||
* **Logs: fixed in code, awaiting deploy.** The pipeline shipped nothing because
|
||||
a rootless Alloy cannot read the `0640` journal files, and the container-log
|
||||
globs point at a root-storage layout that does not exist here. The fix is built
|
||||
and tested in-repo (2026-10-20): Alloy is now a **root system service on
|
||||
`Network=host`**, reading `system.journal` + `user-*.journal` and pushing to
|
||||
Loki's `127.0.0.1:3100` loopback publish — a throwaway root probe read 31,825
|
||||
lines and pushed 26,966 with zero drops. The **live** instance is still the old
|
||||
rootless unit until `deploy.sh dankclaw --service alloy` runs. Detail in
|
||||
[[Alloy]] → Why root; Loki's store state in [[Loki]] → Contents.
|
||||
* **Not configured:** no blackbox exporter (the structural downs have no real
|
||||
uptime signal) and no Alertmanager vhost. The log pipeline is provisioned
|
||||
but carries no data (see *Logs* above).
|
||||
uptime signal) and no Alertmanager vhost. The log pipeline's fix is in code but
|
||||
not yet deployed (see *Logs* above).
|
||||
|
||||
## Citations
|
||||
|
||||
|
||||
+18
-15
@@ -1,22 +1,34 @@
|
||||
// Alloy configuration — ships host and container logs to Loki.
|
||||
// One instance per host. Deployed to /opt/alloy/config/config.alloy.
|
||||
//
|
||||
// Runs as a ROOT system service on Network=host (see
|
||||
// dankclaw/observability/quadlet/alloy.container), so it reaches Loki on the
|
||||
// host's loopback and reads every journal — host services and rootless-quadlet
|
||||
// containers alike.
|
||||
|
||||
logging {
|
||||
level = "info"
|
||||
}
|
||||
|
||||
// The alloy UI service listens on 127.0.0.1:12345 by default; the
|
||||
// in-container healthcheck probes it there. alloy 1.x refuses to
|
||||
// configure the ui service from a config block, so there is no block.
|
||||
// The alloy UI listens on 127.0.0.1:12345; the in-container healthcheck probes
|
||||
// it there. alloy 1.x refuses to configure the ui service from a config block,
|
||||
// so there is no ui block.
|
||||
|
||||
// Ship logs to Loki (container name on the podman network).
|
||||
// Loki is published on the host's loopback (127.0.0.1:3100). Alloy shares the
|
||||
// host network, so it pushes by loopback, never by container name — a root
|
||||
// container is not on the rootless dankclaw_default network.
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "http://loki:3100/loki/api/v1/push"
|
||||
url = "http://127.0.0.1:3100/loki/api/v1/push"
|
||||
}
|
||||
}
|
||||
|
||||
// Systemd journal from the host socket.
|
||||
// The systemd journal. As root with /etc/machine-id + the journal dirs mounted,
|
||||
// this one source reads system.journal AND user-*.journal, so it carries both
|
||||
// host services and the rootless Quadlet containers' logs. There is deliberately
|
||||
// no container-file source: podman container stdout lands in the journal, and
|
||||
// the old /var/lib/containers/.../ctr.log and /var/log/containers globs pointed
|
||||
// at a root-storage layout that does not exist on this host.
|
||||
loki.source.journal "journal" {
|
||||
format_as_json = true
|
||||
labels = {
|
||||
@@ -24,12 +36,3 @@ loki.source.journal "journal" {
|
||||
}
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
// Container logs (podman storage path; adjust for docker).
|
||||
loki.source.file "containers" {
|
||||
targets = [
|
||||
{__path__ = "/var/lib/containers/storage/overlay-containers/*/userdata/ctr.log", job = "podman"},
|
||||
{__path__ = "/var/log/containers/*.log", job = "containers"},
|
||||
]
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
@@ -240,14 +240,12 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
|
||||
}
|
||||
|
||||
@test "alloy's UI address comes from the unit, not the config file" {
|
||||
# alloy 1.x refuses to configure the ui service from a config block
|
||||
# ("service ui does not support being configured"), so the unit moves the
|
||||
# UI off the container's loopback with a command-line flag. Without that
|
||||
# flag the port cannot be published and the service has no reachable
|
||||
# health endpoint at all — which is why the homepage status card for
|
||||
# alloy read "down" while it ran under compose.
|
||||
# alloy 1.x refuses to configure the ui service from a config block, so the
|
||||
# unit sets the UI address on the command line. As a Network=host root unit
|
||||
# that address is the host loopback directly (no PublishPort) — reachable
|
||||
# from the host, from nothing off-host.
|
||||
! grep -qE '^(ui|server) \{' "$REPO_ROOT/shared/alloy/config.alloy"
|
||||
local unit="$REPO_ROOT/dankclaw/observability/quadlet/alloy.container"
|
||||
grep -q -- '--server.http.listen-addr=0.0.0.0:12345' "$unit"
|
||||
grep -q '^PublishPort=127.0.0.1:12345:12345$' "$unit"
|
||||
grep -q -- '--server.http.listen-addr=127.0.0.1:12345' "$unit"
|
||||
! grep -q '^PublishPort=' "$unit"
|
||||
}
|
||||
|
||||
@@ -41,13 +41,17 @@ service_section() {
|
||||
}
|
||||
|
||||
@test "every observability unit is boot-persistent" {
|
||||
# Generated units cannot be enabled on this host's podman 5.x v1 quadlet
|
||||
# generator; [Install] + linger is the entire boot-persistence story.
|
||||
# Generated units cannot be enabled on this host's podman quadlet generator;
|
||||
# [Install] + linger is the boot-persistence story for the rootless five.
|
||||
# Alloy is a ROOT system unit, so its [Install] targets multi-user.target.
|
||||
local svc
|
||||
for svc in "${UNITS[@]}"; do
|
||||
[[ "$svc" == alloy ]] && continue
|
||||
grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc is not boot-persistent"; return 1; }
|
||||
done
|
||||
grep -q '^WantedBy=multi-user.target$' "$UNIT_DIR/alloy.container" \
|
||||
|| { echo "alloy (root unit) is not boot-persistent"; return 1; }
|
||||
}
|
||||
|
||||
@test "every observability unit is self-healing" {
|
||||
@@ -140,15 +144,20 @@ service_section() {
|
||||
grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile"
|
||||
grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile"
|
||||
local svc
|
||||
for svc in prometheus grafana loki alloy; do
|
||||
for svc in prometheus grafana loki; do
|
||||
grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc unit renames its container"; return 1; }
|
||||
grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc is not on the network Caddy and its peers use"; return 1; }
|
||||
done
|
||||
# Alloy is not on that network: it is a root container on Network=host, the
|
||||
# only way to reach Loki's 127.0.0.1:3100 publish from outside a bridge.
|
||||
grep -q '^ContainerName=alloy$' "$UNIT_DIR/alloy.container"
|
||||
grep -q '^Network=host$' "$UNIT_DIR/alloy.container" \
|
||||
|| { echo "alloy must be Network=host to reach Loki on the host loopback"; return 1; }
|
||||
grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
||||
grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
||||
grep -q 'url = "http://loki:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
|
||||
grep -q 'url = "http://127.0.0.1:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
|
||||
}
|
||||
|
||||
@test "every observability unit has a health check" {
|
||||
@@ -239,12 +248,14 @@ service_section() {
|
||||
}
|
||||
|
||||
@test "startup order follows the data dependencies" {
|
||||
# Grafana is useless without Prometheus and Alloy drops logs without Loki;
|
||||
# both pairs are expressed as systemd ordering, not launch luck.
|
||||
# Grafana is useless without Prometheus (both rootless, same manager). Alloy
|
||||
# used to order on loki.service; as a ROOT unit it lives on the SYSTEM
|
||||
# manager and cannot order against a rootless user unit, so it depends only
|
||||
# on the network and retries its push until Loki answers.
|
||||
grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
||||
grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
||||
grep -q 'After=network-online.target loki.service' "$UNIT_DIR/alloy.container"
|
||||
grep -q 'Wants=network-online.target loki.service' "$UNIT_DIR/alloy.container"
|
||||
grep -q '^After=network-online.target$' "$UNIT_DIR/alloy.container"
|
||||
! grep -q 'loki.service' "$UNIT_DIR/alloy.container"
|
||||
}
|
||||
|
||||
@test "the deployer installs, restarts and health-checks each unit" {
|
||||
@@ -281,6 +292,24 @@ PAIRS
|
||||
grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "alloy is a root host-network unit that reads every journal" {
|
||||
# The whole point of the root cutover: a rootless container cannot open the
|
||||
# 0640 journal files. Pin the properties that make the pipeline carry data.
|
||||
local u="$UNIT_DIR/alloy.container"
|
||||
grep -q '^Network=host$' "$u"
|
||||
grep -q '^Volume=/etc/machine-id:/etc/machine-id:ro$' "$u" \
|
||||
|| { echo "alloy missing the load-bearing machine-id mount"; return 1; }
|
||||
grep -q '^Volume=/var/log/journal:/var/log/journal:ro$' "$u"
|
||||
grep -q '^Volume=/run/log/journal:/run/log/journal:ro$' "$u"
|
||||
grep -q -- '--server.http.listen-addr=127.0.0.1:12345' "$u"
|
||||
# Deployer: system install + root image mirror, not the user quadlet loop.
|
||||
grep -q '^deploy_alloy() {' "$REMOTE_DEPLOY"
|
||||
grep -q 'install_system_quadlet dankclaw/observability/quadlet alloy' "$REMOTE_DEPLOY"
|
||||
grep -q '/etc/containers/systemd/' "$REMOTE_DEPLOY"
|
||||
grep -q 'mirror_image_to_root localhost/franksdank/alloy:latest' "$REMOTE_DEPLOY"
|
||||
grep -q 'alloy) deploy_alloy ;;' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "no dankclaw compose file competes with the quadlet units" {
|
||||
# Two owners for one container name is how a compose run resurrects a
|
||||
# stale duplicate next to the quadlet-managed one. This file was retired on
|
||||
|
||||
Reference in New Issue
Block a user