dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(alloy): run it as a root host-network service so it reads the journal

Browse Source 
A rootless Alloy shipped zero lines: the journal files are
root:systemd-journal 0640 (other::---) and a rootless container carries no
supplementary groups, so sd_journal_open sees an empty view silently; the
rootless quadlet container logs also live in user-1000.journal, not the
ctr.log paths the old config tailed.

As root, one loki.source.journal reads system.journal + user-*.journal
and carries both. Network=host because Loki publishes 127.0.0.1:3100 and
no bridge container can reach the host loopback; the UI binds loopback
directly (no PublishPort). The /etc/machine-id mount is load-bearing —
sd_journal finds the journal dir by the local machine id; without it the
source reads 0 lines, health green.

deploy_alloy routes around the rootless loop: build rootless, mirror the
image into the root store (a root build fails on DNS in the build
container), retire the old rootless alloy unit, install to
/etc/containers/systemd. Validated with a throwaway root probe: 31,825
lines read, 26,966 pushed to Loki in 20s, zero drops.
This commit is contained in:
Frank committed 2026-10-20 19:50:15 -07:00
1 parent 70dd7fbe6d
commit 634fef242b
3 files changed
+125 -67
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/deploy-remote.sh
+72 -13
View File
@@ -487,13 +487,15 @@ seed_grafana_env() {
|| { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; }
}
# The five observability containers left compose on 2026-09-11 (they were
# started by hand with podman-compose and never came back after a reboot).
# They are ordinary rootless Quadlet units now: same images, same mounts, same
# arguments, same container names, so nothing that addresses them by name —
# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
# compose services published on 0.0.0.0 and none of them authenticates.
# The observability containers left compose on 2026-09-11 (they were started by
# hand with podman-compose and never came back after a reboot). Five are ordinary
# rootless Quadlet units: same images, same mounts, same arguments, same
# container names, so nothing that addresses them by name — Caddy, Prometheus'
# own scrape config — changed. Only the published ports did: the three HTTP UIs
# bind 127.0.0.1, because the compose services published on 0.0.0.0 and none of
# them authenticates. Alloy is the exception: a ROOT system service on
# Network=host, because reading the host journal needs privileges a rootless
# container cannot get. It is handled by deploy_alloy, not the loop below.
deploy_observability() {
local service="$1"
# Observability units live under <target>/observability/quadlet: Dankclaw
@@ -512,10 +514,6 @@ deploy_observability() {
copy_loki_config
build_compose_image loki
;;
alloy)
copy_alloy_config
build_compose_image alloy
;;
grafana)
copy_grafana_provisioning
seed_grafana_env || return 1
@@ -534,11 +532,71 @@ deploy_observability() {
alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
esac
}
# Mirror an image the deployer built rootless into the ROOT image store. A root
# quadlet reads root storage; a root build (build_compose_image is rootless) is
# not possible on this host — the build container's resolver fails DNS — so the
# only way to get a root-readable copy is save-and-load. Compares image ids so a
# rebuilt alloy actually re-lands in root storage instead of leaving a stale tag.
mirror_image_to_root() {
local ref="$1" rid
rid="$(podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" || {
echo "ERROR: rootless image $ref is not built; cannot mirror to root" >&2
return 1
}
if [[ "$(run_privileged podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" == "$rid" ]]; then
return 0
fi
local tar; tar="$(mktemp /var/tmp/alloy-mirror.XXXXXX.tar)"
podman save -o "$tar" "$ref"
run_privileged podman load -i "$tar"
rm -f "$tar"
}
# Install a quadlet into the SYSTEM generator dir (root manager) instead of the
# user dir. Boot persistence is the unit's own [Install] section (multi-user
# target); the v1 system generator creates the .wants on daemon-reload, so —
# like the user units — no `systemctl enable` (generated units cannot be enabled
# here).
install_system_quadlet() {
local source_dir="$1" unit="$2"
run_privileged mkdir -p /etc/containers/systemd
run_privileged cp "$source_dir/$unit.container" /etc/containers/systemd/
run_privileged systemctl daemon-reload
}
# Alloy as a ROOT system service on Network=host. See the unit file: a rootless
# container cannot read the 0640 journal files, and it cannot reach Loki's
# 127.0.0.1:3100 publish from a bridge netns. Root + host network fixes both;
# the price is that this one container runs as root and on the host network.
deploy_alloy() {
copy_alloy_config
build_compose_image alloy # rootless build (resolver works)
mirror_image_to_root localhost/franksdank/alloy:latest
# Retire the rootless unit BEFORE the system one starts: both managers
# generate alloy.service, and two containers named "alloy" both binding
# 127.0.0.1:12345 cannot coexist. Move the user quadlet out of the generator
# dir (it regenerates from there) and stop the user unit.
if [[ -f "$quadlet_unit_dir/alloy.container" ]] ||
systemctl --user is-active --quiet alloy.service 2>/dev/null; then
systemctl --user stop alloy.service 2>/dev/null || true
if [[ -f "$quadlet_unit_dir/alloy.container" ]]; then
local trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-alloy-root-cutover"
mkdir -p "$trash"
mv -- "$quadlet_unit_dir/alloy.container" "$trash/alloy.container"
echo "retired rootless alloy.container -> $trash"
fi
systemctl --user daemon-reload
fi
install_system_quadlet dankclaw/observability/quadlet alloy
run_privileged systemctl restart alloy.service
wait_for_health alloy http://127.0.0.1:12345/-/healthy 60
}
build_compose_image() {
local service="$1" image cfhash oldhash containerfile
# Every file the image COPIES IN, beside the Containerfile. The skip below
@@ -1941,7 +1999,8 @@ deploy_service() {
copyparty) deploy_copyparty ;;
plex) deploy_plex ;;
tautulli) deploy_tautulli ;;
prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
alloy) deploy_alloy ;;
prometheus|grafana|loki|node-exporter|alertmanager) deploy_observability "$service" ;;
comfyui) deploy_comfyui ;;
*) deploy_compose_service "$service" ;;
esac