feat(observability): make the five Dankclaw observability containers quadlets
prometheus, grafana, loki, alloy and node-exporter ran only from dankclaw/compose.yaml. Rootless podman does not restart bare containers, so they did not survive the 2026-09-11 reboot for the amd_iommu=off change and had to be restored by hand — the same gap that had already cost the fleet Caddy and Pi-hole, which is why the boot-persistence rule exists. Each is now a rootless Quadlet unit under dankclaw/observability/quadlet/, installed and dispatched by deploy_observability in deploy-remote.sh, with the compose definitions retired so only one owner exists per container name. Images, mounts, arguments and container names are unchanged, so Caddy's proxies and Prometheus' own scrape config keep resolving. Two posture changes, both about the unauthenticated surface: * The three HTTP UIs published on 0.0.0.0 with no authentication of any kind (Prometheus even with the lifecycle endpoint enabled). They now publish on 127.0.0.1: the browser path is the guarded franksdank.cloud vhost, Caddy reaches them by name over dankclaw_default, and the host-side homepage status timer probes loopback. Nothing legitimately used the LAN port. * node-exporter bound :9100 on every interface; it now binds exactly its two consumers, loopback (homepage timer) and the LAN address (Prometheus, which runs in a container and cannot reach the host's loopback). alloy gains a reachable health endpoint: its UI listens on the container's own loopback and alloy 1.x refuses a config block for it, so the compose service had nothing to probe — the homepage status card for alloy read 'down' continuously. The unit moves the UI with a command-line flag and publishes 127.0.0.1:12345. grafana keeps its Authentik OAuth client secret without receiving the whole .env: systemd expands the value from a [Service] EnvironmentFile, which never reaches the container.
This commit is contained in:
Frank
@@ -300,6 +300,16 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
# its unit and fleet settings live in one .container file.
|
||||
_add(services, "frank-large")
|
||||
return services
|
||||
if len(parts) >= 2 and parts[1] == "observability":
|
||||
# The five observability containers became Quadlet units on
|
||||
# 2026-09-11. A unit change selects exactly its own service;
|
||||
# SERVICE_ORDER already puts each dependency ahead of its
|
||||
# consumer (prometheus before grafana, loki before alloy).
|
||||
if len(parts) >= 4 and parts[2] == "quadlet":
|
||||
name = parts[3].removesuffix(".container")
|
||||
if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter"}:
|
||||
_add(services, name)
|
||||
return services
|
||||
if len(parts) >= 2 and parts[1] == "gitea":
|
||||
if len(parts) >= 4 and parts[2] == "quadlet":
|
||||
name = parts[3]
|
||||
|
||||
+42
-11
@@ -363,20 +363,11 @@ deploy_compose_service() {
|
||||
return 0
|
||||
fi
|
||||
case "$service" in
|
||||
comfyui|hermes|hermes-dashboard|hermes-webui|prometheus|grafana|loki|alloy|node-exporter)
|
||||
comfyui|hermes|hermes-dashboard|hermes-webui)
|
||||
case "$service" in
|
||||
comfyui|loki|alloy)
|
||||
comfyui)
|
||||
build_compose_image "$service"
|
||||
;;
|
||||
prometheus)
|
||||
copy_prometheus_config
|
||||
;;
|
||||
loki)
|
||||
copy_loki_config
|
||||
;;
|
||||
alloy)
|
||||
copy_alloy_config
|
||||
;;
|
||||
esac
|
||||
podman-compose -f dankclaw/compose.yaml up -d --no-deps "$service"
|
||||
;;
|
||||
@@ -387,6 +378,45 @@ deploy_compose_service() {
|
||||
esac
|
||||
}
|
||||
|
||||
# The five observability containers left compose on 2026-09-11 (they were
|
||||
# started by hand with podman-compose and never came back after a reboot).
|
||||
# They are ordinary rootless Quadlet units now: same images, same mounts, same
|
||||
# arguments, same container names, so nothing that addresses them by name —
|
||||
# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
|
||||
# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
|
||||
# compose services published on 0.0.0.0 and none of them authenticates.
|
||||
deploy_observability() {
|
||||
local service="$1"
|
||||
case "$service" in
|
||||
prometheus)
|
||||
copy_prometheus_config
|
||||
;;
|
||||
loki)
|
||||
copy_loki_config
|
||||
build_compose_image loki
|
||||
;;
|
||||
alloy)
|
||||
copy_alloy_config
|
||||
build_compose_image alloy
|
||||
;;
|
||||
grafana|node-exporter)
|
||||
;;
|
||||
*)
|
||||
echo "unknown observability service: $service" >&2
|
||||
return 2
|
||||
;;
|
||||
esac
|
||||
install_quadlet observability dankclaw/observability/quadlet "$service"
|
||||
systemctl --user restart "$service.service"
|
||||
case "$service" in
|
||||
prometheus) wait_for_health prometheus http://127.0.0.1:9090/-/healthy 60 ;;
|
||||
grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
|
||||
loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
|
||||
alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
|
||||
node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
build_compose_image() {
|
||||
local service="$1" image cfhash oldhash containerfile
|
||||
case "$service" in
|
||||
@@ -1106,6 +1136,7 @@ deploy_service() {
|
||||
copyparty) deploy_copyparty ;;
|
||||
plex) deploy_plex ;;
|
||||
tautulli) deploy_tautulli ;;
|
||||
prometheus|grafana|loki|alloy|node-exporter) deploy_observability "$service" ;;
|
||||
*) deploy_compose_service "$service" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
@@ -107,6 +107,17 @@ for path in "${CHANGED[@]}"; do
|
||||
add_test tests/searxng_quadlet.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
;;
|
||||
# The five observability containers left compose for Quadlet units on
|
||||
# 2026-09-11. Their seam pins boot persistence and the loopback-only
|
||||
# publish posture that keeps them off every host interface. This case
|
||||
# precedes the dankclaw/*/quadlet/*.container fallback below, which
|
||||
# would otherwise swallow it into unrelated seams.
|
||||
dankclaw/observability/*)
|
||||
add_test tests/observability_quadlet.bats
|
||||
add_test tests/caddyfile.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
add_test tests/docs_completeness.bats
|
||||
;;
|
||||
# Caddy left compose.yaml on 2026-08-26 for a quadlet; caddyfile.bats
|
||||
# now asserts against the quadlet itself. Route every change under
|
||||
# dankclaw/caddy/ to both seams (this block must precede the
|
||||
|
||||
Reference in New Issue
Block a user