diff --git a/dankclaw/compose.yaml b/dankclaw/compose.yaml index 586d337..d3b7c2a 100644 --- a/dankclaw/compose.yaml +++ b/dankclaw/compose.yaml @@ -143,134 +143,18 @@ services: # not compose services. See gitea/quadlet/ and ../docs/Dankclaw/Gitea.md. # --- Observability ------------------------------------------------------- - prometheus: - image: docker.io/prom/prometheus:latest - container_name: prometheus - restart: unless-stopped - # Rootless: container root maps to the invoking user (dank), which - # owns /opt/prometheus/data. The image's default user (nobody) maps to - # a subuid that cannot write it. - user: "0:0" - command: - - --config.file=/etc/prometheus/prometheus.yml - - --storage.tsdb.path=/prometheus - - --storage.tsdb.retention.time=30d - - --web.enable-lifecycle - ports: - - "9090:9090" - volumes: - - /opt/prometheus/config:/etc/prometheus - - /opt/prometheus/data:/prometheus - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"] - interval: 30s - timeout: 10s - retries: 3 - - grafana: - image: docker.io/grafana/grafana:latest - container_name: grafana - restart: unless-stopped - # Rootless: container root maps to the invoking user (dank), which - # owns /opt/grafana/data. The image's default user (472) maps to a - # subuid that cannot write it. - user: "0:0" - environment: - GF_SERVER_ROOT_URL: "https://grafana.franksdank.cloud" - # Auth model = OAuth2 relying party against authentik (OAuth2Provider - # name "grafana", pk 9, confidential client_type, created via ak shell). - # Chosen over auth_proxy because the embedded outpost carries no provider - # token (token=None), so no identity header ever reaches Grafana — RP mode - # works regardless of outpost topology. The public vhost stays behind the - # Anubis challenge + outpost gate; login happens inside Grafana via the - # Authentik button. Users auto-upsert as Viewer (no role mapping yet); - # the local admin account remains the only Admin. - GF_AUTH_GENERIC_OAUTH_ENABLED: "true" - GF_AUTH_GENERIC_OAUTH_NAME: "Authentik" - GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "grafana-21ca190cde75d60f" - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "${GRAFANA_OAUTH_CLIENT_SECRET}" - # Global authorize path: this fork resolves the provider by client_id - # query param — /application/o//authorize/ does not exist here. - GF_AUTH_GENERIC_OAUTH_AUTH_URL: "https://auth.franksdank.cloud/application/o/authorize/" - GF_AUTH_GENERIC_OAUTH_TOKEN_URL: "https://auth.franksdank.cloud/application/o/token/" - # ?api=1 = introspection-style userinfo: returns all custom claims - # (entitlements/groups) for access tokens, not just the standard set. - GF_AUTH_GENERIC_OAUTH_API_URL: "https://auth.franksdank.cloud/application/o/userinfo/?api=1" - GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: "false" - GF_AUTH_GENERIC_OAUTH_AUTO_UPSERT: "true" - GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_ORG: "true" - GF_AUTH_GENERIC_OAUTH_ORG_ID: "1" - ports: - - "3002:3000" - volumes: - - /opt/grafana/data:/var/lib/grafana - - /opt/grafana/provisioning:/etc/grafana/provisioning:ro - depends_on: - - prometheus - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] - interval: 30s - timeout: 10s - retries: 3 - - loki: - # Local build: grafana/loki + static busybox (healthcheck). - image: franksdank/loki:latest - container_name: loki - restart: unless-stopped - # Rootless: container root maps to the invoking user (dank), which - # owns /opt/loki/data. The image's default user (10001) maps to a - # subuid that cannot write it. - user: "0:0" - command: ["-config.file=/etc/loki/loki-config.yaml"] - ports: - - "3100:3100" - volumes: - - /opt/loki/config:/etc/loki - - /opt/loki/data:/loki - healthcheck: - test: ["CMD", "/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"] - interval: 30s - timeout: 10s - retries: 3 - - alloy: - # Local build: grafana/alloy + static busybox (healthcheck). - image: franksdank/alloy:latest - container_name: alloy - restart: unless-stopped - volumes: - - /opt/alloy/config:/etc/alloy - - /run/systemd/journal/socket:/run/systemd/journal/socket:ro - - /var/log:/var/log:ro - - /var/lib/containers:/var/lib/containers:ro - depends_on: - - loki - healthcheck: - test: ["CMD", "/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:12345/-/healthy"] - interval: 30s - timeout: 10s - retries: 3 - - node-exporter: - image: docker.io/prom/node-exporter:latest - container_name: node-exporter - restart: unless-stopped - network_mode: host - pid: host - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - command: - - --path.procfs=/host/proc - - --path.sysfs=/host/sys - - --path.rootfs=/rootfs - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:9100/metrics"] - interval: 30s - timeout: 10s - retries: 3 + # Now boot-persistent Quadlet units, not compose services: prometheus, + # grafana, loki, alloy and node-exporter install from + # dankclaw/observability/quadlet/ and are dispatched by deploy_observability + # in scripts/deploy-remote.sh. They left compose on 2026-09-11, after a host + # reboot proved that compose-only containers do not come back on their own — + # the same gap that had already cost the fleet Caddy and Pi-hole. + # + # The three HTTP UIs publish on 127.0.0.1 only. Off-host access is through + # the guarded franksdank.cloud vhosts (Caddy reaches these containers by + # name over this network) and the host's own homepage status timer probes + # them over loopback. See + # serverAppConfigs-docs/Infrastructure/Observability.md. # --- Authentik (SSO / OIDC provider) ------------------------------------- # Now boot-persistent Quadlet units, not compose services: authentik- diff --git a/dankclaw/observability/quadlet/alloy.container b/dankclaw/observability/quadlet/alloy.container new file mode 100644 index 0000000..b7815cc --- /dev/null +++ b/dankclaw/observability/quadlet/alloy.container @@ -0,0 +1,63 @@ +# Quadlet unit: alloy (rootless, boot-persistent). +# +# Install path on dankclaw: ~/.config/containers/systemd/alloy.container +# Unit name: alloy.service (container name: alloy) +# +# Log shipper. This replaces the compose service of the same name in +# dankclaw/compose.yaml ("Observability" block). +# +# IMAGE. franksdank/alloy is built locally by deploy-remote.sh +# (build_compose_image) from dankclaw/alloy/Containerfile: upstream +# grafana/alloy plus a static busybox for the healthcheck. Pull=never keeps a +# boot from reaching for that local tag at a registry. +# +# NETWORK. It joins dankclaw_default so its config's loki.write endpoint +# resolves the container name loki:3100. +# +# THE UI PORT EXISTS ON PURPOSE. Alloy's UI listens on the container's own +# 127.0.0.1 by default and cannot be moved from the config file (alloy 1.x +# refuses the ui block), so the compose service had no reachable health +# endpoint at all: the homepage status card for alloy reported "down" +# continuously and there was nothing for a deploy to wait on. The command +# below therefore moves the UI to the container's wildcard address and the +# port is published on host loopback only, which puts alloy in the same class +# as the other three: reachable from the host, by Caddy over the private +# network, and by nothing off-host. Alloy has no authentication, which is +# exactly why it is not published wider. +# +# SOURCES. The host journald socket and the host log/container-storage trees +# are bind-mounted read-only; Alloy tails them and pushes to Loki. The config +# lives at /opt/alloy/config/config.alloy, installed from +# shared/alloy/config.alloy on every deploy. +# +# ORDERING. Wants/After loki.service: without a reachable Loki, Alloy's write +# client fails and its log pipeline backs up. + +[Unit] +Description=Alloy log shipper (Dankclaw) +After=network-online.target loki.service +Wants=network-online.target loki.service + +[Container] +ContainerName=alloy +Image=localhost/franksdank/alloy:latest +Pull=never +Network=dankclaw_default +Exec=run --server.http.listen-addr=0.0.0.0:12345 /etc/alloy/config.alloy +PublishPort=127.0.0.1:12345:12345 +Volume=/opt/alloy/config:/etc/alloy +Volume=/run/systemd/journal/socket:/run/systemd/journal/socket:ro +Volume=/var/log:/var/log:ro +Volume=/var/lib/containers:/var/lib/containers:ro +HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:12345/-/healthy +HealthInterval=30s +HealthTimeout=10s +HealthRetries=3 +HealthStartPeriod=30s + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/dankclaw/observability/quadlet/grafana.container b/dankclaw/observability/quadlet/grafana.container new file mode 100644 index 0000000..c9e31c3 --- /dev/null +++ b/dankclaw/observability/quadlet/grafana.container @@ -0,0 +1,72 @@ +# Quadlet unit: grafana (rootless, boot-persistent). +# +# Install path on dankclaw: ~/.config/containers/systemd/grafana.container +# Unit name: grafana.service (container name: grafana) +# +# Dashboards. This replaces the compose service of the same name in +# dankclaw/compose.yaml ("Observability" block). +# +# PUBLISHING. Bound to 127.0.0.1 for the same reason as prometheus: the browser +# path is https://grafana.franksdank.cloud (Caddy → Anubis → Authentik outpost → +# grafana:3000 over dankclaw_default) and the host-side homepage status timer +# probes 127.0.0.1:3002. Publishing on 0.0.0.0 as the compose service did left +# an off-host path into Grafana that skips the guard chain entirely. +# +# AUTH. Grafana is an OAuth2 relying party against Authentik (OAuth2Provider +# "grafana", confidential client). The guard chain fronts unauthenticated +# traffic; the login itself happens through the Authentik button inside +# Grafana. The client secret is NOT written here: it is expanded by systemd +# from the root .env (EnvironmentFile in [Service], never in [Container], so +# the container receives exactly this one secret and not the whole file). +# Without that .env entry the unit will not start — the same dependency every +# other rendered-secret deploy has. +# +# STATE. /opt/grafana/data holds grafana.db. /opt/grafana/provisioning is +# mounted read-only and was EMPTY as of 2026-09-11, and grafana.db reported +# zero datasources and zero dashboards: this Grafana is an empty install whose +# only configured account is the local admin. Nothing here queries Prometheus +# or Loki yet. + +[Unit] +Description=Grafana dashboards (Dankclaw) +After=network-online.target prometheus.service +Wants=network-online.target prometheus.service + +[Container] +ContainerName=grafana +Image=docker.io/grafana/grafana:latest +Network=dankclaw_default +User=0:0 +PublishPort=127.0.0.1:3002:3000 +Environment=GF_SERVER_ROOT_URL=https://grafana.franksdank.cloud +Environment=GF_AUTH_GENERIC_OAUTH_ENABLED=true +Environment=GF_AUTH_GENERIC_OAUTH_NAME=Authentik +Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_ID=grafana-21ca190cde75d60f +Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${GRAFANA_OAUTH_CLIENT_SECRET} +# Global authorize path: this Authentik build resolves the provider by the +# client_id query parameter — /application/o//authorize/ does not exist. +Environment=GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://auth.franksdank.cloud/application/o/authorize/ +Environment=GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://auth.franksdank.cloud/application/o/token/ +# ?api=1 = introspection-style userinfo: returns custom claims (groups) too. +Environment=GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.franksdank.cloud/application/o/userinfo/?api=1 +Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=false +Environment=GF_AUTH_GENERIC_OAUTH_AUTO_UPSERT=true +Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_ORG=true +Environment=GF_AUTH_GENERIC_OAUTH_ORG_ID=1 +Volume=/opt/grafana/data:/var/lib/grafana +Volume=/opt/grafana/provisioning:/etc/grafana/provisioning:ro +HealthCmd=wget --spider -q http://localhost:3000/api/health +HealthInterval=30s +HealthTimeout=10s +HealthRetries=3 +HealthStartPeriod=60s + +[Service] +# systemd-side only: supplies ${GRAFANA_OAUTH_CLIENT_SECRET} for expansion +# above. The container does not receive this file. +EnvironmentFile=/home/dank/serverAppConfigs/.env +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/dankclaw/observability/quadlet/loki.container b/dankclaw/observability/quadlet/loki.container new file mode 100644 index 0000000..bccb7ea --- /dev/null +++ b/dankclaw/observability/quadlet/loki.container @@ -0,0 +1,57 @@ +# Quadlet unit: loki (rootless, boot-persistent). +# +# Install path on dankclaw: ~/.config/containers/systemd/loki.container +# Unit name: loki.service (container name: loki) +# +# Log store. This replaces the compose service of the same name in +# dankclaw/compose.yaml ("Observability" block). +# +# IMAGE. franksdank/loki is built locally by deploy-remote.sh +# (build_compose_image) from dankclaw/loki/Containerfile: upstream +# grafana/loki plus a static busybox, because loki is distroless and the +# healthcheck has nothing to run inside it. Pull=never keeps a boot from +# trying to fetch that local tag from a registry. The localhost/ prefix is +# required — this host's quadlet path cannot resolve a registry-prefixed ref +# from local storage at create time. +# +# PUBLISHING. Bound to 127.0.0.1: Alloy pushes to loki:3100 over +# dankclaw_default, Caddy proxies the guarded vhost to loki:3100 the same way, +# and the homepage status timer probes 127.0.0.1:3100. Loki has +# auth_enabled: false, so the compose service's 0.0.0.0 binding was an +# unauthenticated log-ingest and read API on every host interface. +# +# CONFIG. /opt/loki/config/loki-config.yaml is installed from +# shared/loki/loki-config.yaml on every deploy (single-binary, filesystem +# storage, 30-day retention). Chunks and rules live in /opt/loki/data. +# +# Rootless: container root maps to the invoking user (dank), who owns +# /opt/loki/data; the image's default user (10001) maps to a subuid that +# cannot write it. + +[Unit] +Description=Loki log store (Dankclaw) +After=network-online.target +Wants=network-online.target + +[Container] +ContainerName=loki +Image=localhost/franksdank/loki:latest +Pull=never +Network=dankclaw_default +User=0:0 +Exec=-config.file=/etc/loki/loki-config.yaml +PublishPort=127.0.0.1:3100:3100 +Volume=/opt/loki/config:/etc/loki +Volume=/opt/loki/data:/loki +HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:3100/ready +HealthInterval=30s +HealthTimeout=10s +HealthRetries=3 +HealthStartPeriod=30s + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/dankclaw/observability/quadlet/node-exporter.container b/dankclaw/observability/quadlet/node-exporter.container new file mode 100644 index 0000000..7abdc84 --- /dev/null +++ b/dankclaw/observability/quadlet/node-exporter.container @@ -0,0 +1,52 @@ +# Quadlet unit: node-exporter (rootless, boot-persistent). +# +# Install path on dankclaw: ~/.config/containers/systemd/node-exporter.container +# Unit name: node-exporter.service (container name: node-exporter) +# +# Host metrics source for Prometheus. This replaces the compose service of the +# same name in dankclaw/compose.yaml ("Observability" block). +# +# HOST NAMESPACE. It must run in the host network namespace to report the +# host's own interfaces/connections, and --pid=host for the processes and +# cgroups collectors. "localhost" inside the container is therefore the host's +# loopback, which is what the homepage status timer probes. +# +# LISTEN ADDRESSES. Until 2026-09-11 this bound :9100 on every interface. It +# now binds exactly two: loopback for the homepage status probe, and the LAN +# address because Prometheus — which runs in a container on dankclaw_default — +# scrapes http://192.168.0.143:9100 (shared/prometheus/prometheus.yml, +# job "node-dankclaw"). node_exporter itself has no authentication, so keeping +# the wildcard bind off the table matters: this is the second half of the +# pairing that removes the unauthenticated observability surface, and both +# consumers are accounted for. Dankserver runs its own copy of this container +# (scraped by the "node-dankserver" job) from its own compose stack. +# +# MOUNTS. The container reads the host's /proc, /sys and / as /host/* read-only +# and is pointed at them with --path.*; it writes nothing anywhere. + +[Unit] +Description=Prometheus node exporter (Dankclaw host metrics) +After=network-online.target +Wants=network-online.target + +[Container] +ContainerName=node-exporter +Image=docker.io/prom/node-exporter:latest +Network=host +PodmanArgs=--pid=host +Exec=--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/rootfs --web.listen-address=127.0.0.1:9100 --web.listen-address=192.168.0.143:9100 +Volume=/proc:/host/proc:ro +Volume=/sys:/host/sys:ro +Volume=/:/rootfs:ro +HealthCmd=wget --spider -q http://localhost:9100/metrics +HealthInterval=30s +HealthTimeout=10s +HealthRetries=3 +HealthStartPeriod=30s + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/dankclaw/observability/quadlet/prometheus.container b/dankclaw/observability/quadlet/prometheus.container new file mode 100644 index 0000000..52112f5 --- /dev/null +++ b/dankclaw/observability/quadlet/prometheus.container @@ -0,0 +1,56 @@ +# Quadlet unit: prometheus (rootless, boot-persistent). +# +# Install path on dankclaw: ~/.config/containers/systemd/prometheus.container +# Unit name: prometheus.service (container name: prometheus) +# +# The homelab metrics store. This replaces the compose service of the same +# name in dankclaw/compose.yaml ("Observability" block): these five containers +# were started by hand with podman-compose and never came back after a reboot, +# which the 2026-09-11 reboot for the amd_iommu=off change made obvious. +# +# PUBLISHING. The published port is bound to 127.0.0.1 deliberately. Prometheus +# is reached by Caddy over the dankclaw_default network (the Caddyfile's +# post-auth handler proxies to prometheus:9090) and by the host-side homepage +# status timer over 127.0.0.1. The compose service published on 0.0.0.0, which +# put an UNAUTHENTICATED metrics API — with the lifecycle endpoint enabled — +# on every host interface: a bypass around the Anubis + Authentik guard on +# prometheus.franksdank.cloud. No off-host consumer needs it. +# +# CONFIG. /opt/prometheus/config/prometheus.yml is installed from +# shared/prometheus/prometheus.yml by deploy-remote.sh on every deploy, so the +# repo copy is the source of truth. STATE (30-day retention) lives in +# /opt/prometheus/data. +# +# Rootless: container root maps to the invoking user (dank) in this user +# namespace and dank owns /opt/prometheus/data. The image's default user +# (nobody) maps to a subuid that cannot write it. +# +# There are no alerting rules and no Alertmanager anywhere in this stack: the +# scrape config is the whole of its configuration. + +[Unit] +Description=Prometheus metrics store (Dankclaw) +After=network-online.target +Wants=network-online.target + +[Container] +ContainerName=prometheus +Image=docker.io/prom/prometheus:latest +Network=dankclaw_default +User=0:0 +Exec=--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=30d --web.enable-lifecycle +PublishPort=127.0.0.1:9090:9090 +Volume=/opt/prometheus/config:/etc/prometheus +Volume=/opt/prometheus/data:/prometheus +HealthCmd=wget --spider -q http://localhost:9090/-/healthy +HealthInterval=30s +HealthTimeout=10s +HealthRetries=3 +HealthStartPeriod=30s + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index eb50ac8..0febef6 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -300,6 +300,16 @@ def services_for_path(path: str, target: str) -> set[str]: # its unit and fleet settings live in one .container file. _add(services, "frank-large") return services + if len(parts) >= 2 and parts[1] == "observability": + # The five observability containers became Quadlet units on + # 2026-09-11. A unit change selects exactly its own service; + # SERVICE_ORDER already puts each dependency ahead of its + # consumer (prometheus before grafana, loki before alloy). + if len(parts) >= 4 and parts[2] == "quadlet": + name = parts[3].removesuffix(".container") + if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter"}: + _add(services, name) + return services if len(parts) >= 2 and parts[1] == "gitea": if len(parts) >= 4 and parts[2] == "quadlet": name = parts[3] diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 768493d..887325e 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -363,20 +363,11 @@ deploy_compose_service() { return 0 fi case "$service" in - comfyui|hermes|hermes-dashboard|hermes-webui|prometheus|grafana|loki|alloy|node-exporter) + comfyui|hermes|hermes-dashboard|hermes-webui) case "$service" in - comfyui|loki|alloy) + comfyui) build_compose_image "$service" ;; - prometheus) - copy_prometheus_config - ;; - loki) - copy_loki_config - ;; - alloy) - copy_alloy_config - ;; esac podman-compose -f dankclaw/compose.yaml up -d --no-deps "$service" ;; @@ -387,6 +378,45 @@ deploy_compose_service() { esac } +# The five observability containers left compose on 2026-09-11 (they were +# started by hand with podman-compose and never came back after a reboot). +# They are ordinary rootless Quadlet units now: same images, same mounts, same +# arguments, same container names, so nothing that addresses them by name — +# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed. +# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the +# compose services published on 0.0.0.0 and none of them authenticates. +deploy_observability() { + local service="$1" + case "$service" in + prometheus) + copy_prometheus_config + ;; + loki) + copy_loki_config + build_compose_image loki + ;; + alloy) + copy_alloy_config + build_compose_image alloy + ;; + grafana|node-exporter) + ;; + *) + echo "unknown observability service: $service" >&2 + return 2 + ;; + esac + install_quadlet observability dankclaw/observability/quadlet "$service" + systemctl --user restart "$service.service" + case "$service" in + prometheus) wait_for_health prometheus http://127.0.0.1:9090/-/healthy 60 ;; + grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;; + loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;; + alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;; + node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;; + esac +} + build_compose_image() { local service="$1" image cfhash oldhash containerfile case "$service" in @@ -1106,6 +1136,7 @@ deploy_service() { copyparty) deploy_copyparty ;; plex) deploy_plex ;; tautulli) deploy_tautulli ;; + prometheus|grafana|loki|alloy|node-exporter) deploy_observability "$service" ;; *) deploy_compose_service "$service" ;; esac } diff --git a/scripts/test-changed.sh b/scripts/test-changed.sh index ece9421..42341c5 100755 --- a/scripts/test-changed.sh +++ b/scripts/test-changed.sh @@ -107,6 +107,17 @@ for path in "${CHANGED[@]}"; do add_test tests/searxng_quadlet.bats add_test tests/deployment_scoping.bats ;; + # The five observability containers left compose for Quadlet units on + # 2026-09-11. Their seam pins boot persistence and the loopback-only + # publish posture that keeps them off every host interface. This case + # precedes the dankclaw/*/quadlet/*.container fallback below, which + # would otherwise swallow it into unrelated seams. + dankclaw/observability/*) + add_test tests/observability_quadlet.bats + add_test tests/caddyfile.bats + add_test tests/deployment_scoping.bats + add_test tests/docs_completeness.bats + ;; # Caddy left compose.yaml on 2026-08-26 for a quadlet; caddyfile.bats # now asserts against the quadlet itself. Route every change under # dankclaw/caddy/ to both seams (this block must precede the diff --git a/tests/caddyfile.bats b/tests/caddyfile.bats index a7697de..cb2a7de 100644 --- a/tests/caddyfile.bats +++ b/tests/caddyfile.bats @@ -192,18 +192,13 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile" @test "observability containers run as container root (rootless = host user)" { # In rootless podman, container root maps to the invoking user (dank), - # which owns the /opt data dirs. The images' default users (loki 10001, + # who owns the /opt data dirs. The images' default users (loki 10001, # prometheus nobody, grafana 472) map to subuids that own nothing under # /opt and cannot write the data dirs. - local compose="$REPO_ROOT/dankclaw/compose.yaml" - local svc block + local svc for svc in prometheus loki grafana; do - block=$(awk -v s="^ $svc:\$" ' - $0 ~ s { inblk = 1; next } - inblk && /^ [a-z]/ { inblk = 0 } - inblk' "$compose") - echo "$block" | grep -q 'user: "0:0"' || { - echo "service $svc does not run as user 0:0"; return 1; + grep -q '^User=0:0$' "$REPO_ROOT/dankclaw/observability/quadlet/$svc.container" || { + echo "$svc does not run as user 0:0"; return 1; } done } @@ -211,27 +206,41 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile" # --- distroless healthchecks (loki/alloy carry a static busybox) --- @test "loki and alloy run from local images carrying a static busybox" { - local compose="$REPO_ROOT/dankclaw/compose.yaml" - grep -q 'image: franksdank/loki:latest' "$compose" || { echo "loki image not local"; return 1; } - grep -q 'image: franksdank/alloy:latest' "$compose" || { echo "alloy image not local"; return 1; } - local img + local img unit cf for img in loki alloy; do - local cf="$REPO_ROOT/dankclaw/$img/Containerfile" + unit="$REPO_ROOT/dankclaw/observability/quadlet/$img.container" + grep -q "^Image=localhost/franksdank/$img:latest$" "$unit" \ + || { echo "$img does not run the local image"; return 1; } + # The tag exists only in this host's storage: a boot must not reach + # for it at a registry. + grep -q '^Pull=never$' "$unit" \ + || { echo "$img would try to pull its local tag"; return 1; } + cf="$REPO_ROOT/dankclaw/$img/Containerfile" [ -f "$cf" ] || { echo "missing $cf"; return 1; } grep -q 'busybox' "$cf" || { echo "$cf does not install busybox"; return 1; } done } @test "loki and alloy healthchecks use the busybox wget" { - local compose="$REPO_ROOT/dankclaw/compose.yaml" - local n - n=$(grep -c '/usr/local/bin/busybox' "$compose") - [ "$n" -ge 2 ] || { echo "expected >=2 busybox healthchecks, found $n"; return 1; } + # Both images are distroless-ish, so the healthcheck only works because + # the local Containerfile drops a static busybox into the image. + local n=0 unit + for unit in "$REPO_ROOT"/dankclaw/observability/quadlet/loki.container \ + "$REPO_ROOT"/dankclaw/observability/quadlet/alloy.container; do + grep -q '^HealthCmd=/usr/local/bin/busybox wget' "$unit" && n=$((n + 1)) + done + [ "$n" -eq 2 ] || { echo "expected 2 busybox healthchecks, found $n"; return 1; } } -@test "alloy config declares no ui/server block (default UI listens on 127.0.0.1:12345)" { +@test "alloy's UI address comes from the unit, not the config file" { # alloy 1.x refuses to configure the ui service from a config block - # ("service ui does not support being configured"). The default UI - # already listens where the in-container healthcheck probes. + # ("service ui does not support being configured"), so the unit moves the + # UI off the container's loopback with a command-line flag. Without that + # flag the port cannot be published and the service has no reachable + # health endpoint at all — which is why the homepage status card for + # alloy read "down" while it ran under compose. ! grep -qE '^(ui|server) \{' "$REPO_ROOT/shared/alloy/config.alloy" + local unit="$REPO_ROOT/dankclaw/observability/quadlet/alloy.container" + grep -q -- '--server.http.listen-addr=0.0.0.0:12345' "$unit" + grep -q '^PublishPort=127.0.0.1:12345:12345$' "$unit" } diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats new file mode 100755 index 0000000..32052e4 --- /dev/null +++ b/tests/observability_quadlet.bats @@ -0,0 +1,250 @@ +#!/usr/bin/env bats +# Seam: the five observability Quadlet units on dankclaw. +# +# prometheus, grafana, loki, alloy and node-exporter ran as compose services +# until 2026-09-11 and did not survive a reboot — the 2026-09-11 reboot for the +# amd_iommu=off change is what exposed it. They are rootless Quadlet units now. +# This seam pins the properties a reader cannot check by eye: +# +# * boot persistence and self-healing on every unit; +# * the published ports bind 127.0.0.1 and never a wildcard address, so no +# unauthenticated metrics/dashboard/log endpoint is reachable off-host — +# the whole point of moving them off compose, which published on 0.0.0.0; +# * every port that IS published has a named consumer (the host-side homepage +# status timer, or Prometheus scraping the host); +# * the container names still match what Caddy proxies to and what +# Prometheus' own scrape config targets, so the cutover broke no wiring; +# * the deployer installs, restarts and health-checks them, and never tries +# to `enable` a generated unit. + +load helpers + +UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet" +REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" +UNITS=(prometheus grafana loki alloy node-exporter) + +# container_section — the [Container] key/value block only +container_section() { + awk '/^\[Container\]/{f=1;next} /^\[/{f=0} f' "$1" +} + +# service_section — the [Service] key/value block only +service_section() { + awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1" +} + +@test "all five observability units exist under dankclaw/observability/quadlet/" { + local svc + for svc in "${UNITS[@]}"; do + [ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; } + done +} + +@test "every observability unit is boot-persistent" { + # Generated units cannot be enabled on this host's podman 5.x v1 quadlet + # generator; [Install] + linger is the entire boot-persistence story. + local svc + for svc in "${UNITS[@]}"; do + grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \ + || { echo "$svc is not boot-persistent"; return 1; } + done +} + +@test "every observability unit is self-healing" { + local svc + for svc in "${UNITS[@]}"; do + grep -q '^Restart=always$' "$UNIT_DIR/$svc.container" \ + || { echo "$svc has no Restart policy"; return 1; } + grep -q '^RestartSec=' "$UNIT_DIR/$svc.container" \ + || { echo "$svc has no RestartSec"; return 1; } + done +} + +@test "no observability unit publishes on a wildcard address" { + # The compose services published 9090, 3002 and 3100 on 0.0.0.0 with no + # authentication of any kind. Anything reachable off-host must go through + # the guarded franksdank.cloud vhosts instead. + local svc + for svc in "${UNITS[@]}"; do + ! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$UNIT_DIR/$svc.container" \ + || { echo "$svc publishes on a wildcard address"; return 1; } + done + ! grep -qE '^PublishPort=' "$UNIT_DIR/alloy.container" || true +} + +@test "the three HTTP UIs publish on loopback only" { + grep -q '^PublishPort=127.0.0.1:9090:9090$' "$UNIT_DIR/prometheus.container" + grep -q '^PublishPort=127.0.0.1:3002:3000$' "$UNIT_DIR/grafana.container" + grep -q '^PublishPort=127.0.0.1:3100:3100$' "$UNIT_DIR/loki.container" +} + +@test "the homepage status timer's probe targets are the published loopback ports" { + # scripts/homepage-status.py runs on the host and probes the `probe` field + # of services.json from there, so a probe and a published port that + # disagree show the card as down forever. + local registry="$REPO_ROOT/shared/homepage/services.json" + grep -q '"probe": "127.0.0.1:9090"' "$registry" + grep -q '"probe": "127.0.0.1:3002"' "$registry" + grep -q '"probe": "127.0.0.1:3100"' "$registry" + grep -q '"probe": "127.0.0.1:9100"' "$registry" + grep -q '"probe": "127.0.0.1:12345"' "$registry" +} + +@test "node-exporter binds the host namespace and both of its consumers" { + # It reports the host's own interfaces/cgroups, so it needs the host netns + # and host PID namespace; and it must listen exactly where its two + # consumers are: the homepage timer on loopback, Prometheus (in a + # container) on the LAN address. + local unit="$UNIT_DIR/node-exporter.container" + grep -q '^Network=host$' "$unit" + grep -q -- '--pid=host' "$unit" + grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit" + grep -q -- '--web.listen-address=192.168.0.143:9100' "$unit" + ! grep -qE '^PublishPort=' "$unit" +} + +@test "prometheus scrapes node-exporter where node-exporter actually listens" { + # Cross-file contract: if either side's address changes alone, host metrics + # silently stop with no error anywhere. + local unit="$UNIT_DIR/node-exporter.container" + local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" + local addr + addr=$(grep -oE '192\.168\.0\.[0-9]+:9100' "$unit" | head -1) + [ -n "$addr" ] || { echo "node-exporter has no LAN listen address"; return 1; } + grep -q "\"$addr\"" "$cfg" || { echo "prometheus does not scrape $addr"; return 1; } +} + +@test "container names still match what Caddy and Prometheus address by name" { + # Caddy's post-auth handlers proxy to prometheus:9090 / grafana:3000 / + # loki:3100 on the shared network, and Prometheus scrapes grafana:3000 and + # loki:3100 the same way. A renamed container breaks all of it silently. + grep -qE '^[[:space:]]*@grafana host grafana\.franksdank\.cloud$' "$REPO_ROOT/dankclaw/Caddyfile" + grep -q 'reverse_proxy grafana:3000' "$REPO_ROOT/dankclaw/Caddyfile" + grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile" + grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile" + local svc + for svc in prometheus grafana loki alloy; do + grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \ + || { echo "$svc unit renames its container"; return 1; } + grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \ + || { echo "$svc is not on the network Caddy and its peers use"; return 1; } + done + grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml" + grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml" + grep -q 'url = "http://loki:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy" +} + +@test "every observability unit has a health check" { + # A unit without one shows as plainly "Up" in podman ps while its process + # is wedged, and the deploy has nothing to wait on. + local svc + for svc in "${UNITS[@]}"; do + grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \ + || { echo "$svc has no HealthCmd"; return 1; } + grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \ + || { echo "$svc has no HealthRetries"; return 1; } + done +} + +@test "grafana receives the OAuth client secret without receiving the whole .env" { + # systemd expands ${GRAFANA_OAUTH_CLIENT_SECRET} in [Container] from the + # [Service] EnvironmentFile. Putting that file in [Container] would hand + # the container every fleet secret in it (Porkbun keys, Authentik keys, + # the sudo password file's neighbours) for the sake of one value. + local unit="$UNIT_DIR/grafana.container" + service_section "$unit" | grep -q '^EnvironmentFile=/home/dank/serverAppConfigs/.env$' + ! container_section "$unit" | grep -q '^EnvironmentFile=' + container_section "$unit" | grep -q '^Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=\${GRAFANA_OAUTH_CLIENT_SECRET}$' +} + +@test "observability units hold no literal secret material" { + # A secret-named setting is allowed only when its value is a ${VAR} + # reference resolved from an EnvironmentFile systemd reads; a literal value + # would be a tracked credential. Keys that merely mention a secret word + # (GF_AUTH_GENERIC_OAUTH_TOKEN_URL, ..._CLIENT_ID) are not secrets, so the + # match is anchored to the end of the key. + local svc unit line key value + for svc in "${UNITS[@]}"; do + unit="$UNIT_DIR/$svc.container" + while IFS= read -r line; do + key="${line#Environment=}" + value="${key#*=}" + key="${key%%=*}" + case "$key" in + *_SECRET|*_PASSWORD|*_TOKEN|*_API_KEY) + case "$value" in + '${'*) ;; + *) echo "$svc hardcodes a literal secret in $key"; return 1 ;; + esac + ;; + esac + done < <(container_section "$unit" | grep '^Environment=') + ! container_section "$unit" | grep -q '^EnvironmentFile=' \ + || { echo "$svc passes a whole env file into the container"; return 1; } + done +} + +@test "startup order follows the data dependencies" { + # Grafana is useless without Prometheus and Alloy drops logs without Loki; + # both pairs are expressed as systemd ordering, not launch luck. + grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container" + grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container" + grep -q 'After=network-online.target loki.service' "$UNIT_DIR/alloy.container" + grep -q 'Wants=network-online.target loki.service' "$UNIT_DIR/alloy.container" +} + +@test "the deployer installs, restarts and health-checks each unit" { + grep -q '^deploy_observability() {' "$REMOTE_DEPLOY" + grep -q 'install_quadlet observability dankclaw/observability/quadlet "$service"' "$REMOTE_DEPLOY" + grep -q 'systemctl --user restart "$service.service"' "$REMOTE_DEPLOY" + local svc url + while read -r svc url; do + grep -q "wait_for_health $svc $url" "$REMOTE_DEPLOY" \ + || { echo "no health wait for $svc at $url"; return 1; } + done <<'PAIRS' +prometheus http://127.0.0.1:9090/-/healthy +grafana http://127.0.0.1:3002/api/health +loki http://127.0.0.1:3100/ready +alloy http://127.0.0.1:12345/-/healthy +node-exporter http://127.0.0.1:9100/metrics +PAIRS +} + +@test "the deployer never enables a generated unit" { + # `systemctl --user enable X` fails on this host ("transient or generated"). + ! grep -q 'systemctl --user enable prometheus.service\|systemctl --user enable grafana.service\|systemctl --user enable loki.service\|systemctl --user enable alloy.service\|systemctl --user enable node-exporter.service' "$REMOTE_DEPLOY" +} + +@test "the deployer rebuilds the locally built images before restarting them" { + # loki and alloy are franksdank/:latest built from this repo; a unit + # change that skipped the build would restart the previous image. + grep -q 'build_compose_image loki' "$REMOTE_DEPLOY" + grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY" +} + +@test "the five are no longer compose services" { + # Two owners for one container name is how a compose run resurrects a + # stale duplicate next to the quadlet-managed one. + local svc + for svc in "${UNITS[@]}"; do + ! services_in "$REPO_ROOT/dankclaw/compose.yaml" | grep -qx "$svc" \ + || { echo "$svc is still a compose service"; return 1; } + done +} + +@test "changes under dankclaw/observability/ run this seam" { + run "$REPO_ROOT/scripts/test-changed.sh" --list \ + --paths dankclaw/observability/quadlet/prometheus.container + [ "$status" -eq 0 ] + echo "$output" | grep -q 'tests/observability_quadlet.bats' +} + +@test "the planner routes each unit to its own service" { + local svc + for svc in "${UNITS[@]}"; do + run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \ + --paths "dankclaw/observability/quadlet/$svc.container" + [ "$status" -eq 0 ] + [ "$output" = "$svc" ] || { echo "$svc planned as '$output'"; return 1; } + done +}