Frank
25c93984ad
The healthcheck added to the loki image was discarded by the build: 'HEALTHCHECK is not supported for OCI image format and will be ignored'. podman builds OCI by default, and OCI has no HEALTHCHECK field, so the container came back with no health at all — worse than the failing check it replaced, because nothing reports the loss. The build now passes --format docker, and the seam pins it so a future edit cannot quietly reintroduce an image with no health field.
288 lines
14 KiB
Bash
Executable File
288 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the five observability Quadlet units on dankclaw.
|
|
#
|
|
# prometheus, grafana, loki, alloy and node-exporter ran as compose services
|
|
# until 2026-09-11 and did not survive a reboot — the 2026-09-11 reboot for the
|
|
# amd_iommu=off change is what exposed it. They are rootless Quadlet units now.
|
|
# This seam pins the properties a reader cannot check by eye:
|
|
#
|
|
# * boot persistence and self-healing on every unit;
|
|
# * the published ports bind 127.0.0.1 and never a wildcard address, so no
|
|
# unauthenticated metrics/dashboard/log endpoint is reachable off-host —
|
|
# the whole point of moving them off compose, which published on 0.0.0.0;
|
|
# * every port that IS published has a named consumer (the host-side homepage
|
|
# status timer, or Prometheus scraping the host);
|
|
# * the container names still match what Caddy proxies to and what
|
|
# Prometheus' own scrape config targets, so the cutover broke no wiring;
|
|
# * the deployer installs, restarts and health-checks them, and never tries
|
|
# to `enable` a generated unit.
|
|
|
|
load helpers
|
|
|
|
UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
UNITS=(prometheus grafana loki alloy node-exporter)
|
|
|
|
# container_section <unit> — the [Container] key/value block only
|
|
container_section() {
|
|
awk '/^\[Container\]/{f=1;next} /^\[/{f=0} f' "$1"
|
|
}
|
|
|
|
# service_section <unit> — the [Service] key/value block only
|
|
service_section() {
|
|
awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
|
|
}
|
|
|
|
@test "all five observability units exist under dankclaw/observability/quadlet/" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
[ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "every observability unit is boot-persistent" {
|
|
# Generated units cannot be enabled on this host's podman 5.x v1 quadlet
|
|
# generator; [Install] + linger is the entire boot-persistence story.
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc is not boot-persistent"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "every observability unit is self-healing" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
grep -q '^Restart=always$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no Restart policy"; return 1; }
|
|
grep -q '^RestartSec=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no RestartSec"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "no observability unit publishes on a wildcard address" {
|
|
# The compose services published 9090, 3002 and 3100 on 0.0.0.0 with no
|
|
# authentication of any kind. Anything reachable off-host must go through
|
|
# the guarded franksdank.cloud vhosts instead.
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc publishes on a wildcard address"; return 1; }
|
|
done
|
|
! grep -qE '^PublishPort=' "$UNIT_DIR/alloy.container" || true
|
|
}
|
|
|
|
@test "the three HTTP UIs publish on loopback only" {
|
|
grep -q '^PublishPort=127.0.0.1:9090:9090$' "$UNIT_DIR/prometheus.container"
|
|
grep -q '^PublishPort=127.0.0.1:3002:3000$' "$UNIT_DIR/grafana.container"
|
|
grep -q '^PublishPort=127.0.0.1:3100:3100$' "$UNIT_DIR/loki.container"
|
|
}
|
|
|
|
@test "the homepage status timer's probe targets are the published loopback ports" {
|
|
# scripts/homepage-status.py runs on the host and probes the `probe` field
|
|
# of services.json from there, so a probe and a published port that
|
|
# disagree show the card as down forever.
|
|
local registry="$REPO_ROOT/shared/homepage/services.json"
|
|
grep -q '"probe": "127.0.0.1:9090"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:3002"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:3100"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:9100"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:12345"' "$registry"
|
|
}
|
|
|
|
@test "node-exporter binds the host namespace and both of its consumers" {
|
|
# It reports the host's own interfaces/cgroups, so it needs the host netns
|
|
# and host PID namespace; and it must listen exactly where its two
|
|
# consumers are: the homepage timer on loopback, Prometheus (in a
|
|
# container) on the LAN address.
|
|
local unit="$UNIT_DIR/node-exporter.container"
|
|
grep -q '^Network=host$' "$unit"
|
|
grep -q -- '--pid=host' "$unit"
|
|
grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
|
|
grep -q -- '--web.listen-address=192.168.0.143:9100' "$unit"
|
|
! grep -qE '^PublishPort=' "$unit"
|
|
}
|
|
|
|
@test "prometheus scrapes node-exporter where node-exporter actually listens" {
|
|
# Cross-file contract: if either side's address changes alone, host metrics
|
|
# silently stop with no error anywhere.
|
|
local unit="$UNIT_DIR/node-exporter.container"
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
local addr
|
|
addr=$(grep -oE '192\.168\.0\.[0-9]+:9100' "$unit" | head -1)
|
|
[ -n "$addr" ] || { echo "node-exporter has no LAN listen address"; return 1; }
|
|
grep -q "\"$addr\"" "$cfg" || { echo "prometheus does not scrape $addr"; return 1; }
|
|
}
|
|
|
|
@test "container names still match what Caddy and Prometheus address by name" {
|
|
# Caddy's post-auth handlers proxy to prometheus:9090 / grafana:3000 /
|
|
# loki:3100 on the shared network, and Prometheus scrapes grafana:3000 and
|
|
# loki:3100 the same way. A renamed container breaks all of it silently.
|
|
grep -qE '^[[:space:]]*@grafana host grafana\.franksdank\.cloud$' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy grafana:3000' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
local svc
|
|
for svc in prometheus grafana loki alloy; do
|
|
grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc unit renames its container"; return 1; }
|
|
grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc is not on the network Caddy and its peers use"; return 1; }
|
|
done
|
|
grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q 'url = "http://loki:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
|
|
}
|
|
|
|
@test "every observability unit has a health check" {
|
|
# A container without one shows as plainly "Up" in podman ps while its
|
|
# process is wedged, and the deploy has nothing to wait on.
|
|
#
|
|
# Four units declare it inline. loki cannot: its image is distroless, so a
|
|
# unit HealthCmd would reach podman as CMD-SHELL and fail for want of
|
|
# /bin/sh. Its health command ships in the image instead.
|
|
local svc
|
|
for svc in prometheus grafana alloy node-exporter; do
|
|
grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no HealthCmd"; return 1; }
|
|
grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no HealthRetries"; return 1; }
|
|
done
|
|
! grep -q '^HealthCmd=' "$UNIT_DIR/loki.container"
|
|
}
|
|
|
|
@test "loki's health command is exec form in the image, not a shell string" {
|
|
# The whole reason the loki image exists: distroless has no shell, so the
|
|
# command must be a JSON array that podman execs directly.
|
|
local cf="$REPO_ROOT/dankclaw/loki/Containerfile"
|
|
grep -q '^HEALTHCHECK ' "$cf"
|
|
grep -q 'CMD \["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"\]' "$cf"
|
|
# ... and the busybox it execs has to be in the image.
|
|
grep -q 'COPY --from=tools /bin/busybox.static /usr/local/bin/busybox' "$cf"
|
|
}
|
|
|
|
@test "the loki/alloy/comfyui build uses the docker image format" {
|
|
# Without --format docker the loki healthcheck is silently discarded (OCI
|
|
# has no HEALTHCHECK field) and the container stops reporting health.
|
|
grep -q -- '--format docker' "$REMOTE_DEPLOY"
|
|
grep -q 'podman build --format docker --no-cache' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "grafana gets its OAuth client secret without getting a whole env file" {
|
|
# systemd expands ${GRAFANA_OAUTH_CLIENT_SECRET} for the [Container]
|
|
# Environment= line from the [Service] EnvironmentFile, which keeps the
|
|
# file out of the container entirely. That file is the per-service one the
|
|
# deployer seeds (fleet convention: caddy, pihole, copyparty, romm, seerr)
|
|
# — NOT a repo .env. The repo-root .env the compose service happened to
|
|
# read is a gluetun env file carrying HOME=/root and the WireGuard keys;
|
|
# importing it sets HOME for this service and podman then looks for its
|
|
# configuration under /root and refuses to start.
|
|
local unit="$UNIT_DIR/grafana.container"
|
|
service_section "$unit" | grep -q '^EnvironmentFile=/home/dank/Containers/grafana/env$'
|
|
! service_section "$unit" | grep -q 'serverAppConfigs'
|
|
! container_section "$unit" | grep -q '^EnvironmentFile='
|
|
container_section "$unit" | grep -q '^Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=\${GRAFANA_OAUTH_CLIENT_SECRET}$'
|
|
}
|
|
|
|
@test "the deployer seeds exactly one value into the grafana env file" {
|
|
# The single-variable filter is the point: a whole-file copy is how the
|
|
# HOME=/root failure got in.
|
|
grep -q '^seed_grafana_env() {' "$REMOTE_DEPLOY"
|
|
grep -q '^ local dest=/home/dank/Containers/grafana/env$' "$REMOTE_DEPLOY"
|
|
grep -q "awk -F= '/^GRAFANA_OAUTH_CLIENT_SECRET=/ { print }' dankclaw/.env > \"\$dest\"" "$REMOTE_DEPLOY"
|
|
grep -q "grep -q '\^GRAFANA_OAUTH_CLIENT_SECRET=\.' \"\$dest\"" "$REMOTE_DEPLOY"
|
|
grep -q 'seed_grafana_env || return 1' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "observability units hold no literal secret material" {
|
|
# A secret-named setting is allowed only when its value is a ${VAR}
|
|
# reference resolved from an EnvironmentFile systemd reads; a literal value
|
|
# would be a tracked credential. Keys that merely mention a secret word
|
|
# (GF_AUTH_GENERIC_OAUTH_TOKEN_URL, ..._CLIENT_ID) are not secrets, so the
|
|
# match is anchored to the end of the key.
|
|
local svc unit line key value
|
|
for svc in "${UNITS[@]}"; do
|
|
unit="$UNIT_DIR/$svc.container"
|
|
while IFS= read -r line; do
|
|
key="${line#Environment=}"
|
|
value="${key#*=}"
|
|
key="${key%%=*}"
|
|
case "$key" in
|
|
*_SECRET|*_PASSWORD|*_TOKEN|*_API_KEY)
|
|
case "$value" in
|
|
'${'*) ;;
|
|
*) echo "$svc hardcodes a literal secret in $key"; return 1 ;;
|
|
esac
|
|
;;
|
|
esac
|
|
done < <(container_section "$unit" | grep '^Environment=')
|
|
! container_section "$unit" | grep -q '^EnvironmentFile=' \
|
|
|| { echo "$svc passes a whole env file into the container"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "startup order follows the data dependencies" {
|
|
# Grafana is useless without Prometheus and Alloy drops logs without Loki;
|
|
# both pairs are expressed as systemd ordering, not launch luck.
|
|
grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
|
grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
|
grep -q 'After=network-online.target loki.service' "$UNIT_DIR/alloy.container"
|
|
grep -q 'Wants=network-online.target loki.service' "$UNIT_DIR/alloy.container"
|
|
}
|
|
|
|
@test "the deployer installs, restarts and health-checks each unit" {
|
|
grep -q '^deploy_observability() {' "$REMOTE_DEPLOY"
|
|
grep -q 'install_quadlet observability dankclaw/observability/quadlet "$service"' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl --user restart "$service.service"' "$REMOTE_DEPLOY"
|
|
local svc url
|
|
while read -r svc url; do
|
|
grep -q "wait_for_health $svc $url" "$REMOTE_DEPLOY" \
|
|
|| { echo "no health wait for $svc at $url"; return 1; }
|
|
done <<'PAIRS'
|
|
prometheus http://127.0.0.1:9090/-/healthy
|
|
grafana http://127.0.0.1:3002/api/health
|
|
loki http://127.0.0.1:3100/ready
|
|
alloy http://127.0.0.1:12345/-/healthy
|
|
node-exporter http://127.0.0.1:9100/metrics
|
|
PAIRS
|
|
}
|
|
|
|
@test "the deployer never enables a generated unit" {
|
|
# `systemctl --user enable X` fails on this host ("transient or generated").
|
|
! grep -q 'systemctl --user enable prometheus.service\|systemctl --user enable grafana.service\|systemctl --user enable loki.service\|systemctl --user enable alloy.service\|systemctl --user enable node-exporter.service' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the deployer rebuilds the locally built images before restarting them" {
|
|
# loki and alloy are franksdank/<svc>:latest built from this repo; a unit
|
|
# change that skipped the build would restart the previous image.
|
|
grep -q 'build_compose_image loki' "$REMOTE_DEPLOY"
|
|
grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the five are no longer compose services" {
|
|
# Two owners for one container name is how a compose run resurrects a
|
|
# stale duplicate next to the quadlet-managed one.
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
! services_in "$REPO_ROOT/dankclaw/compose.yaml" | grep -qx "$svc" \
|
|
|| { echo "$svc is still a compose service"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "changes under dankclaw/observability/ run this seam" {
|
|
run "$REPO_ROOT/scripts/test-changed.sh" --list \
|
|
--paths dankclaw/observability/quadlet/prometheus.container
|
|
[ "$status" -eq 0 ]
|
|
echo "$output" | grep -q 'tests/observability_quadlet.bats'
|
|
}
|
|
|
|
@test "the planner routes each unit to its own service" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths "dankclaw/observability/quadlet/$svc.container"
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "$svc" ] || { echo "$svc planned as '$output'"; return 1; }
|
|
done
|
|
}
|