Files
serverAppConfigs/tests/caddyfile.bats
T
Frank dab81b91a3 feat(observability): make the five Dankclaw observability containers quadlets
prometheus, grafana, loki, alloy and node-exporter ran only from
dankclaw/compose.yaml. Rootless podman does not restart bare containers, so
they did not survive the 2026-09-11 reboot for the amd_iommu=off change and
had to be restored by hand — the same gap that had already cost the fleet
Caddy and Pi-hole, which is why the boot-persistence rule exists.

Each is now a rootless Quadlet unit under dankclaw/observability/quadlet/,
installed and dispatched by deploy_observability in deploy-remote.sh, with
the compose definitions retired so only one owner exists per container name.
Images, mounts, arguments and container names are unchanged, so Caddy's
proxies and Prometheus' own scrape config keep resolving.

Two posture changes, both about the unauthenticated surface:

* The three HTTP UIs published on 0.0.0.0 with no authentication of any kind
  (Prometheus even with the lifecycle endpoint enabled). They now publish on
  127.0.0.1: the browser path is the guarded franksdank.cloud vhost, Caddy
  reaches them by name over dankclaw_default, and the host-side homepage
  status timer probes loopback. Nothing legitimately used the LAN port.
* node-exporter bound :9100 on every interface; it now binds exactly its two
  consumers, loopback (homepage timer) and the LAN address (Prometheus, which
  runs in a container and cannot reach the host's loopback).

alloy gains a reachable health endpoint: its UI listens on the container's
own loopback and alloy 1.x refuses a config block for it, so the compose
service had nothing to probe — the homepage status card for alloy read 'down'
continuously. The unit moves the UI with a command-line flag and publishes
127.0.0.1:12345.

grafana keeps its Authentik OAuth client secret without receiving the whole
.env: systemd expands the value from a [Service] EnvironmentFile, which never
reaches the container.
2026-09-11 15:56:39 -07:00

247 lines
11 KiB
Bash

#!/usr/bin/env bats
# Seam: Caddyfile validity.
#
# There is exactly one Caddy in the homelab: the container on dankclaw
# (dankclaw/compose.yaml, service 'caddy'). It fronts every service on both
# hosts — local services by container name, dankserver services by published
# LAN port. Site blocks use the flat franksdank.cloud / franksdank.casa
# convention. The basicauth hash is a placeholder in git; deploy.sh renders
# the real hash from the host .env (CADDY_BASICAUTH_HASH).
load helpers
CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
@test "the Caddyfile lives on dankclaw only (Caddy is consolidated there)" {
[ -f "$CADDYFILE" ]
[ ! -f "$REPO_ROOT/dankserver/Caddyfile" ]
}
@test "the Caddyfile uses the flat franksdank.cloud hostname convention" {
grep -qE '\.franksdank\.cloud' "$CADDYFILE"
}
@test "no .local vhost names remain in the Caddyfile" {
! grep -qE '^([a-z0-9.-]+\.)?(dankserver|dankclaw)\.local[[:space:]]*\{' \
"$CADDYFILE"
}
@test "the Caddyfile has balanced braces" {
local open close
open=$(grep -o '{' "$CADDYFILE" | wc -l)
close=$(grep -o '}' "$CADDYFILE" | wc -l)
[ "$open" -eq "$close" ] || { echo "unbalanced braces ($open vs $close)"; return 1; }
}
@test "caddy validate passes when caddy is available" {
if command -v caddy >/dev/null 2>&1; then
caddy validate --config "$CADDYFILE" --adapter caddyfile
else
skip "caddy binary not installed locally"
fi
}
@test "homepage basic auth exists in the Caddyfile" {
grep -qiE 'basicauth|basic_auth' "$CADDYFILE"
}
@test "the Caddyfile carries the renderable basicauth placeholder" {
grep -q 'REPLACE_WITH_HASH' "$CADDYFILE"
}
@test "the Caddyfile does not disable the admin endpoint (hot reload needs it)" {
# admin off disables Caddy's config file watcher: rendered Caddyfile
# changes are never loaded until the container is recreated, and the
# stale config fails silently (2026-08-22: basicauth stayed broken for
# hours across deploys). The admin API binds to 127.0.0.1:2019 inside
# the container and the port is not published, so it is not reachable
# from the LAN.
if grep -qE '^\s*admin\s+off' "$CADDYFILE"; then
echo "Caddyfile: 'admin off' disables hot reload of rendered config"
return 1
fi
}
@test "the basicauth placeholder is a bare token (no bcrypt prefix in the template)" {
# Regression (2026-08-22): the template line was
# dank $2a$14$REPLACE_WITH_HASH
# and deploy.sh substitutes the FULL hash (prefix included), so the
# rendered line became an invalid double-prefixed hash and every
# basicauth vhost rejected every password with 401.
if grep -qE '\$2[abxy]\$[0-9]+\$REPLACE_WITH_HASH' "$CADDYFILE"; then
echo "Caddyfile: bcrypt prefix before REPLACE_WITH_HASH (double-prefix regression)"
return 1
fi
grep -qE 'dankuser REPLACE_WITH_HASH' "$CADDYFILE"
}
@test "compose no longer defines the quadlet-replaced services" {
# gitea, its postgres, open-webui, and caddy run as boot-persistent
# quadlets on dankclaw (see dankclaw/gitea/quadlet,
# dankclaw/open-webui/quadlet, dankclaw/caddy/quadlet).
! grep -qE '^\s+(gitea|postgres|openwebui|caddy):\s*$' "$REPO_ROOT/dankclaw/compose.yaml"
}
@test "the caddy quadlet healthcheck probes the bare-IP catch-all" {
grep -q 'HealthCmd=curl -sf -o /dev/null http://localhost:80/' \
"$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
}
@test "plex vhosts proxy to PMS over a TLS upstream (PMS drops plain HTTP off-loopback)" {
# 2026-08-26 diagnosis: PMS 1.43.x on :32400 accepts, reads, and then
# silently closes every non-TLS connection from a non-loopback source
# (loopback is exempt). A plain-HTTP reverse_proxy therefore gets EOF
# and Caddy answers 502 with an empty body (blank page in browsers).
# Both plex vhosts must dial the upstream over TLS. PMS serves a
# self-signed cert, so verification is skipped on this pinned-LAN-IP
# hop only; client-facing Let's Encrypt TLS is untouched.
local block name
for name in 'plex.franksdank.cloud {' 'http://plex.franksdank.casa {'; do
block=$(awk -v s="$name" '
index($0, s) == 1 { inblk = 1 }
inblk { print }
inblk && /^}/ { exit }' "$CADDYFILE")
echo "$block" | grep -q 'reverse_proxy 192.168.0.235:32400' \
|| { echo "$name: missing PMS upstream"; return 1; }
echo "$block" | grep -q 'transport http' \
|| { echo "$name: upstream must use the http transport with TLS"; return 1; }
echo "$block" | grep -q 'tls_insecure_skip_verify' \
|| { echo "$name: TLS upstream must set tls_insecure_skip_verify (PMS self-signed cert)"; return 1; }
done
}
@test "Gitea has no standalone reverse-proxy Quadlet" {
[ ! -e "$REPO_ROOT/dankclaw/gitea/quadlet/gitea-caddy.container" ]
}
@test "the canonical Caddy owns the Gitea route" {
grep -qF 'gitea.franksdank.cloud {' "$CADDYFILE"
grep -qF 'reverse_proxy 192.168.0.143:3000' "$CADDYFILE"
}
# --- Phase 2: real TLS via DNS-01 (Porkbun wildcard cert) ---
@test "all vhosts use TLS except the legacy .casa name" {
# franksdank.casa is not in the same Porkbun account, so it cannot get
# a DNS-01 Let's Encrypt cert; it stays explicitly http. Every other
# client-facing site address must carry no http:// prefix. The internal
# listeners (http://:8016, http://:8017) are not client-facing — Anubis
# and Authentik dial them over container DNS after verifying a client,
# so plain HTTP is intended.
local offenders
offenders=$(grep -E '^http://' "$CADDYFILE" | grep -v 'plex\.franksdank\.casa' | grep -vE '^http://:801[67]' || true)
[ -z "$offenders" ] || { echo "non-.casa http:// vhosts: $offenders"; return 1; }
}
@test "the Caddyfile configures the Porkbun DNS-01 provider globally" {
grep -q 'acme_dns porkbun' "$CADDYFILE"
grep -q '{env.PORKBUN_API_KEY}' "$CADDYFILE"
grep -q '{env.PORKBUN_API_SECRET_KEY}' "$CADDYFILE"
}
@test "the bare-IP catch-all redirects onto the HTTPS homepage at www" {
local block
block=$(awk '/^:80 \{/,/^}/' "$CADDYFILE")
echo "$block" | grep -q 'redir https://www.franksdank.cloud/ 308'
}
@test "the caddy quadlet uses the locally built porkbun image without pulling" {
# The image is built on-host from dankclaw/caddy/Containerfile; a boot
# must not stall on a registry that does not have it.
grep -q '^Image=franksdank/caddy-porkbun:latest$' \
"$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
grep -q '^Pull=never$' "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
}
@test "the caddy quadlet reads the Porkbun credentials from its env file" {
grep -q '^EnvironmentFile=/home/dank/Containers/caddy/env$' \
"$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
grep -q '^PORKBUN_API_KEY=changeme$' "$REPO_ROOT/dankclaw/caddy/env.example"
grep -q '^PORKBUN_API_SECRET_KEY=changeme$' "$REPO_ROOT/dankclaw/caddy/env.example"
}
@test ".env.example carries the Porkbun secret under the module's name" {
grep -q 'PORKBUN_API_SECRET_KEY=' "$REPO_ROOT/dankclaw/.env.example"
! grep -q '^PORKBUN_API_SECRET=' "$REPO_ROOT/dankclaw/.env.example"
}
@test "the porkbun caddy image build lives in the repo" {
local cf="$REPO_ROOT/dankclaw/caddy/Containerfile"
[ -f "$cf" ] || { echo "missing $cf"; return 1; }
grep -q 'xcaddy build' "$cf"
grep -q 'github.com/caddy-dns/porkbun' "$cf"
}
# --- Observability configs (deployed by deploy.sh step 3) ---
@test "no legacy .local service names remain in shared configs or compose" {
! grep -rn 'dankclaw\.local\|dankserver\.local' \
"$REPO_ROOT/shared" "$REPO_ROOT/dankclaw/compose.yaml"
}
@test "loki compactor has a delete_request_store (required when retention is on)" {
local loki="$REPO_ROOT/shared/loki/loki-config.yaml"
if grep -q 'retention_enabled: true' "$loki"; then
grep -q 'delete_request_store:' "$loki"
fi
}
@test "prometheus targets are host:port (no URL schemes)" {
! grep -nE '^\s*-\s*"?http' "$REPO_ROOT/shared/prometheus/prometheus.yml"
}
@test "observability containers run as container root (rootless = host user)" {
# In rootless podman, container root maps to the invoking user (dank),
# who owns the /opt data dirs. The images' default users (loki 10001,
# prometheus nobody, grafana 472) map to subuids that own nothing under
# /opt and cannot write the data dirs.
local svc
for svc in prometheus loki grafana; do
grep -q '^User=0:0$' "$REPO_ROOT/dankclaw/observability/quadlet/$svc.container" || {
echo "$svc does not run as user 0:0"; return 1;
}
done
}
# --- distroless healthchecks (loki/alloy carry a static busybox) ---
@test "loki and alloy run from local images carrying a static busybox" {
local img unit cf
for img in loki alloy; do
unit="$REPO_ROOT/dankclaw/observability/quadlet/$img.container"
grep -q "^Image=localhost/franksdank/$img:latest$" "$unit" \
|| { echo "$img does not run the local image"; return 1; }
# The tag exists only in this host's storage: a boot must not reach
# for it at a registry.
grep -q '^Pull=never$' "$unit" \
|| { echo "$img would try to pull its local tag"; return 1; }
cf="$REPO_ROOT/dankclaw/$img/Containerfile"
[ -f "$cf" ] || { echo "missing $cf"; return 1; }
grep -q 'busybox' "$cf" || { echo "$cf does not install busybox"; return 1; }
done
}
@test "loki and alloy healthchecks use the busybox wget" {
# Both images are distroless-ish, so the healthcheck only works because
# the local Containerfile drops a static busybox into the image.
local n=0 unit
for unit in "$REPO_ROOT"/dankclaw/observability/quadlet/loki.container \
"$REPO_ROOT"/dankclaw/observability/quadlet/alloy.container; do
grep -q '^HealthCmd=/usr/local/bin/busybox wget' "$unit" && n=$((n + 1))
done
[ "$n" -eq 2 ] || { echo "expected 2 busybox healthchecks, found $n"; return 1; }
}
@test "alloy's UI address comes from the unit, not the config file" {
# alloy 1.x refuses to configure the ui service from a config block
# ("service ui does not support being configured"), so the unit moves the
# UI off the container's loopback with a command-line flag. Without that
# flag the port cannot be published and the service has no reachable
# health endpoint at all — which is why the homepage status card for
# alloy read "down" while it ran under compose.
! grep -qE '^(ui|server) \{' "$REPO_ROOT/shared/alloy/config.alloy"
local unit="$REPO_ROOT/dankclaw/observability/quadlet/alloy.container"
grep -q -- '--server.http.listen-addr=0.0.0.0:12345' "$unit"
grep -q '^PublishPort=127.0.0.1:12345:12345$' "$unit"
}