diff --git a/dankclaw/caddy/env.example b/dankclaw/caddy/env.example new file mode 100644 index 0000000..f94315f --- /dev/null +++ b/dankclaw/caddy/env.example @@ -0,0 +1,7 @@ +# Caddy environment (placeholders only). The deploy writes the live file at +# /home/dank/Containers/caddy/env from the checkout's .env; the quadlet reads +# it via EnvironmentFile. Consumed by the Caddyfile's acme_dns block +# (DNS-01 wildcard cert issuance). PORKBUN_API_SECRET_KEY is the name the +# caddy-dns/porkbun module expects. +PORKBUN_API_KEY=changeme +PORKBUN_API_SECRET_KEY=changeme diff --git a/dankclaw/caddy/quadlet/caddy.container b/dankclaw/caddy/quadlet/caddy.container new file mode 100644 index 0000000..5b7caeb --- /dev/null +++ b/dankclaw/caddy/quadlet/caddy.container @@ -0,0 +1,54 @@ +# Quadlet unit: canonical Caddy reverse proxy + ACME (rootless, boot-persistent). +# +# Install path on dankclaw: ~/.config/containers/systemd/caddy.container +# Unit name: caddy.service (container name: caddy) +# +# The homelab's single Caddy: it fronts every service on both hosts and issues +# Let's Encrypt wildcard certs via DNS-01 (Porkbun). It left compose.yaml on +# 2026-08-26 after a reboot exposed that the bare compose container never came +# back: rootless Podman has no boot-time auto-start for bare containers, and +# `restart: unless-stopped` does not survive the explicit stop that shutdown +# performs. See AGENTS.md "Boot Persistence & Self-Healing" and +# docs/Infrastructure/Caddy.md. +# +# It joins the existing compose network (dankclaw_default) so the Caddyfile's +# container-name upstreams (authentik-server, grafana, ...) still resolve; if +# no compose service has created that network yet at boot, quadlet creates it +# first and compose reuses it by name later. The image is built locally from +# dankclaw/caddy/Containerfile and exists only on this host, so Pull=never: a +# boot must not stall on a registry that does not have it. Porkbun credentials +# come from /home/dank/Containers/caddy/env (deploy writes it from the +# checkout's .env; secrets never enter the repo). + +[Unit] +Description=Caddy reverse proxy + ACME (Porkbun DNS-01) container +After=network-online.target +Wants=network-online.target + +[Container] +ContainerName=caddy +Image=franksdank/caddy-porkbun:latest +Pull=never +Network=dankclaw_default +PublishPort=0.0.0.0:80:80/tcp +PublishPort=0.0.0.0:443:443/tcp +EnvironmentFile=/home/dank/Containers/caddy/env +Volume=/home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro +Volume=/home/dank/Containers/caddy/data:/data +Volume=/home/dank/Containers/caddy/config:/config +# The homepage: the host's /srv/homepage (deploy-managed, not the repo's +# shared/homepage). +Volume=/srv/homepage:/srv/homepage:ro +# The bare-IP catch-all answers 302 (curl -f passes on <400); this proves +# Caddy itself is serving without touching any upstream. +HealthCmd=curl -sf -o /dev/null http://localhost:80/ +HealthInterval=30s +HealthTimeout=10s +HealthRetries=3 + +[Service] +Restart=always +RestartSec=5 + +[Install] +WantedBy=default.target diff --git a/dankclaw/compose.yaml b/dankclaw/compose.yaml index a0b2de1..205e22c 100644 --- a/dankclaw/compose.yaml +++ b/dankclaw/compose.yaml @@ -310,44 +310,8 @@ services: authentik-postgres: condition: service_healthy - # --- Reverse proxy + homepage (the homelab's single Caddy) --------------- - # The Caddyfile mounted here is the RENDERED copy (basicauth hash filled in - # from the host .env by scripts/deploy.sh); the repo copy carries the - # REPLACE_WITH_HASH placeholder. Certs/config live under + # Caddy (the homelab's single reverse proxy) left compose on 2026-08-26: + # it is boot-persistent via its Quadlet unit (dankclaw/caddy/quadlet/ + # caddy.container), installed by deploy_caddy in scripts/deploy-remote.sh. + # The rendered Caddyfile, certs, and config still live under # /home/dank/Containers/caddy (rootless-writable). - caddy: - # Locally built: stock Caddy + the Porkbun DNS provider - # (dankclaw/caddy/Containerfile). See that file for build instructions. - image: franksdank/caddy-porkbun:latest - container_name: caddy - restart: unless-stopped - ports: - - "80:80" - - "443:443" - environment: - # Consumed by the Caddyfile's acme_dns block (DNS-01 wildcard certs). - PORKBUN_API_KEY: ${PORKBUN_API_KEY} - PORKBUN_API_SECRET_KEY: ${PORKBUN_API_SECRET_KEY} - volumes: - - /home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro - - /home/dank/Containers/caddy/data:/data - - /home/dank/Containers/caddy/config:/config - # The homepage: the host's /srv/homepage. The deploy copies the static - # files there; the status checker writes status.json and the deploy - # writes deployed.js there. (Not the repo's shared/homepage, which - # lacks the generated files.) - - /srv/homepage:/srv/homepage:ro - depends_on: - - authentik-server - - comfyui - - hermes - - grafana - - prometheus - - loki - healthcheck: - # The bare-IP catch-all answers 302 (curl -f passes on <400); this - # proves Caddy itself is serving without touching any upstream. - test: ["CMD", "curl", "-sf", "-o", "/dev/null", "http://localhost:80/"] - interval: 30s - timeout: 10s - retries: 3 diff --git a/dankclaw/pihole/sysctl/99-dnsmasq.conf b/dankclaw/pihole/sysctl/99-dnsmasq.conf new file mode 100644 index 0000000..6239744 --- /dev/null +++ b/dankclaw/pihole/sysctl/99-dnsmasq.conf @@ -0,0 +1,8 @@ +# Rootless DNS must bind port 53: lower the unprivileged-port threshold so a +# rootless container may publish 0.0.0.0:53 (80/443 stay allowed; 53 < 80). +# Single source of truth for net.ipv4.ip_unprivileged_port_start on dankclaw; +# deploy_pihole installs it to /etc/sysctl.d/ and applies it with +# `sysctl --system` (the same code path boot uses via systemd-sysctl). +# A later-sorted drop-in that reset this to 80 took LAN DNS down after the +# 2026-08-26 reboot — deploy_pihole neutralizes such conflicts. +net.ipv4.ip_unprivileged_port_start=53 diff --git a/docs/Dankclaw/Dankclaw Stack.md b/docs/Dankclaw/Dankclaw Stack.md index d917b5b..647b5bb 100644 --- a/docs/Dankclaw/Dankclaw Stack.md +++ b/docs/Dankclaw/Dankclaw Stack.md @@ -55,9 +55,9 @@ repo owns their scripts, units, and knowledge base (see ## Boot Persistence -The Gitea database and server, the SearXNG trio (gluetun, searxng, valkey), and open-webui run as Quadlet units, so they start at boot. Gitea's web UI is reverse-proxied by the canonical Caddy compose service; the former standalone Gitea Caddy unit is retired. User linger is on. ComfyUI runs as a systemd unit around a bare process. The remaining services in the table are defined in `compose.yaml` and are not yet boot-persistent. +The Gitea database and server, the SearXNG trio (gluetun, searxng, valkey), open-webui, Pi-hole, and Caddy run as Quadlet units, so they start at boot. Caddy left `compose.yaml` on 2026-08-26 after a reboot exposed that bare compose containers never come back; see [[Caddy]] for the unit and the incident. The former standalone Gitea Caddy unit is retired. User linger is on. ComfyUI runs as a systemd unit around a bare process. The remaining services in the table are defined in `compose.yaml` and are not yet boot-persistent — per AGENTS.md (Boot Persistence & Self-Healing) they each need a quadlet before they may be considered reliably deployed. -Deploy the SearXNG trio with `scripts/deploy-searxng-quadlet.sh dankclaw` and open-webui with `scripts/deploy-open-webui-quadlet.sh dankclaw`. The Gitea units install to `~/.config/containers/systemd/` (see [[Gitea]]); Caddy is deployed through `scripts/deploy.sh dankclaw --service caddy`. +Deploy the SearXNG trio with `scripts/deploy-searxng-quadlet.sh dankclaw`, open-webui with `scripts/deploy-open-webui-quadlet.sh dankclaw`, and Pi-hole with `scripts/deploy-pihole-quadlet.sh dankclaw`. The Gitea units install to `~/.config/containers/systemd/` (see [[Gitea]]); Caddy is deployed through `scripts/deploy.sh dankclaw --service caddy`. ## Routing Rules diff --git a/docs/Infrastructure/Caddy.md b/docs/Infrastructure/Caddy.md index a2bb061..1d2f75c 100644 --- a/docs/Infrastructure/Caddy.md +++ b/docs/Infrastructure/Caddy.md @@ -1,6 +1,6 @@ # Caddy -Caddy is the single reverse proxy for both hosts. It runs as a container on Dankclaw (192.168.0.143), terminates TLS with real Let's Encrypt certificates, routes hostnames to containers on both hosts, and protects the homepage and sensitive services with HTTP basic auth. +Caddy is the single reverse proxy for both hosts. It runs as a boot-persistent quadlet container (`caddy.service`) on Dankclaw (192.168.0.143), terminates TLS with real Let's Encrypt certificates, routes hostnames to containers on both hosts, and protects the homepage and sensitive services with HTTP basic auth. **Docs:** https://caddyserver.com/docs @@ -8,6 +8,7 @@ Caddy is the single reverse proxy for both hosts. It runs as a container on Dank | Item | Value | |------|-------| +| Runtime | Quadlet unit `caddy.service` (`dankclaw/caddy/quadlet/caddy.container`) — boot-persistent, self-healing | | Image | `franksdank/caddy-porkbun:latest` (local build, `dankclaw/caddy/Containerfile`) | | Config | `/home/dank/Containers/caddy/Caddyfile` (bind mount, rendered by deploy) | | Data | `/home/dank/Containers/caddy/data` (certs) and `.../config` (ACME state) | @@ -15,7 +16,19 @@ Caddy is the single reverse proxy for both hosts. It runs as a container on Dank | TLS | Let's Encrypt, DNS-01 via the Porkbun module (one cert per vhost) | | Runs on | Dankclaw only — Dankserver has no Caddy | -The Caddyfile in git (`dankclaw/Caddyfile`) carries a `REPLACE_WITH_HASH` placeholder. `scripts/deploy.sh` renders the real basicauth hash from the host `.env` (`CADDY_BASICAUTH_HASH`) into the mounted path before `up`. See [[Deployment Flow]]. +The Caddyfile in git (`dankclaw/Caddyfile`) carries a `REPLACE_WITH_HASH` placeholder. `scripts/deploy.sh` renders the real basicauth hash from the host `.env` (`CADDY_BASICAUTH_HASH`) into the mounted path before start. See [[Deployment Flow]]. + +## Quadlet (boot persistence) + +Caddy left `compose.yaml` on 2026-08-26. A reboot exposed that the bare compose container never comes back: rootless Podman has no boot-time auto-start for bare containers, and `restart: unless-stopped` does not survive the explicit stop that shutdown performs. The quadlet unit is now the only way Caddy starts (see AGENTS.md "Boot Persistence & Self-Healing"). + +- **Install path on Dankclaw:** `~/.config/containers/systemd/caddy.container` (unit `caddy.service`) +- **Network:** `dankclaw_default` (the compose bridge — the Caddyfile's container-name upstreams must keep resolving; if no compose service has created the network yet at boot, quadlet creates it first and compose reuses it by name later) +- **Published ports:** 80/tcp, 443/tcp +- **Env file:** `/home/dank/Containers/caddy/env` (`PORKBUN_API_KEY`, `PORKBUN_API_SECRET_KEY`) — the deploy writes it from the checkout's `.env`; secrets never enter the repo (placeholders in `dankclaw/caddy/env.example`) +- **Restart:** `Restart=always`, `RestartSec=5`; the health check probes the bare-IP catch-all (302) inside the container +- **Image policy:** `Pull=never` — the image is built on-host and exists nowhere else; a boot must not stall on a registry +- **Deploy:** `./scripts/deploy.sh dankclaw --service caddy` — renders the Caddyfile, rebuilds the image if the Containerfile changed, retires the legacy split proxy, writes the env file, removes the old bare container (all state is in host volumes), then installs, enables, and starts the unit ## The porkbun image @@ -34,7 +47,7 @@ acme_dns porkbun { } ``` -compose passes both variables from the host `.env`. Keep the real keys in `.env` only; `.env.example` holds placeholders. The Porkbun account holds only `franksdank.cloud`. +The deploy writes both variables from the host `.env` into `/home/dank/Containers/caddy/env`, which the quadlet reads via `EnvironmentFile`. Keep the real keys in `.env` only; `.env.example` (and `dankclaw/caddy/env.example`) hold placeholders. The Porkbun account holds only `franksdank.cloud`. ## Vhost map @@ -83,6 +96,7 @@ Certificates and ACME state live in the caddy data volume. Back it up with the s ## Gotchas +- A bare compose container is **not** boot-persistent: after the 2026-08-26 reboot Caddy stayed down for hours because `restart: unless-stopped` does not survive shutdown's explicit stop. It now runs only from its quadlet unit — do not re-add it to `compose.yaml`. - `admin off` disables the config file watcher: rendered changes then sit on disk while Caddy serves its stale in-memory config until the container is recreated, and nothing logs an error (2026-08-22: a broken basicauth hash survived several deploys this way). `tests/caddyfile.bats` forbids `admin off`. - The basicauth template must stay a bare `REPLACE_WITH_HASH` token (no `$2a$14$` prefix) — the rendered value is the complete hash. A double prefix silently rejects every password with 401; no Caddy error is logged. - A new vhost needs three things: a Pi-hole `addn-hosts` line, a Porkbun A record, and the Caddyfile block. See [[LAN-DNS]]. diff --git a/docs/Infrastructure/Pi-hole.md b/docs/Infrastructure/Pi-hole.md index 338a089..a39bd85 100644 --- a/docs/Infrastructure/Pi-hole.md +++ b/docs/Infrastructure/Pi-hole.md @@ -9,6 +9,7 @@ Pi-hole is the LAN DNS server for the homelab. It runs as a quadlet container on | `pihole` quadlet | `dankclaw/pihole/quadlet/pihole.container` | Boot-persistent container unit | | FTL | `docker.io/pihole/pihole:latest` | DNS engine (a dnsmasq fork) + ad blocking | | `addn-hosts` | `dankclaw/pihole/addn-hosts` | One A record per Caddy vhost | +| sysctl drop-in | `dankclaw/pihole/sysctl/99-dnsmasq.conf` (repo) → `/etc/sysctl.d/` | Rootless port-53 threshold, applied at boot | | env file | `/home/dank/Containers/pihole/env` (host) | `ServerIP`, `TZ`, `DNS1`, `DNS2`, `WEBPASSWORD` | | Web UI | `pihole.franksdank.cloud` (Caddy → 8083) | Dashboard + admin, family login | @@ -49,7 +50,7 @@ The slirp netns has no useful `resolv.conf`. The env file pins the upstreams: `D ## Host prerequisites -1. `sysctl net.ipv4.ip_unprivileged_port_start=53` (persisted in `/etc/sysctl.d/99-dnsmasq.conf`) — rootless Podman may then publish port 53. +1. `net.ipv4.ip_unprivileged_port_start=53`, persisted in `/etc/sysctl.d/99-dnsmasq.conf` — rootless Podman may then publish port 53. The repo file `dankclaw/pihole/sysctl/99-dnsmasq.conf` is the single source of truth: `deploy_pihole` installs it, stages any later-sorted drop-in that resets the value in the home trash, and applies it with `sysctl --system` (the same code path boot uses via systemd-sysctl). **Incident (2026-08-26):** a second drop-in (`99-rootless-ports.conf`, sorted after) reset the value to 80. The runtime override from an earlier manual `sysctl -w` died with a reboot, and `pihole.service` crash-looped on `bind: permission denied` for port 53 (restart counter 1645) until the conflict was removed. 2. `DNSStubListener=no` in `/etc/systemd/resolved.conf` — the resolved stub would otherwise hold port 53 inside the container netns. Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them. @@ -62,6 +63,7 @@ Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them. | A name does not resolve for a new client | The client's `resolv.conf` does not show `192.168.0.143` — the G36 hand-off is missing. See [[LAN-DNS]] | | The web UI asks for a password loop | `WEBPASSWORD` in the env file is the placeholder `changeme` — fill the real value, restart the unit | | Upstream queries fail (local names still answer) | `DNS1`/`DNS2` in the env file are missing or wrong | +| Port 53 `bind: permission denied` crash loop after a reboot | sysctl conflict: exactly one file under `/etc/sysctl.d/` may set `ip_unprivileged_port_start`, and it must say 53. Redeploy pihole — `deploy_pihole` reinstalls the drop-in and neutralizes conflicts | | Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 `), and FTL serves it. Use the FQDN, or add an `/etc/hosts` entry on the client | ## See Also diff --git a/docs/log.md b/docs/log.md index 43c98a9..f017be0 100644 --- a/docs/log.md +++ b/docs/log.md @@ -30,3 +30,10 @@ timestamp: 2026-09-14 - `/scripts/deploy.sh` and `/scripts/deploy-remote.sh` — Removed standalone proxy dispatch and added idempotent retirement before canonical Caddy deployment. - `/tests/caddyfile.bats`, `/tests/deployment_scoping.bats`, `/tests/deployment_flow.bats`, `/tests/frank_large_config.bats` — Added migration coverage and corrected the existing ubatch harness assertion. - `/docs/index.md` — Bundle index remains the root navigation entry for the updated pages. +- **Boot-persistence audit (post-reboot, 2026-08-26 evening):** after Frank's reboot, Caddy and Pi-hole were both down. Caddy ran as a bare compose container (`restart: unless-stopped` does not survive shutdown's explicit stop; rootless Podman has no boot-time auto-start for bare containers). Pi-hole's quadlet existed but crash-looped (restart counter 1645): a later-sorted sysctl drop-in (`99-rootless-ports.conf`) reset `net.ipv4.ip_unprivileged_port_start` to 80, so the rootless port-53 publish failed with `bind: permission denied`. +- `/AGENTS.md` — new MANDATORY section "Boot Persistence & Self-Healing": every deployed, proven-working container must have a self-healing, boot-persistent quadlet unit (`WantedBy=default.target` + enabled + linger, `Restart=always`, repo-provenance `.container`), verified before a service is declared deployed. +- `/dankclaw/caddy/quadlet/caddy.container`, `/dankclaw/caddy/env.example` — Caddy is now a boot-persistent quadlet (`caddy.service`): joins `dankclaw_default` so container-name upstreams keep resolving, `Pull=never` (local-only image), Porkbun credentials via `/home/dank/Containers/caddy/env` (deploy writes it from `.env`), bare-IP-catch-all health check. The caddy service was removed from `/dankclaw/compose.yaml`. +- `/dankclaw/pihole/sysctl/99-dnsmasq.conf` — repo source of truth for `ip_unprivileged_port_start=53`; `deploy_pihole` installs it to `/etc/sysctl.d/`, stages conflicting later-sorted drop-ins in the home trash, and applies `sysctl --system` (the boot code path). +- `/scripts/deploy-remote.sh` — `deploy_caddy` installs/enables/starts the quadlet (retiring the bare container; all state is in host volumes); `deploy_pihole` enforces the sysctl prerequisite. +- `/tests/caddy_quadlet.bats` (new seam), `/tests/caddyfile.bats`, `/tests/deployment_flow.bats`, `/tests/pihole.bats` — quadlet boot-persistence, self-healing, network, env, and sysctl coverage; compose assertions repointed from the removed caddy service to the quadlet. +- `/docs/Infrastructure/Caddy.md`, `/docs/Infrastructure/Pi-hole.md`, `/docs/Dankclaw/Dankclaw Stack.md` — quadlet runtime facts, the 2026-08-26 incident, and the sysctl-conflict failure mode. diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index c115669..b89f41f 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -305,7 +305,26 @@ deploy_caddy() { # Retire the old split proxy before claiming ports 80/443. This is # idempotent and leaves the old unit recoverable in the target trash. retire_legacy_gitea_caddy - podman-compose -f dankclaw/compose.yaml up -d --no-deps caddy + # Caddy's env: Porkbun DNS-01 credentials from the checkout's .env. The + # live file holds secrets, so it exists only on the host; the quadlet + # reads it via EnvironmentFile. + mkdir -p /home/dank/Containers/caddy + if [[ ! -f /home/dank/Containers/caddy/env ]]; then + awk -F= '/^PORKBUN_API_KEY=|^PORKBUN_API_SECRET_KEY=/ { print }' \ + dankclaw/.env > /home/dank/Containers/caddy/env + fi + local var + for var in PORKBUN_API_KEY PORKBUN_API_SECRET_KEY; do + grep -q "^${var}=." /home/dank/Containers/caddy/env \ + || { echo "caddy env missing ${var}; set it in dankclaw/.env" >&2; return 1; } + done + # Retire the bare compose container: all state (Caddyfile, certs, config) + # lives in host volumes, so removing the container loses nothing. + podman stop caddy >/dev/null 2>&1 || true + podman rm caddy >/dev/null 2>&1 || true + install_quadlet caddy dankclaw/caddy/quadlet caddy + systemctl --user enable caddy.service + systemctl --user restart caddy.service if podman container exists caddy; then local reloaded=0 for _ in 1 2 3 4 5; do @@ -429,6 +448,19 @@ deploy_open_webui() { } deploy_pihole() { + # Rootless port 53 needs net.ipv4.ip_unprivileged_port_start <= 53, and it + # must hold at boot (systemd-sysctl applies /etc/sysctl.d). The repo + # drop-in is the single source of truth; a later-sorted file that resets + # the value to 80 took LAN DNS down after the 2026-08-26 reboot, so stage + # any such conflict in the home trash instead of deleting it. + run_privileged install -m 0644 dankclaw/pihole/sysctl/99-dnsmasq.conf \ + /etc/sysctl.d/99-dnsmasq.conf + if grep -qs 'ip_unprivileged_port_start' /etc/sysctl.d/99-rootless-ports.conf; then + local sysctl_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-sysctl-rootless-ports" + mkdir -p "$sysctl_trash" + run_privileged mv /etc/sysctl.d/99-rootless-ports.conf "$sysctl_trash/" + fi + run_privileged sysctl --system >/dev/null mkdir -p /home/dank/Containers/pihole/etc-pihole /home/dank/Containers/pihole/etc-dockers if [[ ! -f /home/dank/Containers/pihole/env ]]; then cp dankclaw/pihole/env.example /home/dank/Containers/pihole/env diff --git a/scripts/test-changed.sh b/scripts/test-changed.sh index 8f96f5b..7b9e649 100755 --- a/scripts/test-changed.sh +++ b/scripts/test-changed.sh @@ -104,6 +104,16 @@ for path in "${CHANGED[@]}"; do add_test tests/searxng_quadlet.bats add_test tests/deployment_scoping.bats ;; + # Caddy left compose.yaml on 2026-08-26 for a quadlet; caddyfile.bats + # now asserts against the quadlet itself. Route every change under + # dankclaw/caddy/ to both seams (this block must precede the + # dankclaw/*.container glob, which would otherwise swallow the unit). + dankclaw/caddy/*) + add_test tests/caddy_quadlet.bats + add_test tests/caddyfile.bats + add_test tests/docs_completeness.bats + add_test tests/deployment_scoping.bats + ;; dankclaw/*.container|dankclaw/*.network) add_test tests/open_webui_quadlet.bats add_test tests/pihole.bats @@ -116,7 +126,7 @@ for path in "${CHANGED[@]}"; do # their seam tests. The pihole-specific block must come first: shell # case matches the first pattern, and dankclaw/pihole/quadlet/*.container # also satisfies the dankclaw/*/quadlet/*.container fallback. - dankclaw/pihole/addn-hosts|dankclaw/pihole/env.example|dankclaw/pihole/quadlet/*.container|dankclaw/pihole/quadlet/*.network) + dankclaw/pihole/*) add_test tests/pihole.bats add_test tests/docs_completeness.bats add_test tests/deployment_scoping.bats diff --git a/tests/caddy_quadlet.bats b/tests/caddy_quadlet.bats new file mode 100644 index 0000000..80c34e4 --- /dev/null +++ b/tests/caddy_quadlet.bats @@ -0,0 +1,63 @@ +#!/usr/bin/env bats +# Seam: canonical Caddy as a boot-persistent, self-healing quadlet on +# dankclaw. It left compose.yaml on 2026-08-26 after a reboot exposed that a +# bare compose container never comes back (rootless Podman has no boot-time +# auto-start for bare containers; `restart: unless-stopped` does not survive +# the explicit stop that shutdown performs). See AGENTS.md "Boot Persistence +# & Self-Healing" and docs/Infrastructure/Caddy.md. + +load helpers + +CADDY_DIR="$REPO_ROOT/dankclaw/caddy" +QUADLET="$CADDY_DIR/quadlet/caddy.container" +ENV_EXAMPLE="$CADDY_DIR/env.example" + +@test "the caddy quadlet container file exists" { + [ -f "$QUADLET" ] +} + +@test "the caddy quadlet is installed for boot" { + grep -q 'WantedBy=default.target' "$QUADLET" +} + +@test "the caddy quadlet restarts on failure (self-healing)" { + grep -q '^Restart=always' "$QUADLET" + grep -qE '^RestartSec=[1-9][0-9]*$' "$QUADLET" +} + +@test "the caddy quadlet publishes 80 and 443 to the LAN" { + grep -qE '^PublishPort=0\.0\.0\.0:80:80/tcp$' "$QUADLET" + grep -qE '^PublishPort=0\.0\.0\.0:443:443/tcp$' "$QUADLET" +} + +@test "the caddy quadlet joins the compose network for container-name upstreams" { + # The Caddyfile reverse-proxies to container names (authentik-server, + # grafana, ...); those resolve only on the shared bridge. + grep -q '^Network=dankclaw_default$' "$QUADLET" +} + +@test "the caddy quadlet never pulls the local-only image at boot" { + grep -q '^Pull=never$' "$QUADLET" +} + +@test "the caddy quadlet reads its env file and mounts its state" { + grep -q '^EnvironmentFile=/home/dank/Containers/caddy/env$' "$QUADLET" + grep -qE '^Volume=/home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro$' "$QUADLET" + grep -qE '^Volume=/home/dank/Containers/caddy/data:/data$' "$QUADLET" + grep -qE '^Volume=/home/dank/Containers/caddy/config:/config$' "$QUADLET" + grep -qE '^Volume=/srv/homepage:/srv/homepage:ro$' "$QUADLET" +} + +@test "the caddy env example has placeholder values only" { + [ -f "$ENV_EXAMPLE" ] + grep -q '^PORKBUN_API_KEY=changeme$' "$ENV_EXAMPLE" + grep -q '^PORKBUN_API_SECRET_KEY=changeme$' "$ENV_EXAMPLE" +} + +@test "no secret material in the caddy files" { + local f + while IFS= read -r f; do + has_no_leaked_secret "$f" + has_no_leaked_sudo_pw "$f" + done < <(find "$CADDY_DIR" -type f | sort) +} diff --git a/tests/caddyfile.bats b/tests/caddyfile.bats index 8162690..2e4741e 100644 --- a/tests/caddyfile.bats +++ b/tests/caddyfile.bats @@ -76,13 +76,15 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile" } @test "compose no longer defines the quadlet-replaced services" { - # gitea, its postgres, and open-webui run as boot-persistent quadlets on - # dankclaw (see dankclaw/gitea/quadlet, dankclaw/open-webui/quadlet). - ! grep -qE '^\s+(gitea|postgres|openwebui):\s*$' "$REPO_ROOT/dankclaw/compose.yaml" + # gitea, its postgres, open-webui, and caddy run as boot-persistent + # quadlets on dankclaw (see dankclaw/gitea/quadlet, + # dankclaw/open-webui/quadlet, dankclaw/caddy/quadlet). + ! grep -qE '^\s+(gitea|postgres|openwebui|caddy):\s*$' "$REPO_ROOT/dankclaw/compose.yaml" } -@test "the caddy service healthcheck probes the bare-IP catch-all" { - grep -q 'curl.*-sf.*http://localhost:80/' "$REPO_ROOT/dankclaw/compose.yaml" +@test "the caddy quadlet healthcheck probes the bare-IP catch-all" { + grep -q 'HealthCmd=curl -sf -o /dev/null http://localhost:80/' \ + "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container" } @test "plex vhosts proxy to PMS over a TLS upstream (PMS drops plain HTTP off-loopback)" { @@ -138,13 +140,19 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile" grep -q 'redir https://franksdank.cloud/' "$CADDYFILE" } -@test "the caddy service uses the locally built porkbun image" { - grep -q 'franksdank/caddy-porkbun' "$REPO_ROOT/dankclaw/compose.yaml" +@test "the caddy quadlet uses the locally built porkbun image without pulling" { + # The image is built on-host from dankclaw/caddy/Containerfile; a boot + # must not stall on a registry that does not have it. + grep -q '^Image=franksdank/caddy-porkbun:latest$' \ + "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container" + grep -q '^Pull=never$' "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container" } -@test "the caddy service passes the Porkbun credentials from .env" { - grep -q 'PORKBUN_API_KEY: \${PORKBUN_API_KEY}' "$REPO_ROOT/dankclaw/compose.yaml" - grep -q 'PORKBUN_API_SECRET_KEY: \${PORKBUN_API_SECRET_KEY}' "$REPO_ROOT/dankclaw/compose.yaml" +@test "the caddy quadlet reads the Porkbun credentials from its env file" { + grep -q '^EnvironmentFile=/home/dank/Containers/caddy/env$' \ + "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container" + grep -q '^PORKBUN_API_KEY=changeme$' "$REPO_ROOT/dankclaw/caddy/env.example" + grep -q '^PORKBUN_API_SECRET_KEY=changeme$' "$REPO_ROOT/dankclaw/caddy/env.example" } @test ".env.example carries the Porkbun secret under the module's name" { diff --git a/tests/deployment_flow.bats b/tests/deployment_flow.bats index 4b5ce35..387cc5d 100644 --- a/tests/deployment_flow.bats +++ b/tests/deployment_flow.bats @@ -54,8 +54,13 @@ load helpers ! grep -q 'deploy_gitea_unit gitea-caddy' "$REPO_ROOT/scripts/deploy-remote.sh" } -@test "canonical Caddy deployment recreates the container when needed" { - grep -q 'podman-compose -f dankclaw/compose.yaml up -d --no-deps caddy' "$REPO_ROOT/scripts/deploy-remote.sh" +@test "canonical Caddy deployment installs its boot-persistent quadlet" { + # 2026-08-26: a bare compose container never came back after reboot. + # deploy_caddy must install, enable, and start the quadlet unit instead + # of `podman-compose up`. + grep -q 'install_quadlet caddy dankclaw/caddy/quadlet caddy' "$REPO_ROOT/scripts/deploy-remote.sh" + grep -q 'systemctl --user enable caddy.service' "$REPO_ROOT/scripts/deploy-remote.sh" + ! grep -q 'podman-compose -f dankclaw/compose.yaml up -d --no-deps caddy' "$REPO_ROOT/scripts/deploy-remote.sh" ! grep -q 'no proxy was stopped or replaced' "$REPO_ROOT/scripts/deploy-remote.sh" } diff --git a/tests/pihole.bats b/tests/pihole.bats index 5aa081f..7fc395d 100644 --- a/tests/pihole.bats +++ b/tests/pihole.bats @@ -48,6 +48,20 @@ HOSTS="$PIHOLE_DIR/addn-hosts" grep -qE '^Volume=/home/dank/Containers/pihole/addn-hosts:/etc/pihole/hosts/addn-hosts:ro$' "$QUADLET" } +@test "the pihole sysctl drop-in pins the unprivileged port threshold to 53" { + # Rootless port 53 needs ip_unprivileged_port_start <= 53 at boot. The + # repo drop-in is the single source of truth; a later-sorted file that + # reset it to 80 took LAN DNS down after the 2026-08-26 reboot. + local f="$PIHOLE_DIR/sysctl/99-dnsmasq.conf" + [ -f "$f" ] + grep -q '^net.ipv4.ip_unprivileged_port_start=53$' "$f" +} + +@test "deploy_pihole installs the sysctl drop-in and applies it at boot-path" { + grep -q 'dankclaw/pihole/sysctl/99-dnsmasq.conf' "$REPO_ROOT/scripts/deploy-remote.sh" + grep -q 'sysctl --system' "$REPO_ROOT/scripts/deploy-remote.sh" +} + @test "the pihole env example has placeholder values only" { [ -f "$ENV_EXAMPLE" ] grep -q '^WEBPASSWORD=changeme$' "$ENV_EXAMPLE"