diff --git a/dankclaw/Caddyfile b/dankclaw/Caddyfile index 4d05768..9930331 100644 --- a/dankclaw/Caddyfile +++ b/dankclaw/Caddyfile @@ -56,6 +56,21 @@ } } +# ═══════════════════════════════════════════════════════════════ +# Named snippet: the pre-auth guard chain. Anubis (:8923) challenges, +# the embedded Authentik outpost authenticates, and authenticated +# traffic returns to the http://:8017 listener's post-auth handlers. +# Every guarded vhost is `import guarded` — the chain is EDITED HERE, +# never inlined per site (rank 6: it used to be nine verbatim copies). +# Exemptions live in their own vhosts (auth. is the IdP itself; git +# smart-HTTP and self-authenticating services document their reason). +# ═══════════════════════════════════════════════════════════════ +(guarded) { + reverse_proxy anubis:8923 { + header_up X-Real-Ip {remote_host} + } +} + # --- Bare-IP access (health checks, convenience) --- :80 { redir https://www.franksdank.cloud/ 308 @@ -72,9 +87,7 @@ franksdank.cloud { } www.franksdank.cloud { - reverse_proxy anubis:8923 { - header_up X-Real-Ip {remote_host} - } + import guarded } # --- Legacy name: the old homepage name redirects straight onto www --- @@ -100,9 +113,7 @@ auth.franksdank.cloud { # --- SearXNG (quadlet; behind the Gluetun tunnel, published 8081) --- search.franksdank.cloud { - reverse_proxy anubis:8923 { - header_up X-Real-Ip {remote_host} - } + import guarded } # --- OpenWebUI (quadlet on slirp4netns; published 3005) --- @@ -126,30 +137,22 @@ gitea.franksdank.cloud { # never reach Grafana — RP mode needs none and works regardless of topology. # The basicauth edge this replaces (CADDY_BASICAUTH_HASH) is retired. grafana.franksdank.cloud { - reverse_proxy anubis:8923 { - header_up X-Real-Ip {remote_host} - } + import guarded } # --- Prometheus --- prometheus.franksdank.cloud { - reverse_proxy anubis:8923 { - header_up X-Real-Ip {remote_host} - } + import guarded } # --- Loki (guarded: Anubis → Authentik; loki-proxy provider, internal_host caddy:8017) --- loki.franksdank.cloud { - reverse_proxy anubis:8923 { - header_up X-Real-Ip {remote_host} - } + import guarded } # --- Seerr (guarded: Anubis → Authentik; seerr provider, internal_host caddy:8017) --- seerr.franksdank.cloud { - reverse_proxy anubis:8923 { - header_up X-Real-Ip {remote_host} - } + import guarded } # --- Marreta (guarded: Anubis → Authentik; marreta provider, internal_host caddy:8017) --- @@ -157,9 +160,7 @@ seerr.franksdank.cloud { # group gate). Egress rides the searxng gluetun tunnel; reached via the # host-published port 8082 (gluetun netns), not by container name. marreta.franksdank.cloud { - reverse_proxy anubis:8923 { - header_up X-Real-Ip {remote_host} - } + import guarded } # --- Pi-hole (quadlet on slirp4netns; published 8083) --- @@ -233,16 +234,12 @@ books.franksdank.cloud { # --- RomM --- romm.franksdank.cloud { - reverse_proxy anubis:8923 { - header_up X-Real-Ip {remote_host} - } + import guarded } # --- Copyparty --- files.franksdank.cloud { - reverse_proxy anubis:8923 { - header_up X-Real-Ip {remote_host} - } + import guarded } # --- Jellyfin --- diff --git a/dankclaw/observability/quadlet/node-exporter.container b/dankclaw/observability/quadlet/node-exporter.container index c4cb04c..c1dfbeb 100644 --- a/dankclaw/observability/quadlet/node-exporter.container +++ b/dankclaw/observability/quadlet/node-exporter.container @@ -1,30 +1,32 @@ -# Quadlet unit: node-exporter (rootless, boot-persistent). +# Quadlet unit: node-exporter (rootless, boot-persistent) — RENDERED ARTIFACT. +# Regenerate: python3 scripts/service-map.py render node-exporter-unit --host dankclaw +# Source: shared/services.yml (entry node-exporter) + this template. # # Install path on dankclaw: ~/.config/containers/systemd/node-exporter.container # Unit name: node-exporter.service (container name: node-exporter) # -# Host metrics source for Prometheus. This replaces the compose service of the -# same name in dankclaw/compose.yaml ("Observability" block). +# Host metrics source for Prometheus. Replaces the compose service of the same +# name (the "Observability" block): Dankclaw moved to this quadlet on +# 2026-09-11, Dankserver on 2026-09-16 — its compose-only copy died at a reboot +# and stayed dead (rootless Podman does not restart bare containers after the +# explicit stop shutdown performs), dead for a day and a half before anyone +# noticed. One owner per container name; no compose file remains anywhere. # # HOST NAMESPACE. It must run in the host network namespace to report the # host's own interfaces/connections, and --pid=host for the processes and # cgroups collectors. "localhost" inside the container is therefore the host's # loopback, which is what the homepage status timer probes. # -# LISTEN ADDRESSES. Until 2026-09-11 this bound :9100 on every interface. It -# now binds exactly two: loopback for the homepage status probe, and the LAN -# address because Prometheus — which runs in a container on dankclaw_default — -# scrapes http://192.168.0.143:9100 (shared/prometheus/prometheus.yml, -# job "node-dankclaw"). node_exporter itself has no authentication, so keeping -# the wildcard bind off the table matters: this is the second half of the -# pairing that removes the unauthenticated observability surface, and both -# consumers are accounted for. Dankserver runs its own copy of this container -# (scraped by the "node-dankserver" job) — since 2026-09-16 as a Quadlet unit in -# the same shape (dankserver/observability/quadlet/), because the compose-only -# version it replaced had been dead for a day and a half before anyone noticed. +# LISTEN ADDRESSES. Exactly two: loopback for the homepage status probe, and +# the LAN address, because Prometheus — which runs in a container on +# dankclaw_default — scrapes http://192.168.0.143:9100/metrics +# (shared/prometheus/prometheus.yml, job "node-dankclaw"). node_exporter has no +# authentication, so the wildcard bind stays off the table; this is the +# pairing that removes the unauthenticated observability surface on every +# host, and both consumers are accounted for. # -# MOUNTS. The container reads the host's /proc, /sys and / as /host/* read-only -# and is pointed at them with --path.*; it writes nothing anywhere. +# MOUNTS. The container reads the host's /proc, /sys and / as /host/* +# read-only and is pointed at them with --path.*; it writes nothing anywhere. [Unit] Description=Prometheus node exporter (Dankclaw host metrics) diff --git a/dankserver/observability/quadlet/node-exporter.container b/dankserver/observability/quadlet/node-exporter.container index 97a5cbf..f2dd230 100644 --- a/dankserver/observability/quadlet/node-exporter.container +++ b/dankserver/observability/quadlet/node-exporter.container @@ -1,34 +1,32 @@ -# Quadlet unit: node-exporter (rootless, boot-persistent) — DANKSERVER. +# Quadlet unit: node-exporter (rootless, boot-persistent) — RENDERED ARTIFACT. +# Regenerate: python3 scripts/service-map.py render node-exporter-unit --host dankserver +# Source: shared/services.yml (entry node-exporter) + this template. # # Install path on dankserver: ~/.config/containers/systemd/node-exporter.container # Unit name: node-exporter.service (container name: node-exporter) # -# WHY THIS FILE EXISTS. Until 2026-09-16 the Dankserver node-exporter ran only -# from `dankserver/compose.yaml` (the "Observability" block, `restart: -# unless-stopped`). Rootless Podman does not restart bare containers after the -# explicit stop that shutdown performs, so it died at a reboot and stayed dead: -# `podman ps -a` showed it `Exited (143)` and Prometheus' `node-dankserver` job -# had been scraping nothing since. The compose entry also bound the default -# wildcard address, which is the unauthenticated-metrics-API-on-every-interface -# posture that was already removed on Dankclaw. This unit replaces both: one -# owner per container name (the compose service is deleted), explicit listen -# addresses, and a generator-provided boot path. +# Host metrics source for Prometheus. Replaces the compose service of the same +# name (the "Observability" block): Dankclaw moved to this quadlet on +# 2026-09-11, Dankserver on 2026-09-16 — its compose-only copy died at a reboot +# and stayed dead (rootless Podman does not restart bare containers after the +# explicit stop shutdown performs), dead for a day and a half before anyone +# noticed. One owner per container name; no compose file remains anywhere. # -# HOST NAMESPACE. Networking and PID in the host's namespaces: the collectors -# report the host's own interfaces, connections, processes and cgroups. The -# host's /proc, /sys and / are mounted read-only under /host and /rootfs, and -# node_exporter writes nothing. +# HOST NAMESPACE. It must run in the host network namespace to report the +# host's own interfaces/connections, and --pid=host for the processes and +# cgroups collectors. "localhost" inside the container is therefore the host's +# loopback, which is what the homepage status timer probes. # -# LISTEN ADDRESSES. Exactly two: -# - 127.0.0.1:9100 for host-local probes -# - 192.168.0.235:9100 for Prometheus, which scrapes this host from Dankclaw -# (shared/prometheus/prometheus.yml, job "node-dankserver"). It cannot use -# loopback across hosts, and node_exporter has no authentication, so the -# wildcard bind stays off the table. This is the same deliberate LAN -# listener posture the observability runbook documents. +# LISTEN ADDRESSES. Exactly two: loopback for the homepage status probe, and +# the LAN address, because Prometheus — which runs in a container on +# dankclaw_default — scrapes http://192.168.0.235:9100/metrics +# (shared/prometheus/prometheus.yml, job "node-dankserver"). node_exporter has no +# authentication, so the wildcard bind stays off the table; this is the +# pairing that removes the unauthenticated observability surface on every +# host, and both consumers are accounted for. # -# MIRROR. Keep this file in step with dankclaw/observability/quadlet/ -# node-exporter.container; only the Description and the LAN address differ. +# MOUNTS. The container reads the host's /proc, /sys and / as /host/* +# read-only and is pointed at them with --path.*; it writes nothing anywhere. [Unit] Description=Prometheus node exporter (Dankserver host metrics) diff --git a/scripts/service-map.py b/scripts/service-map.py index 4485cff..3f092db 100755 --- a/scripts/service-map.py +++ b/scripts/service-map.py @@ -100,6 +100,12 @@ class Inventory: return str(host["dns_edge"]) raise SystemExit(f"{self.origin}: no host declares dns_edge") + def service(self, name: str) -> dict | None: + for svc in self.services: + if svc["name"] == name: + return svc + return None + def unit_dirs(self, svc: dict[str, Any], host: str | None = None) -> list[str]: """The repo dirs holding this service's unit files, for one host.""" dirs = svc.get("unit_dir") @@ -492,6 +498,85 @@ def check_config(inv: Inventory) -> list[str]: return problems +# --- node-exporter quadlet template (rank 6) -------------------------------- +# The two node-exporter units were a MIRROR pair kept in step by a comment. +# The template is now the law; `check` drift-gates both tracked copies. +NODE_EXPORTER_TEMPLATE = """\ +# Quadlet unit: node-exporter (rootless, boot-persistent) — RENDERED ARTIFACT. +# Regenerate: python3 scripts/service-map.py render node-exporter-unit --host {host} +# Source: shared/services.yml (entry node-exporter) + this template. +# +# Install path on {host}: ~/.config/containers/systemd/node-exporter.container +# Unit name: node-exporter.service (container name: node-exporter) +# +# Host metrics source for Prometheus. Replaces the compose service of the same +# name (the "Observability" block): Dankclaw moved to this quadlet on +# 2026-09-11, Dankserver on 2026-09-16 — its compose-only copy died at a reboot +# and stayed dead (rootless Podman does not restart bare containers after the +# explicit stop shutdown performs), dead for a day and a half before anyone +# noticed. One owner per container name; no compose file remains anywhere. +# +# HOST NAMESPACE. It must run in the host network namespace to report the +# host's own interfaces/connections, and --pid=host for the processes and +# cgroups collectors. "localhost" inside the container is therefore the host's +# loopback, which is what the homepage status timer probes. +# +# LISTEN ADDRESSES. Exactly two: loopback for the homepage status probe, and +# the LAN address, because Prometheus — which runs in a container on +# dankclaw_default — scrapes http://{lan}:{port}/metrics +# (shared/prometheus/prometheus.yml, job "node-{host}"). node_exporter has no +# authentication, so the wildcard bind stays off the table; this is the +# pairing that removes the unauthenticated observability surface on every +# host, and both consumers are accounted for. +# +# MOUNTS. The container reads the host's /proc, /sys and / as /host/* +# read-only and is pointed at them with --path.*; it writes nothing anywhere. + +[Unit] +Description=Prometheus node exporter ({title} host metrics) +After=network-online.target +Wants=network-online.target + +[Container] +ContainerName=node-exporter +Image=docker.io/prom/node-exporter:latest +Network=host +PodmanArgs=--pid=host +Exec=--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/rootfs --web.listen-address=127.0.0.1:{port} --web.listen-address={lan}:{port} +Volume=/proc:/host/proc:ro +Volume=/sys:/host/sys:ro +Volume=/:/rootfs:ro +HealthCmd=wget --spider -q http://localhost:{port}/metrics +HealthInterval=30s +HealthTimeout=10s +HealthRetries=3 +HealthStartPeriod=30s + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target +""" + + +def render_node_exporter_unit(inv: Inventory, host: str) -> str: + """The node-exporter quadlet for one host: shared template, per-host + title, LAN address from the hosts map, port from the scrape job.""" + svc = inv.service("node-exporter") + if svc is None: + raise SystemExit("inventory has no node-exporter entry") + job = f"node-{host}" + port = next((str(e["target"]).rsplit(":", 1)[1] for e in svc.get("scrape", []) + if e.get("job") == job), None) + if port is None: + raise SystemExit(f"node-exporter scrape declares no job {job}") + lan = inv.hosts[host]["address"] + return NODE_EXPORTER_TEMPLATE.format(host=host, title=host.capitalize(), + lan=lan, port=port) + + def check_drift(inv: Inventory) -> list[str]: problems: list[str] = [] rendered = render_pihole_hosts(inv) @@ -508,6 +593,15 @@ def check_drift(inv: Inventory) -> list[str]: for missing in [f for f in want if f not in have]: problems.append(f"inventory declares a scrape prometheus.yml lacks: {missing}") + svc = inv.service("node-exporter") + for host in (svc or {}).get("hosts", []): + for directory in inv.unit_dirs(svc, host): + path = REPO_ROOT / directory / "node-exporter.container" + if path.exists() and path.read_text() != render_node_exporter_unit(inv, host): + problems.append( + f"{directory}/node-exporter.container is not the render of the " + f"template (scripts/service-map.py render node-exporter-unit --host {host})") + probes_want = sorted(set(probe_facts(inv))) probes_have = homepage_probes(REPO_ROOT / "shared" / "homepage" / "services.json") for extra in [p for p in probes_have if p not in probes_want]: @@ -568,7 +662,8 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--service", required=True) p.add_argument("field") p = sub.add_parser("render") - p.add_argument("artifact", choices=("pihole-hosts",)) + p.add_argument("artifact", choices=("pihole-hosts", "node-exporter-unit")) + p.add_argument("--host", choices=list(Hosts) if False else None) p = sub.add_parser("check") p.add_argument("names", nargs="*") @@ -632,7 +727,13 @@ def main(argv: list[str] | None = None) -> int: values = [s.get(args.field) for s in matches if args.field in s] print(json.dumps(values[0] if len(values) == 1 else values)) elif args.command == "render": - sys.stdout.write(render_pihole_hosts(inv)) + if args.artifact == "pihole-hosts": + sys.stdout.write(render_pihole_hosts(inv)) + else: + if not args.host: + print("render node-exporter-unit requires --host", file=sys.stderr) + return 2 + sys.stdout.write(render_node_exporter_unit(inv, args.host)) elif args.command == "check": problems = check(inv, args.names) if problems: diff --git a/serverAppConfigs-docs/Architecture/Service Inventory.md b/serverAppConfigs-docs/Architecture/Service Inventory.md index db0e469..deb8355 100644 --- a/serverAppConfigs-docs/Architecture/Service Inventory.md +++ b/serverAppConfigs-docs/Architecture/Service Inventory.md @@ -32,6 +32,7 @@ Now there is **one inventory and one reader**: |`scripts/deploy-remote.sh` (`service_is_allowed`)|same| |`scripts/test-changed.sh`|`seams [paths...]` — a service's declared `tests` are its seams| |`dankclaw/pihole/addn-hosts`|`render pihole-hosts` — the tracked file is a rendered artifact| +|`*/observability/quadlet/node-exporter.container`|`render node-exporter-unit --host H` — the MIRROR pair became one template; per-host title, LAN address (hosts map), and port (scrape job) fill it| |Prometheus scrape jobs, LAN DNS, homepage probes|`check` compares the tracked configs against the inventory's `scrape`/`dns`/`probe` fields (drift gate)| Selection rules the engine encodes (see `tests/fixtures/service-plan-golden.tsv`, @@ -64,8 +65,9 @@ the pinned characterization of every tracked path × every target): - **paths** — every claimed `files`/`dirs`/`retires` glob matches ≥1 tracked file; - **compose** — the container names in each host's `compose.yaml` have exactly one quadlet owner (the [[Quadlet Lifecycle]] two-owners trap); -- **drift** — `addn-hosts` equals its render; the Prometheus job set, the DNS - names, and the homepage probe ports equal the inventory's fields. +- **drift** — `addn-hosts` equals its render, each tracked node-exporter unit + equals its per-host render; the Prometheus job set, the DNS names, and the + homepage probe ports equal the inventory's fields. ## 3. Adding, moving, retiring a service diff --git a/serverAppConfigs-docs/Infrastructure/Authentik.md b/serverAppConfigs-docs/Infrastructure/Authentik.md index 851a407..f154d0e 100644 --- a/serverAppConfigs-docs/Infrastructure/Authentik.md +++ b/serverAppConfigs-docs/Infrastructure/Authentik.md @@ -357,11 +357,15 @@ client → TLS edge → Anubis :8923 → outpost decision → internal routing matches on hostname, verified 2026-09-02). `internal_host = http://caddy:8017`, all three flows set, and an Application attached — the two provisioning rules above. -2. **Route flip in ONE commit/deploy**: the vhost becomes - `reverse_proxy anubis:8923` with `header_up X-Real-Ip {remote_host}`, and - inside the `http://:8017` listener add `@ host ` + +2. **Route flip in ONE commit/deploy**: the vhost body becomes + `import guarded` — the named snippet at the top of the Caddyfile is the + whole pre-auth chain (`reverse_proxy anubis:8923` + `header_up X-Real-Ip + {remote_host}`; rank 6 retired the nine verbatim copies), and inside the + `http://:8017` listener add `@ host ` + `handle @ { reverse_proxy }`. Anubis guard set, Caddy TLS edge, `:8017` routing table, and provider list change together. + `tests/caddyfile.bats` pins that the chain exists exactly once as the + `(guarded)` snippet and every guarded vhost imports it. 3. Deploy via the repo deploy flow (commit first; deploy pushes main itself). **Verification (all three):** diff --git a/tests/authentik.bats b/tests/authentik.bats index b1ca883..d16eeb9 100644 --- a/tests/authentik.bats +++ b/tests/authentik.bats @@ -122,7 +122,7 @@ UNIT_DIR="$REPO_ROOT/dankclaw/authentik/quadlet" @test "the homepage vhost is www, guarded by Anubis" { local block block=$(awk '/^www\.franksdank\.cloud[[:space:]]*\{/,/^}/' "$CADDYFILE") - echo "$block" | grep -q 'reverse_proxy anubis:8923' + echo "$block" | grep -q 'import guarded' } @test "the naked domain redirects to www at the TLS edge" { diff --git a/tests/caddyfile.bats b/tests/caddyfile.bats index 13a7315..b417852 100644 --- a/tests/caddyfile.bats +++ b/tests/caddyfile.bats @@ -251,3 +251,14 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile" grep -q -- '--server.http.listen-addr=0.0.0.0:12345' "$unit" grep -q '^PublishPort=127.0.0.1:12345:12345$' "$unit" } + +@test "the guard chain is a named snippet the guarded vhosts import" { + # Rank 6: the Anubis pre-auth chain (reverse_proxy anubis:8923 with the + # X-Real-Ip header_up) existed as nine verbatim copies. One snippet, + # per-vhost imports; a chain edit is a one-line change. + grep -q '^(guarded) {' "$CADDYFILE" + local imports + imports=$(grep -c $'^\timport guarded$' "$CADDYFILE") + [ "$imports" -ge 9 ] + [ "$(grep -c 'reverse_proxy anubis:8923' "$CADDYFILE")" -eq 1 ] +} diff --git a/tests/homepage_caddy.bats b/tests/homepage_caddy.bats index ea4f7d8..fa89fe5 100644 --- a/tests/homepage_caddy.bats +++ b/tests/homepage_caddy.bats @@ -51,7 +51,7 @@ setup() { # domain never enters the chain (it redirects at the TLS edge first). local block block=$(awk '/^www\.franksdank\.cloud[[:space:]]*\{/,/^}/' "$CADDY") - echo "$block" | grep -q 'reverse_proxy anubis:8923' + echo "$block" | grep -q 'import guarded' # forwards accepted traffic to; Host survives the hop, so per-vhost # routing resumes there — guarded vhosts proxy the whole request # through Authentik, which forwards authenticated traffic back on the diff --git a/tests/marreta_quadlet.bats b/tests/marreta_quadlet.bats index 64c35f2..1742783 100755 --- a/tests/marreta_quadlet.bats +++ b/tests/marreta_quadlet.bats @@ -112,11 +112,11 @@ GLUETUN="$REPO_ROOT/dankclaw/searxng/quadlet/gluetun.container" # Public edge: the vhost must proxy to the challenge proxy, not straight to the app. awk ' /^marreta\.franksdank\.cloud \{/ { inblk=1 } - inblk && /reverse_proxy anubis:8923/ { challenge=1 } + inblk && /import guarded/ { challenge=1 } inblk && /^\}/ { inblk=0 } END { exit challenge ? 0 : 1 } ' "$REPO_ROOT/dankclaw/Caddyfile" \ - || { echo "marreta vhost must reverse_proxy anubis:8923"; return 1; } + || { echo "marreta vhost must import the guarded chain (the one named Anubis snippet)"; return 1; } } @test "the Caddyfile post-auth listener routes marreta to the published 8082" { diff --git a/tests/service_inventory.bats b/tests/service_inventory.bats index 0c3d52d..ebeb2ae 100755 --- a/tests/service_inventory.bats +++ b/tests/service_inventory.bats @@ -123,3 +123,16 @@ if problems: sys.exit(1) PYEOF } + +@test "inventory: the node-exporter unit is a per-host rendered template" { + # Rank 6: the two node-exporter quadlets were a MIRROR pair kept in + # step by a comment. The render is now the law on both hosts. + for host in dankclaw dankserver; do + run bash -c "cd '$REPO_ROOT' && python3 scripts/service-map.py render node-exporter-unit --host $host | \ + diff -u $host/observability/quadlet/node-exporter.container -" + [ "$status" -eq 0 ] || { echo "$host unit drift"; return 1; } + done + # Hand-edited copies fail the drift gate: + grep -q "Keep this file in step" dankserver/observability/quadlet/node-exporter.container && return 1 + true +}