feat(rank6): node-exporter renders from a template; guard chain is one snippet

- service-map render node-exporter-unit --host H: the MIRROR pair comment
  ('keep this file in step') becomes a template. Per-host title, LAN
  address from the hosts map, port from the node-<host> scrape job. Both
  tracked quadlets are now rendered artifacts; check_drift gates them.
- dankclaw/Caddyfile: the nine verbatim Anubis pre-auth chains
  (reverse_proxy anubis:8923 + X-Real-Ip) collapse into the (guarded)
  named snippet; guarded vhosts are 'import guarded'. Validated with real
  caddy 2.11.7 (import resolution clean; the stock image stops at the
  porkbun module and the git basicauth placeholder, as designed).
- vhost-scoped guard greps (authentik, homepage_caddy, marreta_quadlet)
  now pin 'import guarded' in the vhost block.
- Service Inventory + Authentik docs carry the new laws.
This commit is contained in:
Frank
2026-10-13 22:40:50 -07:00
parent 5976ad0550
commit 65bd5ccea7
11 changed files with 206 additions and 78 deletions
+24 -27
View File
@@ -56,6 +56,21 @@
}
}
# ═══════════════════════════════════════════════════════════════
# Named snippet: the pre-auth guard chain. Anubis (:8923) challenges,
# the embedded Authentik outpost authenticates, and authenticated
# traffic returns to the http://:8017 listener's post-auth handlers.
# Every guarded vhost is `import guarded` — the chain is EDITED HERE,
# never inlined per site (rank 6: it used to be nine verbatim copies).
# Exemptions live in their own vhosts (auth. is the IdP itself; git
# smart-HTTP and self-authenticating services document their reason).
# ═══════════════════════════════════════════════════════════════
(guarded) {
reverse_proxy anubis:8923 {
header_up X-Real-Ip {remote_host}
}
}
# --- Bare-IP access (health checks, convenience) ---
:80 {
redir https://www.franksdank.cloud/ 308
@@ -72,9 +87,7 @@ franksdank.cloud {
}
www.franksdank.cloud {
reverse_proxy anubis:8923 {
header_up X-Real-Ip {remote_host}
}
import guarded
}
# --- Legacy name: the old homepage name redirects straight onto www ---
@@ -100,9 +113,7 @@ auth.franksdank.cloud {
# --- SearXNG (quadlet; behind the Gluetun tunnel, published 8081) ---
search.franksdank.cloud {
reverse_proxy anubis:8923 {
header_up X-Real-Ip {remote_host}
}
import guarded
}
# --- OpenWebUI (quadlet on slirp4netns; published 3005) ---
@@ -126,30 +137,22 @@ gitea.franksdank.cloud {
# never reach Grafana — RP mode needs none and works regardless of topology.
# The basicauth edge this replaces (CADDY_BASICAUTH_HASH) is retired.
grafana.franksdank.cloud {
reverse_proxy anubis:8923 {
header_up X-Real-Ip {remote_host}
}
import guarded
}
# --- Prometheus ---
prometheus.franksdank.cloud {
reverse_proxy anubis:8923 {
header_up X-Real-Ip {remote_host}
}
import guarded
}
# --- Loki (guarded: Anubis → Authentik; loki-proxy provider, internal_host caddy:8017) ---
loki.franksdank.cloud {
reverse_proxy anubis:8923 {
header_up X-Real-Ip {remote_host}
}
import guarded
}
# --- Seerr (guarded: Anubis → Authentik; seerr provider, internal_host caddy:8017) ---
seerr.franksdank.cloud {
reverse_proxy anubis:8923 {
header_up X-Real-Ip {remote_host}
}
import guarded
}
# --- Marreta (guarded: Anubis → Authentik; marreta provider, internal_host caddy:8017) ---
@@ -157,9 +160,7 @@ seerr.franksdank.cloud {
# group gate). Egress rides the searxng gluetun tunnel; reached via the
# host-published port 8082 (gluetun netns), not by container name.
marreta.franksdank.cloud {
reverse_proxy anubis:8923 {
header_up X-Real-Ip {remote_host}
}
import guarded
}
# --- Pi-hole (quadlet on slirp4netns; published 8083) ---
@@ -233,16 +234,12 @@ books.franksdank.cloud {
# --- RomM ---
romm.franksdank.cloud {
reverse_proxy anubis:8923 {
header_up X-Real-Ip {remote_host}
}
import guarded
}
# --- Copyparty ---
files.franksdank.cloud {
reverse_proxy anubis:8923 {
header_up X-Real-Ip {remote_host}
}
import guarded
}
# --- Jellyfin ---
@@ -1,30 +1,32 @@
# Quadlet unit: node-exporter (rootless, boot-persistent).
# Quadlet unit: node-exporter (rootless, boot-persistent) — RENDERED ARTIFACT.
# Regenerate: python3 scripts/service-map.py render node-exporter-unit --host dankclaw
# Source: shared/services.yml (entry node-exporter) + this template.
#
# Install path on dankclaw: ~/.config/containers/systemd/node-exporter.container
# Unit name: node-exporter.service (container name: node-exporter)
#
# Host metrics source for Prometheus. This replaces the compose service of the
# same name in dankclaw/compose.yaml ("Observability" block).
# Host metrics source for Prometheus. Replaces the compose service of the same
# name (the "Observability" block): Dankclaw moved to this quadlet on
# 2026-09-11, Dankserver on 2026-09-16 — its compose-only copy died at a reboot
# and stayed dead (rootless Podman does not restart bare containers after the
# explicit stop shutdown performs), dead for a day and a half before anyone
# noticed. One owner per container name; no compose file remains anywhere.
#
# HOST NAMESPACE. It must run in the host network namespace to report the
# host's own interfaces/connections, and --pid=host for the processes and
# cgroups collectors. "localhost" inside the container is therefore the host's
# loopback, which is what the homepage status timer probes.
#
# LISTEN ADDRESSES. Until 2026-09-11 this bound :9100 on every interface. It
# now binds exactly two: loopback for the homepage status probe, and the LAN
# address because Prometheus — which runs in a container on dankclaw_default —
# scrapes http://192.168.0.143:9100 (shared/prometheus/prometheus.yml,
# job "node-dankclaw"). node_exporter itself has no authentication, so keeping
# the wildcard bind off the table matters: this is the second half of the
# pairing that removes the unauthenticated observability surface, and both
# consumers are accounted for. Dankserver runs its own copy of this container
# (scraped by the "node-dankserver" job) — since 2026-09-16 as a Quadlet unit in
# the same shape (dankserver/observability/quadlet/), because the compose-only
# version it replaced had been dead for a day and a half before anyone noticed.
# LISTEN ADDRESSES. Exactly two: loopback for the homepage status probe, and
# the LAN address, because Prometheus — which runs in a container on
# dankclaw_default — scrapes http://192.168.0.143:9100/metrics
# (shared/prometheus/prometheus.yml, job "node-dankclaw"). node_exporter has no
# authentication, so the wildcard bind stays off the table; this is the
# pairing that removes the unauthenticated observability surface on every
# host, and both consumers are accounted for.
#
# MOUNTS. The container reads the host's /proc, /sys and / as /host/* read-only
# and is pointed at them with --path.*; it writes nothing anywhere.
# MOUNTS. The container reads the host's /proc, /sys and / as /host/*
# read-only and is pointed at them with --path.*; it writes nothing anywhere.
[Unit]
Description=Prometheus node exporter (Dankclaw host metrics)
@@ -1,34 +1,32 @@
# Quadlet unit: node-exporter (rootless, boot-persistent) — DANKSERVER.
# Quadlet unit: node-exporter (rootless, boot-persistent) — RENDERED ARTIFACT.
# Regenerate: python3 scripts/service-map.py render node-exporter-unit --host dankserver
# Source: shared/services.yml (entry node-exporter) + this template.
#
# Install path on dankserver: ~/.config/containers/systemd/node-exporter.container
# Unit name: node-exporter.service (container name: node-exporter)
#
# WHY THIS FILE EXISTS. Until 2026-09-16 the Dankserver node-exporter ran only
# from `dankserver/compose.yaml` (the "Observability" block, `restart:
# unless-stopped`). Rootless Podman does not restart bare containers after the
# explicit stop that shutdown performs, so it died at a reboot and stayed dead:
# `podman ps -a` showed it `Exited (143)` and Prometheus' `node-dankserver` job
# had been scraping nothing since. The compose entry also bound the default
# wildcard address, which is the unauthenticated-metrics-API-on-every-interface
# posture that was already removed on Dankclaw. This unit replaces both: one
# owner per container name (the compose service is deleted), explicit listen
# addresses, and a generator-provided boot path.
# Host metrics source for Prometheus. Replaces the compose service of the same
# name (the "Observability" block): Dankclaw moved to this quadlet on
# 2026-09-11, Dankserver on 2026-09-16 — its compose-only copy died at a reboot
# and stayed dead (rootless Podman does not restart bare containers after the
# explicit stop shutdown performs), dead for a day and a half before anyone
# noticed. One owner per container name; no compose file remains anywhere.
#
# HOST NAMESPACE. Networking and PID in the host's namespaces: the collectors
# report the host's own interfaces, connections, processes and cgroups. The
# host's /proc, /sys and / are mounted read-only under /host and /rootfs, and
# node_exporter writes nothing.
# HOST NAMESPACE. It must run in the host network namespace to report the
# host's own interfaces/connections, and --pid=host for the processes and
# cgroups collectors. "localhost" inside the container is therefore the host's
# loopback, which is what the homepage status timer probes.
#
# LISTEN ADDRESSES. Exactly two:
# - 127.0.0.1:9100 for host-local probes
# - 192.168.0.235:9100 for Prometheus, which scrapes this host from Dankclaw
# (shared/prometheus/prometheus.yml, job "node-dankserver"). It cannot use
# loopback across hosts, and node_exporter has no authentication, so the
# wildcard bind stays off the table. This is the same deliberate LAN
# listener posture the observability runbook documents.
# LISTEN ADDRESSES. Exactly two: loopback for the homepage status probe, and
# the LAN address, because Prometheus — which runs in a container on
# dankclaw_default — scrapes http://192.168.0.235:9100/metrics
# (shared/prometheus/prometheus.yml, job "node-dankserver"). node_exporter has no
# authentication, so the wildcard bind stays off the table; this is the
# pairing that removes the unauthenticated observability surface on every
# host, and both consumers are accounted for.
#
# MIRROR. Keep this file in step with dankclaw/observability/quadlet/
# node-exporter.container; only the Description and the LAN address differ.
# MOUNTS. The container reads the host's /proc, /sys and / as /host/*
# read-only and is pointed at them with --path.*; it writes nothing anywhere.
[Unit]
Description=Prometheus node exporter (Dankserver host metrics)
+103 -2
View File
@@ -100,6 +100,12 @@ class Inventory:
return str(host["dns_edge"])
raise SystemExit(f"{self.origin}: no host declares dns_edge")
def service(self, name: str) -> dict | None:
for svc in self.services:
if svc["name"] == name:
return svc
return None
def unit_dirs(self, svc: dict[str, Any], host: str | None = None) -> list[str]:
"""The repo dirs holding this service's unit files, for one host."""
dirs = svc.get("unit_dir")
@@ -492,6 +498,85 @@ def check_config(inv: Inventory) -> list[str]:
return problems
# --- node-exporter quadlet template (rank 6) --------------------------------
# The two node-exporter units were a MIRROR pair kept in step by a comment.
# The template is now the law; `check` drift-gates both tracked copies.
NODE_EXPORTER_TEMPLATE = """\
# Quadlet unit: node-exporter (rootless, boot-persistent) — RENDERED ARTIFACT.
# Regenerate: python3 scripts/service-map.py render node-exporter-unit --host {host}
# Source: shared/services.yml (entry node-exporter) + this template.
#
# Install path on {host}: ~/.config/containers/systemd/node-exporter.container
# Unit name: node-exporter.service (container name: node-exporter)
#
# Host metrics source for Prometheus. Replaces the compose service of the same
# name (the "Observability" block): Dankclaw moved to this quadlet on
# 2026-09-11, Dankserver on 2026-09-16 — its compose-only copy died at a reboot
# and stayed dead (rootless Podman does not restart bare containers after the
# explicit stop shutdown performs), dead for a day and a half before anyone
# noticed. One owner per container name; no compose file remains anywhere.
#
# HOST NAMESPACE. It must run in the host network namespace to report the
# host's own interfaces/connections, and --pid=host for the processes and
# cgroups collectors. "localhost" inside the container is therefore the host's
# loopback, which is what the homepage status timer probes.
#
# LISTEN ADDRESSES. Exactly two: loopback for the homepage status probe, and
# the LAN address, because Prometheus — which runs in a container on
# dankclaw_default — scrapes http://{lan}:{port}/metrics
# (shared/prometheus/prometheus.yml, job "node-{host}"). node_exporter has no
# authentication, so the wildcard bind stays off the table; this is the
# pairing that removes the unauthenticated observability surface on every
# host, and both consumers are accounted for.
#
# MOUNTS. The container reads the host's /proc, /sys and / as /host/*
# read-only and is pointed at them with --path.*; it writes nothing anywhere.
[Unit]
Description=Prometheus node exporter ({title} host metrics)
After=network-online.target
Wants=network-online.target
[Container]
ContainerName=node-exporter
Image=docker.io/prom/node-exporter:latest
Network=host
PodmanArgs=--pid=host
Exec=--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/rootfs --web.listen-address=127.0.0.1:{port} --web.listen-address={lan}:{port}
Volume=/proc:/host/proc:ro
Volume=/sys:/host/sys:ro
Volume=/:/rootfs:ro
HealthCmd=wget --spider -q http://localhost:{port}/metrics
HealthInterval=30s
HealthTimeout=10s
HealthRetries=3
HealthStartPeriod=30s
[Service]
Restart=always
RestartSec=10
[Install]
WantedBy=default.target
"""
def render_node_exporter_unit(inv: Inventory, host: str) -> str:
"""The node-exporter quadlet for one host: shared template, per-host
title, LAN address from the hosts map, port from the scrape job."""
svc = inv.service("node-exporter")
if svc is None:
raise SystemExit("inventory has no node-exporter entry")
job = f"node-{host}"
port = next((str(e["target"]).rsplit(":", 1)[1] for e in svc.get("scrape", [])
if e.get("job") == job), None)
if port is None:
raise SystemExit(f"node-exporter scrape declares no job {job}")
lan = inv.hosts[host]["address"]
return NODE_EXPORTER_TEMPLATE.format(host=host, title=host.capitalize(),
lan=lan, port=port)
def check_drift(inv: Inventory) -> list[str]:
problems: list[str] = []
rendered = render_pihole_hosts(inv)
@@ -508,6 +593,15 @@ def check_drift(inv: Inventory) -> list[str]:
for missing in [f for f in want if f not in have]:
problems.append(f"inventory declares a scrape prometheus.yml lacks: {missing}")
svc = inv.service("node-exporter")
for host in (svc or {}).get("hosts", []):
for directory in inv.unit_dirs(svc, host):
path = REPO_ROOT / directory / "node-exporter.container"
if path.exists() and path.read_text() != render_node_exporter_unit(inv, host):
problems.append(
f"{directory}/node-exporter.container is not the render of the "
f"template (scripts/service-map.py render node-exporter-unit --host {host})")
probes_want = sorted(set(probe_facts(inv)))
probes_have = homepage_probes(REPO_ROOT / "shared" / "homepage" / "services.json")
for extra in [p for p in probes_have if p not in probes_want]:
@@ -568,7 +662,8 @@ def main(argv: list[str] | None = None) -> int:
p.add_argument("--service", required=True)
p.add_argument("field")
p = sub.add_parser("render")
p.add_argument("artifact", choices=("pihole-hosts",))
p.add_argument("artifact", choices=("pihole-hosts", "node-exporter-unit"))
p.add_argument("--host", choices=list(Hosts) if False else None)
p = sub.add_parser("check")
p.add_argument("names", nargs="*")
@@ -632,7 +727,13 @@ def main(argv: list[str] | None = None) -> int:
values = [s.get(args.field) for s in matches if args.field in s]
print(json.dumps(values[0] if len(values) == 1 else values))
elif args.command == "render":
sys.stdout.write(render_pihole_hosts(inv))
if args.artifact == "pihole-hosts":
sys.stdout.write(render_pihole_hosts(inv))
else:
if not args.host:
print("render node-exporter-unit requires --host", file=sys.stderr)
return 2
sys.stdout.write(render_node_exporter_unit(inv, args.host))
elif args.command == "check":
problems = check(inv, args.names)
if problems:
@@ -32,6 +32,7 @@ Now there is **one inventory and one reader**:
|`scripts/deploy-remote.sh` (`service_is_allowed`)|same|
|`scripts/test-changed.sh`|`seams [paths...]` — a service's declared `tests` are its seams|
|`dankclaw/pihole/addn-hosts`|`render pihole-hosts` — the tracked file is a rendered artifact|
|`*/observability/quadlet/node-exporter.container`|`render node-exporter-unit --host H` — the MIRROR pair became one template; per-host title, LAN address (hosts map), and port (scrape job) fill it|
|Prometheus scrape jobs, LAN DNS, homepage probes|`check` compares the tracked configs against the inventory's `scrape`/`dns`/`probe` fields (drift gate)|
Selection rules the engine encodes (see `tests/fixtures/service-plan-golden.tsv`,
@@ -64,8 +65,9 @@ the pinned characterization of every tracked path × every target):
- **paths** — every claimed `files`/`dirs`/`retires` glob matches ≥1 tracked file;
- **compose** — the container names in each host's `compose.yaml` have exactly
one quadlet owner (the [[Quadlet Lifecycle]] two-owners trap);
- **drift** — `addn-hosts` equals its render; the Prometheus job set, the DNS
names, and the homepage probe ports equal the inventory's fields.
- **drift** — `addn-hosts` equals its render, each tracked node-exporter unit
equals its per-host render; the Prometheus job set, the DNS names, and the
homepage probe ports equal the inventory's fields.
## 3. Adding, moving, retiring a service
@@ -357,11 +357,15 @@ client → TLS edge → Anubis :8923 → outpost decision → internal
routing matches on hostname, verified 2026-09-02).
`internal_host = http://caddy:8017`, all three flows set, and an
Application attached — the two provisioning rules above.
2. **Route flip in ONE commit/deploy**: the vhost becomes
`reverse_proxy anubis:8923` with `header_up X-Real-Ip {remote_host}`, and
inside the `http://:8017` listener add `@<svc> host <vhost>` +
2. **Route flip in ONE commit/deploy**: the vhost body becomes
`import guarded` — the named snippet at the top of the Caddyfile is the
whole pre-auth chain (`reverse_proxy anubis:8923` + `header_up X-Real-Ip
{remote_host}`; rank 6 retired the nine verbatim copies), and inside the
`http://:8017` listener add `@<svc> host <vhost>` +
`handle @<svc> { reverse_proxy <upstream> }`. Anubis guard set, Caddy TLS
edge, `:8017` routing table, and provider list change together.
`tests/caddyfile.bats` pins that the chain exists exactly once as the
`(guarded)` snippet and every guarded vhost imports it.
3. Deploy via the repo deploy flow (commit first; deploy pushes main itself).
**Verification (all three):**
+1 -1
View File
@@ -122,7 +122,7 @@ UNIT_DIR="$REPO_ROOT/dankclaw/authentik/quadlet"
@test "the homepage vhost is www, guarded by Anubis" {
local block
block=$(awk '/^www\.franksdank\.cloud[[:space:]]*\{/,/^}/' "$CADDYFILE")
echo "$block" | grep -q 'reverse_proxy anubis:8923'
echo "$block" | grep -q 'import guarded'
}
@test "the naked domain redirects to www at the TLS edge" {
+11
View File
@@ -251,3 +251,14 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
grep -q -- '--server.http.listen-addr=0.0.0.0:12345' "$unit"
grep -q '^PublishPort=127.0.0.1:12345:12345$' "$unit"
}
@test "the guard chain is a named snippet the guarded vhosts import" {
# Rank 6: the Anubis pre-auth chain (reverse_proxy anubis:8923 with the
# X-Real-Ip header_up) existed as nine verbatim copies. One snippet,
# per-vhost imports; a chain edit is a one-line change.
grep -q '^(guarded) {' "$CADDYFILE"
local imports
imports=$(grep -c $'^\timport guarded$' "$CADDYFILE")
[ "$imports" -ge 9 ]
[ "$(grep -c 'reverse_proxy anubis:8923' "$CADDYFILE")" -eq 1 ]
}
+1 -1
View File
@@ -51,7 +51,7 @@ setup() {
# domain never enters the chain (it redirects at the TLS edge first).
local block
block=$(awk '/^www\.franksdank\.cloud[[:space:]]*\{/,/^}/' "$CADDY")
echo "$block" | grep -q 'reverse_proxy anubis:8923'
echo "$block" | grep -q 'import guarded'
# forwards accepted traffic to; Host survives the hop, so per-vhost
# routing resumes there — guarded vhosts proxy the whole request
# through Authentik, which forwards authenticated traffic back on the
+2 -2
View File
@@ -112,11 +112,11 @@ GLUETUN="$REPO_ROOT/dankclaw/searxng/quadlet/gluetun.container"
# Public edge: the vhost must proxy to the challenge proxy, not straight to the app.
awk '
/^marreta\.franksdank\.cloud \{/ { inblk=1 }
inblk && /reverse_proxy anubis:8923/ { challenge=1 }
inblk && /import guarded/ { challenge=1 }
inblk && /^\}/ { inblk=0 }
END { exit challenge ? 0 : 1 }
' "$REPO_ROOT/dankclaw/Caddyfile" \
|| { echo "marreta vhost must reverse_proxy anubis:8923"; return 1; }
|| { echo "marreta vhost must import the guarded chain (the one named Anubis snippet)"; return 1; }
}
@test "the Caddyfile post-auth listener routes marreta to the published 8082" {
+13
View File
@@ -123,3 +123,16 @@ if problems:
sys.exit(1)
PYEOF
}
@test "inventory: the node-exporter unit is a per-host rendered template" {
# Rank 6: the two node-exporter quadlets were a MIRROR pair kept in
# step by a comment. The render is now the law on both hosts.
for host in dankclaw dankserver; do
run bash -c "cd '$REPO_ROOT' && python3 scripts/service-map.py render node-exporter-unit --host $host | \
diff -u $host/observability/quadlet/node-exporter.container -"
[ "$status" -eq 0 ] || { echo "$host unit drift"; return 1; }
done
# Hand-edited copies fail the drift gate:
grep -q "Keep this file in step" dankserver/observability/quadlet/node-exporter.container && return 1
true
}