observability: give Dankserver its node-exporter back as a Quadlet

Dankserver's node-exporter has been dead since a reboot: `podman ps -a` showed
it `Exited (143)` and Prometheus' `node-dankserver` job has been scraping
nothing. It ran only from `dankserver/compose.yaml`, which is the failure mode
this repository's boot-persistence rule exists for (rootless Podman does not
restart bare containers after the stop that shutdown performs), and the compose
form bound the wildcard address — the unauthenticated-metrics-API-on-every-
interface posture already removed on Dankclaw.

  - dankserver/observability/quadlet/node-exporter.container — mirrors the
    Dankclaw unit, listening on exactly 127.0.0.1:9100 (host-local probes) and
    192.168.0.235:9100 (Prometheus on Dankclaw), with Restart=always and
    WantedBy=default.target.
  - the compose service is deleted: one owner per container name.
  - deploy_observability now installs from "$TARGET/observability/quadlet"; it
    hardcoded the Dankclaw path, so a Dankserver unit could not be installed by
    the shared dispatcher at all.
  - planner: "node-exporter" joins the Dankserver service set, with a path rule
    for dankserver/observability/**; the dispatcher's allowlist gains
    dankserver:node-exporter.

Seams cover the silent halves: both listen addresses present and the wildcard
absent, the compose service gone, the allowlist entry present, and the planner
mapping the path to the service.
This commit is contained in:
Frank
2026-09-16 07:07:18 -07:00
parent d104eef4e0
commit da9cff046e
5 changed files with 121 additions and 25 deletions
+41 -1
View File
@@ -232,7 +232,11 @@ service_section() {
@test "the deployer installs, restarts and health-checks each unit" {
grep -q '^deploy_observability() {' "$REMOTE_DEPLOY"
grep -q 'install_quadlet observability dankclaw/observability/quadlet "$service"' "$REMOTE_DEPLOY"
# Installs from the TARGET's own tree, not a hardcoded dankclaw path: the
# two hosts own different observability units now (Dankclaw five, Dankserver
# node-exporter), so the source directory has to follow $TARGET.
grep -q 'local unit_dir="\$TARGET/observability/quadlet"' "$REMOTE_DEPLOY"
grep -q 'install_quadlet observability "\$unit_dir" "\$service"' "$REMOTE_DEPLOY"
grep -q 'systemctl --user restart "$service.service"' "$REMOTE_DEPLOY"
local svc url
while read -r svc url; do
@@ -352,3 +356,39 @@ exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
assert any('ALERTS' in e for e in exprs), exprs
"
}
@test "dankserver runs node-exporter as a boot-persistent unit, not from compose" {
# Dankserver's node-exporter ran only from compose until 2026-09-16 and died
# at a reboot (`Exited (143)`), leaving Prometheus' node-dankserver job
# scraping nothing for a day and a half. It also bound the wildcard address,
# which is the unauthenticated-metrics-on-every-interface posture already
# removed on Dankclaw.
local unit="$REPO_ROOT/dankserver/observability/quadlet/node-exporter.container"
[ -f "$unit" ]
grep -q '^WantedBy=default.target$' "$unit"
grep -q '^Restart=always$' "$unit"
grep -q '^Network=host$' "$unit"
# Exactly two listen addresses: loopback for host-local probes and this
# host's LAN address for Prometheus on Dankclaw. Never the wildcard — then
# fix the half of the sentence the runtime cannot see.
grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
grep -q -- '--web.listen-address=192.168.0.235:9100' "$unit"
! grep -qE -- '--web\.listen-address=(0\.0\.0\.0|:9100|\[::\])' "$unit"
# One owner per container name: the compose service must be gone, or two
# definitions fight over the same container.
! grep -qE '^\s+node-exporter:' "$REPO_ROOT/dankserver/compose.yaml"
# The deploy path has to know about it on this target.
grep -q 'dankserver:node-exporter' "$REPO_ROOT/scripts/deploy-remote.sh"
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
--paths dankserver/observability/quadlet/node-exporter.container
[ "$status" -eq 0 ]
[ "$output" = "node-exporter" ]
}
@test "the observability handler installs units from the target's own tree" {
# It used to hardcode dankclaw/observability/quadlet, so a dankserver unit
# could never be installed by the shared dispatcher.
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
grep -q 'local unit_dir="$TARGET/observability/quadlet"' "$handler"
grep -q 'install_quadlet observability "$unit_dir" "$service"' "$handler"
}