observability: give Dankserver its node-exporter back as a Quadlet
Dankserver's node-exporter has been dead since a reboot: `podman ps -a` showed
it `Exited (143)` and Prometheus' `node-dankserver` job has been scraping
nothing. It ran only from `dankserver/compose.yaml`, which is the failure mode
this repository's boot-persistence rule exists for (rootless Podman does not
restart bare containers after the stop that shutdown performs), and the compose
form bound the wildcard address — the unauthenticated-metrics-API-on-every-
interface posture already removed on Dankclaw.
- dankserver/observability/quadlet/node-exporter.container — mirrors the
Dankclaw unit, listening on exactly 127.0.0.1:9100 (host-local probes) and
192.168.0.235:9100 (Prometheus on Dankclaw), with Restart=always and
WantedBy=default.target.
- the compose service is deleted: one owner per container name.
- deploy_observability now installs from "$TARGET/observability/quadlet"; it
hardcoded the Dankclaw path, so a Dankserver unit could not be installed by
the shared dispatcher at all.
- planner: "node-exporter" joins the Dankserver service set, with a path rule
for dankserver/observability/**; the dispatcher's allowlist gains
dankserver:node-exporter.
Seams cover the silent halves: both listen addresses present and the wildcard
absent, the compose service gone, the allowlist entry present, and the planner
mapping the path to the service.
This commit is contained in:
@@ -232,7 +232,11 @@ service_section() {
|
||||
|
||||
@test "the deployer installs, restarts and health-checks each unit" {
|
||||
grep -q '^deploy_observability() {' "$REMOTE_DEPLOY"
|
||||
grep -q 'install_quadlet observability dankclaw/observability/quadlet "$service"' "$REMOTE_DEPLOY"
|
||||
# Installs from the TARGET's own tree, not a hardcoded dankclaw path: the
|
||||
# two hosts own different observability units now (Dankclaw five, Dankserver
|
||||
# node-exporter), so the source directory has to follow $TARGET.
|
||||
grep -q 'local unit_dir="\$TARGET/observability/quadlet"' "$REMOTE_DEPLOY"
|
||||
grep -q 'install_quadlet observability "\$unit_dir" "\$service"' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl --user restart "$service.service"' "$REMOTE_DEPLOY"
|
||||
local svc url
|
||||
while read -r svc url; do
|
||||
@@ -352,3 +356,39 @@ exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
|
||||
assert any('ALERTS' in e for e in exprs), exprs
|
||||
"
|
||||
}
|
||||
|
||||
@test "dankserver runs node-exporter as a boot-persistent unit, not from compose" {
|
||||
# Dankserver's node-exporter ran only from compose until 2026-09-16 and died
|
||||
# at a reboot (`Exited (143)`), leaving Prometheus' node-dankserver job
|
||||
# scraping nothing for a day and a half. It also bound the wildcard address,
|
||||
# which is the unauthenticated-metrics-on-every-interface posture already
|
||||
# removed on Dankclaw.
|
||||
local unit="$REPO_ROOT/dankserver/observability/quadlet/node-exporter.container"
|
||||
[ -f "$unit" ]
|
||||
grep -q '^WantedBy=default.target$' "$unit"
|
||||
grep -q '^Restart=always$' "$unit"
|
||||
grep -q '^Network=host$' "$unit"
|
||||
# Exactly two listen addresses: loopback for host-local probes and this
|
||||
# host's LAN address for Prometheus on Dankclaw. Never the wildcard — then
|
||||
# fix the half of the sentence the runtime cannot see.
|
||||
grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
|
||||
grep -q -- '--web.listen-address=192.168.0.235:9100' "$unit"
|
||||
! grep -qE -- '--web\.listen-address=(0\.0\.0\.0|:9100|\[::\])' "$unit"
|
||||
# One owner per container name: the compose service must be gone, or two
|
||||
# definitions fight over the same container.
|
||||
! grep -qE '^\s+node-exporter:' "$REPO_ROOT/dankserver/compose.yaml"
|
||||
# The deploy path has to know about it on this target.
|
||||
grep -q 'dankserver:node-exporter' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
|
||||
--paths dankserver/observability/quadlet/node-exporter.container
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$output" = "node-exporter" ]
|
||||
}
|
||||
|
||||
@test "the observability handler installs units from the target's own tree" {
|
||||
# It used to hardcode dankclaw/observability/quadlet, so a dankserver unit
|
||||
# could never be installed by the shared dispatcher.
|
||||
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
grep -q 'local unit_dir="$TARGET/observability/quadlet"' "$handler"
|
||||
grep -q 'install_quadlet observability "$unit_dir" "$service"' "$handler"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user