diff --git a/dankserver/compose.yaml b/dankserver/compose.yaml index 3ccdf10..ec654dc 100644 --- a/dankserver/compose.yaml +++ b/dankserver/compose.yaml @@ -147,25 +147,8 @@ services: retries: 3 - # ═══════════════════════════════════════════════ - # Node Exporter — host metrics - # ═══════════════════════════════════════════════ - node-exporter: - image: prom/node-exporter:latest - container_name: node-exporter - restart: unless-stopped - network_mode: host - pid: host - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - command: - - --path.procfs=/host/proc - - --path.sysfs=/host/sys - - --path.rootfs=/rootfs - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:9100/metrics"] - interval: 30s - timeout: 10s - retries: 3 + # node-exporter left this file on 2026-09-16 for + # dankserver/observability/quadlet/node-exporter.container: a compose-only + # container does not survive a reboot here — it died `Exited (143)` and stayed + # dead, leaving Prometheus' node-dankserver job scraping nothing — and the + # compose form bound the wildcard address. One owner per container name. diff --git a/dankserver/observability/quadlet/node-exporter.container b/dankserver/observability/quadlet/node-exporter.container new file mode 100644 index 0000000..97a5cbf --- /dev/null +++ b/dankserver/observability/quadlet/node-exporter.container @@ -0,0 +1,58 @@ +# Quadlet unit: node-exporter (rootless, boot-persistent) — DANKSERVER. +# +# Install path on dankserver: ~/.config/containers/systemd/node-exporter.container +# Unit name: node-exporter.service (container name: node-exporter) +# +# WHY THIS FILE EXISTS. Until 2026-09-16 the Dankserver node-exporter ran only +# from `dankserver/compose.yaml` (the "Observability" block, `restart: +# unless-stopped`). Rootless Podman does not restart bare containers after the +# explicit stop that shutdown performs, so it died at a reboot and stayed dead: +# `podman ps -a` showed it `Exited (143)` and Prometheus' `node-dankserver` job +# had been scraping nothing since. The compose entry also bound the default +# wildcard address, which is the unauthenticated-metrics-API-on-every-interface +# posture that was already removed on Dankclaw. This unit replaces both: one +# owner per container name (the compose service is deleted), explicit listen +# addresses, and a generator-provided boot path. +# +# HOST NAMESPACE. Networking and PID in the host's namespaces: the collectors +# report the host's own interfaces, connections, processes and cgroups. The +# host's /proc, /sys and / are mounted read-only under /host and /rootfs, and +# node_exporter writes nothing. +# +# LISTEN ADDRESSES. Exactly two: +# - 127.0.0.1:9100 for host-local probes +# - 192.168.0.235:9100 for Prometheus, which scrapes this host from Dankclaw +# (shared/prometheus/prometheus.yml, job "node-dankserver"). It cannot use +# loopback across hosts, and node_exporter has no authentication, so the +# wildcard bind stays off the table. This is the same deliberate LAN +# listener posture the observability runbook documents. +# +# MIRROR. Keep this file in step with dankclaw/observability/quadlet/ +# node-exporter.container; only the Description and the LAN address differ. + +[Unit] +Description=Prometheus node exporter (Dankserver host metrics) +After=network-online.target +Wants=network-online.target + +[Container] +ContainerName=node-exporter +Image=docker.io/prom/node-exporter:latest +Network=host +PodmanArgs=--pid=host +Exec=--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/rootfs --web.listen-address=127.0.0.1:9100 --web.listen-address=192.168.0.235:9100 +Volume=/proc:/host/proc:ro +Volume=/sys:/host/sys:ro +Volume=/:/rootfs:ro +HealthCmd=wget --spider -q http://localhost:9100/metrics +HealthInterval=30s +HealthTimeout=10s +HealthRetries=3 +HealthStartPeriod=30s + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index 40e249a..918acce 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -108,6 +108,7 @@ DANKSERVER_SERVICES = { "lidarr", "readarr", "frank-medium", + "node-exporter", } # FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on @@ -356,6 +357,16 @@ def services_for_path(path: str, target: str) -> set[str]: "frank-medium": {"frank-medium"}, } _add(services, *quadlet_services.get(parts[1], set())) + if len(parts) >= 2 and parts[1] == "observability": + # Dankserver's node-exporter became a Quadlet unit on 2026-09-16 + # (serverAppConfigs-docs/Infrastructure/Observability.md): its + # compose service died at a reboot and stayed dead, so Prometheus' + # node-dankserver job had nothing to scrape. + if len(parts) >= 4 and parts[2] == "quadlet": + name = parts[3].removesuffix(".container") + if name == "node-exporter": + _add(services, "node-exporter") + return services if len(parts) >= 2 and parts[1] == "frank-medium": # The vllm-radiance runtime since 2026-09-15: the quadlet file # above, plus the entrypoint and healthcheck it mounts straight diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index fb9d7ff..79f896a 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -95,7 +95,7 @@ service_is_allowed() { dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\ dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\ dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\ - dankserver:frank-medium|\ + dankserver:frank-medium|dankserver:node-exporter|\ frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\ frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\ frankbook-pro:readarr) @@ -425,6 +425,10 @@ seed_grafana_env() { # compose services published on 0.0.0.0 and none of them authenticates. deploy_observability() { local service="$1" + # Observability units live under /observability/quadlet: Dankclaw + # has five, Dankserver one (node-exporter, since 2026-09-16). The config + # copies below are Dankclaw-owned state and run only there. + local unit_dir="$TARGET/observability/quadlet" case "$service" in prometheus) copy_prometheus_config @@ -448,7 +452,7 @@ deploy_observability() { return 2 ;; esac - install_quadlet observability dankclaw/observability/quadlet "$service" + install_quadlet observability "$unit_dir" "$service" systemctl --user restart "$service.service" case "$service" in prometheus) wait_for_health prometheus http://127.0.0.1:9090/-/healthy 60 ;; diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index 16bd079..55ea31d 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -232,7 +232,11 @@ service_section() { @test "the deployer installs, restarts and health-checks each unit" { grep -q '^deploy_observability() {' "$REMOTE_DEPLOY" - grep -q 'install_quadlet observability dankclaw/observability/quadlet "$service"' "$REMOTE_DEPLOY" + # Installs from the TARGET's own tree, not a hardcoded dankclaw path: the + # two hosts own different observability units now (Dankclaw five, Dankserver + # node-exporter), so the source directory has to follow $TARGET. + grep -q 'local unit_dir="\$TARGET/observability/quadlet"' "$REMOTE_DEPLOY" + grep -q 'install_quadlet observability "\$unit_dir" "\$service"' "$REMOTE_DEPLOY" grep -q 'systemctl --user restart "$service.service"' "$REMOTE_DEPLOY" local svc url while read -r svc url; do @@ -352,3 +356,39 @@ exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])] assert any('ALERTS' in e for e in exprs), exprs " } + +@test "dankserver runs node-exporter as a boot-persistent unit, not from compose" { + # Dankserver's node-exporter ran only from compose until 2026-09-16 and died + # at a reboot (`Exited (143)`), leaving Prometheus' node-dankserver job + # scraping nothing for a day and a half. It also bound the wildcard address, + # which is the unauthenticated-metrics-on-every-interface posture already + # removed on Dankclaw. + local unit="$REPO_ROOT/dankserver/observability/quadlet/node-exporter.container" + [ -f "$unit" ] + grep -q '^WantedBy=default.target$' "$unit" + grep -q '^Restart=always$' "$unit" + grep -q '^Network=host$' "$unit" + # Exactly two listen addresses: loopback for host-local probes and this + # host's LAN address for Prometheus on Dankclaw. Never the wildcard — then + # fix the half of the sentence the runtime cannot see. + grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit" + grep -q -- '--web.listen-address=192.168.0.235:9100' "$unit" + ! grep -qE -- '--web\.listen-address=(0\.0\.0\.0|:9100|\[::\])' "$unit" + # One owner per container name: the compose service must be gone, or two + # definitions fight over the same container. + ! grep -qE '^\s+node-exporter:' "$REPO_ROOT/dankserver/compose.yaml" + # The deploy path has to know about it on this target. + grep -q 'dankserver:node-exporter' "$REPO_ROOT/scripts/deploy-remote.sh" + run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \ + --paths dankserver/observability/quadlet/node-exporter.container + [ "$status" -eq 0 ] + [ "$output" = "node-exporter" ] +} + +@test "the observability handler installs units from the target's own tree" { + # It used to hardcode dankclaw/observability/quadlet, so a dankserver unit + # could never be installed by the shared dispatcher. + local handler="$REPO_ROOT/scripts/deploy-remote.sh" + grep -q 'local unit_dir="$TARGET/observability/quadlet"' "$handler" + grep -q 'install_quadlet observability "$unit_dir" "$service"' "$handler" +}