From 139a3cd4faeb7717367ee4f687d53199f186f2a7 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 15:41:37 -0700 Subject: [PATCH] fix(blackbox): unit had no Image and no Network=host - quadlet refused to generate it The deploy died at 'Unit blackbox-exporter.service not found': a .container without Image= is not generated at all, and the header argued for Network=host it never declared. Completes the unit (digest pinned prom/blackbox-exporter, Network=host) and fixes the bind topology the first cut got wrong: - Prometheus is bridge-networked and the netavark hairpin gap bars it from its own host's LAN IP, so a LAN-only prober bind is unscrapable and a loopback-only bind was claimed to ride the gateway on faith. The prober now binds 9115 on the podman gateway (scrape path) AND the LAN (healthcheck/diagnostics); the listen_address line leaves blackbox.yml so config and Exec cannot disagree. - comfyui probe target http://comfyui:8188 -> http://127.0.0.1:8188: bridge container names do not resolve in the host netns; the published 0.0.0.0:8188 answers on loopback there. - New seam: every observability unit must declare Image= (red on the shipped file by name); copier test pinned module presence instead of the removed listen line. Docs updated to the real topology. --- .../quadlet/blackbox-exporter.container | 33 ++++++++++++------- .../quadlet/prometheus.container | 9 ++--- .../Observability Provisioning.md | 2 +- .../Infrastructure/Observability.md | 2 +- .../Infrastructure/Prometheus.md | 11 ++++--- shared/blackbox/blackbox.yml | 9 +++-- shared/prometheus/prometheus.yml | 14 ++++---- tests/observability_quadlet.bats | 14 +++++++- 8 files changed, 59 insertions(+), 35 deletions(-) diff --git a/dankclaw/observability/quadlet/blackbox-exporter.container b/dankclaw/observability/quadlet/blackbox-exporter.container index 636b763..d6c2405 100644 --- a/dankclaw/observability/quadlet/blackbox-exporter.container +++ b/dankclaw/observability/quadlet/blackbox-exporter.container @@ -1,29 +1,38 @@ -# Quadlet unit: blackbox exporter (rootless, boot-persistent). +# Quadlet unit: blackbox exporter (rootless, host network, boot-persistent). # # Install path on dankclaw: # ~/.config/containers/systemd/blackbox-exporter.container # # WHY Network=host. The probes it runs are (a) the host's own LAN IP -# (192.168.0.143:) and (b) container names on dankclaw_default (comfyui). -# A bridge container cannot reach the host's LAN IP at an unpublished port -# (netavark DNATs published ports only — see Container Host Reachability in the -# vault); a host-netns process reaches both. Prometheus (bridge) reaches THIS -# container the same way it reaches the signal relay: the quadlet-published -# host.containers.internal address (169.254.1.2), mapped via HostAddr= on the -# prometheus unit. +# (192.168.0.143:8081 SearXNG, :3005 OpenWebUI — host-process/portmap binds a +# bridge container cannot reach; netavark DNATs published ports only, see +# Container Host Reachability in the vault) and (b) ComfyUI's published port +# via 127.0.0.1 (rootless --network=host shares the real host netns, so the +# 0.0.0.0:8188 publish answers on loopback). A bridge-networked prober could +# reach neither class reliably. # -# The 9115 metrics port publishes on loopback only — it is scraped by the -# prometheus container through the gateway, and never opened on the LAN. +# WHY the listener binds TWO addresses. The netavark hairpin gap (measured +# 2026-10-21, cadvisor job comment): a BRIDGE CONTAINER CANNOT DIAL ITS OWN +# HOST'S LAN IP at all — so a LAN-only bind is unscrapable by Prometheus, +# whatever the signal-relay law suggests for services containers dial as +# clients. The podman gateway (169.254.1.2 = host.containers.internal) is the +# address bridge containers reach same-host targets at, so the exporter binds +# it explicitly for the scrape rather than relying on loopback-forwarding +# semantics, and keeps the LAN bind for its own healthcheck and diagnostics. +# No public DNS name, no guard chain: internal net only, per the exposure +# policy. [Unit] Description=Blackbox exporter (HTTP probe prober) After=network-online.target [Container] +Image=docker.io/prom/blackbox-exporter@sha256:ea1c7aa3ef3f307cd03e85f34abb604478ccb90b567fc690da0cd9776da76b13 ContainerName=blackbox-exporter -Exec=/bin/blackbox_exporter --config.file=/etc/blackbox_exporter/config.yml +Network=host +Exec=/bin/blackbox_exporter --config.file=/etc/blackbox_exporter/config.yml --listen-address=192.168.0.143:9115,169.254.1.2:9115 Volume=/opt/blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro -HealthCmd=/bin/sh -c "wget -q -O /dev/null http://127.0.0.1:9115/-/healthy || exit 1" +HealthCmd=/bin/sh -c "wget -q -O /dev/null http://169.254.1.2:9115/-/healthy || exit 1" HealthInterval=30s HealthRetries=3 diff --git a/dankclaw/observability/quadlet/prometheus.container b/dankclaw/observability/quadlet/prometheus.container index e6905d9..bbbb125 100644 --- a/dankclaw/observability/quadlet/prometheus.container +++ b/dankclaw/observability/quadlet/prometheus.container @@ -45,10 +45,11 @@ Wants=network-online.target ContainerName=prometheus Image=docker.io/prom/prometheus:latest Network=dankclaw_default -# Name the podman gateway so scrape jobs can reach host-netns processes (the -# blackbox exporter listens on the host's loopback; a bridge container reaches -# it at this address, never at 127.0.0.1). Same mechanism the hermes Caddy -# upstream learned the hard way (Container Host Reachability in the vault). +# Name the podman gateway so scrape jobs can reach same-host targets (the +# netavark hairpin gap bars this container from dialing its own host's LAN +# IP; the cadvisor/gpu jobs and the blackbox scrape all ride this name). +# Same mechanism the hermes Caddy upstream learned the hard way (Container +# Host Reachability in the vault). HostAddr=host.containers.internal=169.254.1.2 User=0:0 Exec=--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=30d --web.enable-lifecycle diff --git a/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md b/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md index a59771a..6580009 100644 --- a/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md +++ b/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md @@ -21,7 +21,7 @@ committed. | `shared/prometheus/rules/llm-fleet.yml` | alert rules + recording rules | | `shared/grafana/provisioning/` | datasources + dashboards (provisioned tree) | | `shared/alloy/config.alloy` | the log pipeline: journal source → label promotion → Loki push | -| `shared/blackbox/blackbox.yml` | prober modules (`http_2xx`) + the loopback bind | +| `shared/blackbox/blackbox.yml` | prober modules (`http_2xx`); the listen bind lives on the unit's `Exec` | Targets and datasources reference **container names** (`grafana:3000` `loki:3100` — `shared/prometheus/prometheus.yml:52-61`), never diff --git a/serverAppConfigs-docs/Infrastructure/Observability.md b/serverAppConfigs-docs/Infrastructure/Observability.md index 0660283..8205bb6 100644 --- a/serverAppConfigs-docs/Infrastructure/Observability.md +++ b/serverAppConfigs-docs/Infrastructure/Observability.md @@ -163,6 +163,6 @@ verify it through the APIs. - `scripts/gpu-exporter.py`, `scripts/gpu-exporter.service` (per-GPU host exporter) - `shared/prometheus/prometheus.yml`, `shared/prometheus/alertmanager.yml`, `shared/prometheus/rules/llm-fleet.yml` - `shared/grafana/provisioning/`, `shared/loki/loki-config.yaml`, `shared/alloy/config.alloy` -- `shared/blackbox/blackbox.yml` (prober modules + loopback bind) +- `shared/blackbox/blackbox.yml` (prober modules; the listen bind is on the unit's `Exec`) - `tests/observability_quadlet.bats` (boot persistence, no wildcard publish, probe addresses, delivery contract) - Host evidence 2026-10-18: `ss -ltnp`, `podman ps`, `systemctl --user is-enabled|is-active`, `curl` on `127.0.0.1:9090/api/v1/{targets,rules,alerts}`, `127.0.0.1:3100/loki/api/v1/labels`, `127.0.0.1:12345/metrics`, `127.0.0.1:3002/api/health`, `192.168.0.143:18093/health`, `journalctl --user -u signal-relay` diff --git a/serverAppConfigs-docs/Infrastructure/Prometheus.md b/serverAppConfigs-docs/Infrastructure/Prometheus.md index fffed02..dc47a11 100644 --- a/serverAppConfigs-docs/Infrastructure/Prometheus.md +++ b/serverAppConfigs-docs/Infrastructure/Prometheus.md @@ -48,16 +48,17 @@ retired 2026-10-21; census re-run after each deploy): | Job | Target | how | |---|---|---| | `gitea` | `192.168.0.143:3000/metrics` | metrics enabled in the quadlet (`GITEA__metrics__ENABLED=true`, `ENABLED_HOST=0.0.0.0`) — the page once said "stays off until `[metrics] ENABLED = true` in its own config"; the env form is what the deployer ships | -| `services` | `http://192.168.0.143:8081` (SearXNG), `:3005` (OpenWebUI), `http://comfyui:8188` | **blackbox exporter** probe, module `http_2xx` — the comfyui metrics job is retired, one owner per service | +| `services` | `http://192.168.0.143:8081` (SearXNG), `:3005` (OpenWebUI), `http://127.0.0.1:8188` (ComfyUI) | **blackbox exporter** probe, module `http_2xx` — the comfyui metrics job is retired, one owner per service. The prober runs in the host netns, so comfyui is probed over loopback (its published port); bridge container names do not resolve there | Scraping a web root is not a health check: an HTML page fails the scrape protocol (`unsupported Content-Type`), which is what kept those targets red beside healthy services. The classification rule and the honesty law live in [[Stale Signal Audit]]. The prober is `blackbox-exporter.container` (host -netns, config `shared/blackbox/blackbox.yml`, bound 127.0.0.1:9115; Prometheus -reaches it at the `host.containers.internal` gateway name). Plain targets stay -`host:port`, never URLs — the probe job's URL targets are the documented -exception, relabeled onto the prober. +netns, config `shared/blackbox/blackbox.yml`); it binds 9115 on both the LAN +and the podman gateway, and Prometheus scrapes the gateway address — the +netavark hairpin gap bars this container from its own host's LAN IP. +Plain targets stay `host:port`, never URLs — the probe job's URL targets are +the documented exception, relabeled onto the prober. **`frank-zealot` reads up since 2026-10-18, and did not before.** llama.cpp only answers `/metrics` when started with `--metrics`, so `up{job="frank-zealot"}` was 0 across the whole retained history (the oldest sample, 2026-10-11, is already 0) while the service answered `/health` 200 the whole time; the flag and its opt-out are on [[frank-zealot]]. Read `lastError` before believing an outage: a target can read down beside a healthy service. diff --git a/shared/blackbox/blackbox.yml b/shared/blackbox/blackbox.yml index dbf44e3..7666600 100644 --- a/shared/blackbox/blackbox.yml +++ b/shared/blackbox/blackbox.yml @@ -3,11 +3,10 @@ # 200 that fails the scrape protocol), which is why the old `services` and # `comfyui` jobs sat permanently red on `unsupported Content-Type`. # See serverAppConfigs-docs/Architecture/Stale Signal Audit.md §1. -# Loopback bind: the image's default 0.0.0.0:9115 would ride the LAN on this -# Network=host container. Prometheus (bridge) still reaches it — its traffic -# arrives on the podman gateway interface, which is not loopback. -network: - listen_address: 127.0.0.1:9115 +# The listen address is NOT set here: the quadlet unit's Exec passes +# --listen-address (LAN bind — see the unit header for why loopback is +# unreachable from the bridge-networked Prometheus). Keep this file to +# modules only, so the two sources cannot disagree. modules: http_2xx: diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index 85f05a4..05ad5b1 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -64,11 +64,13 @@ scrape_configs: # web root is NOT a health check — an HTML front page fails the scrape # protocol (`unsupported Content-Type`), which is exactly why the old # `services` and `comfyui` jobs were red forever beside healthy services - # (Stale Signal Audit §1). The prober is a host-netns container reached at - # the podman gateway name (HostAddr= on the prometheus unit); it in turn - # reaches both the host LAN IP and bridge container names. comfyui's UI probe - # lives here now — its own metrics-scrape job is retired, one owner per - # service. + # (Stale Signal Audit §1). The prober is a host-netns container listening on + # the podman gateway (the scrape address below - the netavark hairpin gap + # bars this container from its own host's LAN IP, cadvisor job note) and on + # the LAN; from that netns it reaches the host LAN IP, and published ports + # via 127.0.0.1 - bridge container names do NOT resolve in the host netns, + # which is why comfyui's probe is a loopback URL. comfyui's own + # metrics-scrape job is retired, one owner per service. - job_name: "services" metrics_path: /probe params: @@ -77,7 +79,7 @@ scrape_configs: - targets: - "http://192.168.0.143:8081" # SearXNG (via the Gluetun tunnel) - "http://192.168.0.143:3005" # OpenWebUI - - "http://comfyui:8188" # ComfyUI UI (gfx1151 generator) + - "http://127.0.0.1:8188" # ComfyUI UI (published 0.0.0.0:8188; probed from the prober's host netns) relabel_configs: - source_labels: [__address__] target_label: __param_target diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index d360cb1..36ba95e 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -669,6 +669,18 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names), [ -f "$sandbox/opt/grafana/provisioning/datasources/prometheus.yml" ] } +@test "every observability unit declares an Image" { + # Quadlet refuses to GENERATE a service for a .container without Image= + # - the deployer then dies at `systemctl restart` with "Unit not found" + # and no diagnosis (blackbox-exporter, live 2026-10-21: merged with the + # copier, gates and health check wired, but no Image line). + local svc + for svc in "${UNITS[@]}"; do + grep -q '^Image=' "$UNIT_DIR/$svc.container" \ + || { echo "$svc.container has no Image= - quadlet will not generate it"; return 1; } + done +} + @test "changes under shared/blackbox/ plan the prober" { run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \ --paths shared/blackbox/blackbox.yml @@ -716,7 +728,7 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names), copy_blackbox_config ) [ -f "$sandbox/blackbox.yml" ] - grep -q "listen_address: 127.0.0.1:9115" "$sandbox/blackbox.yml" + grep -q '^ http_2xx:' "$sandbox/blackbox.yml" } @test "the alloy copier refuses to install a config the engine rejects" { -- 2.54.0