fix(blackbox): unit had no Image and no Network=host - quadlet refused to generate it

The deploy died at 'Unit blackbox-exporter.service not found': a
.container without Image= is not generated at all, and the header
argued for Network=host it never declared. Completes the unit (digest
pinned prom/blackbox-exporter, Network=host) and fixes the bind
topology the first cut got wrong:

- Prometheus is bridge-networked and the netavark hairpin gap bars it
  from its own host's LAN IP, so a LAN-only prober bind is unscrapable
  and a loopback-only bind was claimed to ride the gateway on faith.
  The prober now binds 9115 on the podman gateway (scrape path) AND
  the LAN (healthcheck/diagnostics); the listen_address line leaves
  blackbox.yml so config and Exec cannot disagree.
- comfyui probe target http://comfyui:8188 -> http://127.0.0.1:8188:
  bridge container names do not resolve in the host netns; the
  published 0.0.0.0:8188 answers on loopback there.
- New seam: every observability unit must declare Image= (red on the
  shipped file by name); copier test pinned module presence instead
  of the removed listen line. Docs updated to the real topology.

dankclaw/observability/quadlet/blackbox-exporter.container

@@ -1,29 +1,38 @@
-# Quadlet unit: blackbox exporter (rootless, boot-persistent).
+# Quadlet unit: blackbox exporter (rootless, host network, boot-persistent).
 #
 # Install path on dankclaw:
 #   ~/.config/containers/systemd/blackbox-exporter.container
 #
 # WHY Network=host. The probes it runs are (a) the host's own LAN IP
-# (192.168.0.143:<port>) and (b) container names on dankclaw_default (comfyui).
-# A bridge container cannot reach the host's LAN IP at an unpublished port
-# (netavark DNATs published ports only — see Container Host Reachability in the
-# vault); a host-netns process reaches both. Prometheus (bridge) reaches THIS
-# container the same way it reaches the signal relay: the quadlet-published
-# host.containers.internal address (169.254.1.2), mapped via HostAddr= on the
-# prometheus unit.
+# (192.168.0.143:8081 SearXNG, :3005 OpenWebUI — host-process/portmap binds a
+# bridge container cannot reach; netavark DNATs published ports only, see
+# Container Host Reachability in the vault) and (b) ComfyUI's published port
+# via 127.0.0.1 (rootless --network=host shares the real host netns, so the
+# 0.0.0.0:8188 publish answers on loopback). A bridge-networked prober could
+# reach neither class reliably.
 #
-# The 9115 metrics port publishes on loopback only — it is scraped by the
-# prometheus container through the gateway, and never opened on the LAN.
+# WHY the listener binds TWO addresses. The netavark hairpin gap (measured
+# 2026-10-21, cadvisor job comment): a BRIDGE CONTAINER CANNOT DIAL ITS OWN
+# HOST'S LAN IP at all — so a LAN-only bind is unscrapable by Prometheus,
+# whatever the signal-relay law suggests for services containers dial as
+# clients. The podman gateway (169.254.1.2 = host.containers.internal) is the
+# address bridge containers reach same-host targets at, so the exporter binds
+# it explicitly for the scrape rather than relying on loopback-forwarding
+# semantics, and keeps the LAN bind for its own healthcheck and diagnostics.
+# No public DNS name, no guard chain: internal net only, per the exposure
+# policy.
 
 [Unit]
 Description=Blackbox exporter (HTTP probe prober)
 After=network-online.target
 
 [Container]
+Image=docker.io/prom/blackbox-exporter@sha256:ea1c7aa3ef3f307cd03e85f34abb604478ccb90b567fc690da0cd9776da76b13
 ContainerName=blackbox-exporter
-Exec=/bin/blackbox_exporter --config.file=/etc/blackbox_exporter/config.yml
+Network=host
+Exec=/bin/blackbox_exporter --config.file=/etc/blackbox_exporter/config.yml --listen-address=192.168.0.143:9115,169.254.1.2:9115
 Volume=/opt/blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
-HealthCmd=/bin/sh -c "wget -q -O /dev/null http://127.0.0.1:9115/-/healthy || exit 1"
+HealthCmd=/bin/sh -c "wget -q -O /dev/null http://169.254.1.2:9115/-/healthy || exit 1"
 HealthInterval=30s
 HealthRetries=3
 

dankclaw/observability/quadlet/prometheus.container

@@ -45,10 +45,11 @@ Wants=network-online.target
 ContainerName=prometheus
 Image=docker.io/prom/prometheus:latest
 Network=dankclaw_default
-# Name the podman gateway so scrape jobs can reach host-netns processes (the
-# blackbox exporter listens on the host's loopback; a bridge container reaches
-# it at this address, never at 127.0.0.1). Same mechanism the hermes Caddy
-# upstream learned the hard way (Container Host Reachability in the vault).
+# Name the podman gateway so scrape jobs can reach same-host targets (the
+# netavark hairpin gap bars this container from dialing its own host's LAN
+# IP; the cadvisor/gpu jobs and the blackbox scrape all ride this name).
+# Same mechanism the hermes Caddy upstream learned the hard way (Container
+# Host Reachability in the vault).
 HostAddr=host.containers.internal=169.254.1.2
 User=0:0
 Exec=--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=30d --web.enable-lifecycle

scripts/deploy-remote.sh

@@ -86,7 +86,7 @@ service_is_allowed() {
         dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
         dankclaw:comfyui|dankclaw:hermes|\
         dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
-        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:alertmanager|\
+        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:alertmanager|dankclaw:blackbox-exporter|\
         dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
         dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
@@ -2086,7 +2086,7 @@ deploy_service() {
         plex) deploy_plex ;;
         tautulli) deploy_tautulli ;;
         alloy) deploy_alloy ;;
-        prometheus|grafana|loki|node-exporter|cadvisor|alertmanager) deploy_observability "$service" ;;
+        prometheus|grafana|loki|node-exporter|cadvisor|alertmanager|blackbox-exporter) deploy_observability "$service" ;;
         gpu-exporter) deploy_gpu_exporter ;;
         comfyui) deploy_comfyui ;;
         *) deploy_compose_service "$service" ;;

scripts/deploy.sh

@@ -140,7 +140,7 @@ valid_service() {
         dankclaw:gitea-server|dankclaw:homepage|\
         dankclaw:comfyui|dankclaw:hermes|dankclaw:hermes-dashboard|\
         dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|dankclaw:loki|dankclaw:alloy|\
-        dankclaw:alertmanager|\
+        dankclaw:alertmanager|dankclaw:blackbox-exporter|\
         dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|dankclaw:seerr|\
         dankclaw:open-notebook|\

serverAppConfigs-docs/Infrastructure/Observability Provisioning.md

@@ -21,7 +21,7 @@ committed.
 | `shared/prometheus/rules/llm-fleet.yml` | alert rules + recording rules |
 | `shared/grafana/provisioning/` | datasources + dashboards (provisioned tree) |
 | `shared/alloy/config.alloy` | the log pipeline: journal source → label promotion → Loki push |
-| `shared/blackbox/blackbox.yml` | prober modules (`http_2xx`) + the loopback bind |
+| `shared/blackbox/blackbox.yml` | prober modules (`http_2xx`); the listen bind lives on the unit's `Exec` |
 
 Targets and datasources reference **container names** (`grafana:3000`
 `loki:3100` — `shared/prometheus/prometheus.yml:52-61`), never

serverAppConfigs-docs/Infrastructure/Observability.md

@@ -163,6 +163,6 @@ verify it through the APIs.
 - `scripts/gpu-exporter.py`, `scripts/gpu-exporter.service` (per-GPU host exporter)
 - `shared/prometheus/prometheus.yml`, `shared/prometheus/alertmanager.yml`, `shared/prometheus/rules/llm-fleet.yml`
 - `shared/grafana/provisioning/`, `shared/loki/loki-config.yaml`, `shared/alloy/config.alloy`
-- `shared/blackbox/blackbox.yml` (prober modules + loopback bind)
+- `shared/blackbox/blackbox.yml` (prober modules; the listen bind is on the unit's `Exec`)
 - `tests/observability_quadlet.bats` (boot persistence, no wildcard publish, probe addresses, delivery contract)
 - Host evidence 2026-10-18: `ss -ltnp`, `podman ps`, `systemctl --user is-enabled|is-active`, `curl` on `127.0.0.1:9090/api/v1/{targets,rules,alerts}`, `127.0.0.1:3100/loki/api/v1/labels`, `127.0.0.1:12345/metrics`, `127.0.0.1:3002/api/health`, `192.168.0.143:18093/health`, `journalctl --user -u signal-relay`

serverAppConfigs-docs/Infrastructure/Prometheus.md

@@ -48,16 +48,17 @@ retired 2026-10-21; census re-run after each deploy):
 | Job | Target | how |
 |---|---|---|
 | `gitea` | `192.168.0.143:3000/metrics` | metrics enabled in the quadlet (`GITEA__metrics__ENABLED=true`, `ENABLED_HOST=0.0.0.0`) — the page once said "stays off until `[metrics] ENABLED = true` in its own config"; the env form is what the deployer ships |
-| `services` | `http://192.168.0.143:8081` (SearXNG), `:3005` (OpenWebUI), `http://comfyui:8188` | **blackbox exporter** probe, module `http_2xx` — the comfyui metrics job is retired, one owner per service |
+| `services` | `http://192.168.0.143:8081` (SearXNG), `:3005` (OpenWebUI), `http://127.0.0.1:8188` (ComfyUI) | **blackbox exporter** probe, module `http_2xx` — the comfyui metrics job is retired, one owner per service. The prober runs in the host netns, so comfyui is probed over loopback (its published port); bridge container names do not resolve there |
 
 Scraping a web root is not a health check: an HTML page fails the scrape
 protocol (`unsupported Content-Type`), which is what kept those targets red
 beside healthy services. The classification rule and the honesty law live in
 [[Stale Signal Audit]]. The prober is `blackbox-exporter.container` (host
-netns, config `shared/blackbox/blackbox.yml`, bound 127.0.0.1:9115; Prometheus
-reaches it at the `host.containers.internal` gateway name). Plain targets stay
-`host:port`, never URLs — the probe job's URL targets are the documented
-exception, relabeled onto the prober.
+netns, config `shared/blackbox/blackbox.yml`); it binds 9115 on both the LAN
+and the podman gateway, and Prometheus scrapes the gateway address — the
+netavark hairpin gap bars this container from its own host's LAN IP.
+Plain targets stay `host:port`, never URLs — the probe job's URL targets are
+the documented exception, relabeled onto the prober.
 
 **`frank-zealot` reads up since 2026-10-18, and did not before.** llama.cpp only answers `/metrics` when started with `--metrics`, so `up{job="frank-zealot"}` was 0 across the whole retained history (the oldest sample, 2026-10-11, is already 0) while the service answered `/health` 200 the whole time; the flag and its opt-out are on [[frank-zealot]]. Read `lastError` before believing an outage: a target can read down beside a healthy service.
 

shared/blackbox/blackbox.yml

@@ -3,11 +3,10 @@
 # 200 that fails the scrape protocol), which is why the old `services` and
 # `comfyui` jobs sat permanently red on `unsupported Content-Type`.
 # See serverAppConfigs-docs/Architecture/Stale Signal Audit.md §1.
-# Loopback bind: the image's default 0.0.0.0:9115 would ride the LAN on this
-# Network=host container. Prometheus (bridge) still reaches it — its traffic
-# arrives on the podman gateway interface, which is not loopback.
-network:
-  listen_address: 127.0.0.1:9115
+# The listen address is NOT set here: the quadlet unit's Exec passes
+# --listen-address (LAN bind — see the unit header for why loopback is
+# unreachable from the bridge-networked Prometheus). Keep this file to
+# modules only, so the two sources cannot disagree.
 
 modules:
   http_2xx:

shared/prometheus/prometheus.yml

@@ -64,11 +64,13 @@ scrape_configs:
   # web root is NOT a health check — an HTML front page fails the scrape
   # protocol (`unsupported Content-Type`), which is exactly why the old
   # `services` and `comfyui` jobs were red forever beside healthy services
-  # (Stale Signal Audit §1). The prober is a host-netns container reached at
-  # the podman gateway name (HostAddr= on the prometheus unit); it in turn
-  # reaches both the host LAN IP and bridge container names. comfyui's UI probe
-  # lives here now — its own metrics-scrape job is retired, one owner per
-  # service.
+  # (Stale Signal Audit §1). The prober is a host-netns container listening on
+  # the podman gateway (the scrape address below - the netavark hairpin gap
+  # bars this container from its own host's LAN IP, cadvisor job note) and on
+  # the LAN; from that netns it reaches the host LAN IP, and published ports
+  # via 127.0.0.1 - bridge container names do NOT resolve in the host netns,
+  # which is why comfyui's probe is a loopback URL. comfyui's own
+  # metrics-scrape job is retired, one owner per service.
   - job_name: "services"
     metrics_path: /probe
     params:
@@ -77,7 +79,7 @@ scrape_configs:
       - targets:
           - "http://192.168.0.143:8081"   # SearXNG (via the Gluetun tunnel)
           - "http://192.168.0.143:3005"   # OpenWebUI
-          - "http://comfyui:8188"         # ComfyUI UI (gfx1151 generator)
+          - "http://127.0.0.1:8188"        # ComfyUI UI (published 0.0.0.0:8188; probed from the prober's host netns)
     relabel_configs:
       - source_labels: [__address__]
         target_label: __param_target

tests/deployment_flow.bats

@@ -150,3 +150,23 @@ load helpers
         grep -q "$pair" "$BATS_TEST_TMPDIR/cli" || { echo "client gate missing $pair"; return 1; }
     done
 }
+
+@test "every gated dankclaw service is dispatched" {
+    # A service can clear both allowlists and still die inside the deployer:
+    # dankclaw has no compose file, and deploy_compose_service (the case
+    # catch-all) hard-fails with "no quadlet handler" since 2026-09-16. So
+    # every name the remote gate allows for dankclaw MUST be named by a
+    # deploy_service pattern — blackbox-exporter (2026-10-21) shipped with
+    # both gates and the health gate wired but no dispatch case, and only
+    # the deploy attempt found it.
+    run env OUT="$BATS_TEST_TMPDIR" bash -c '
+        grep -oE "dankclaw:[a-z0-9-]+" \
+            <(sed -n "/^service_is_allowed()/,/^}/p" "$1/scripts/deploy-remote.sh") \
+            | cut -d: -f2 | sort -u > "$OUT/gated"
+        sed -n "/^deploy_service()/,/^}/p" "$1/scripts/deploy-remote.sh" \
+            | grep -oE "^ +[a-z][a-z0-9|-]*\)" | tr -d " )" | tr "|" "\n" \
+            | sort -u > "$OUT/dispatched"
+        comm -23 "$OUT/gated" "$OUT/dispatched"
+    ' _ "$REPO_ROOT"
+    [ -z "$output" ] || { echo "gated dankclaw services with no deploy_service case:"; echo "$output"; return 1; }
+}

tests/observability_quadlet.bats

@@ -669,6 +669,18 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names),
     [ -f "$sandbox/opt/grafana/provisioning/datasources/prometheus.yml" ]
 }
 
+@test "every observability unit declares an Image" {
+    # Quadlet refuses to GENERATE a service for a .container without Image=
+    # - the deployer then dies at `systemctl restart` with "Unit not found"
+    # and no diagnosis (blackbox-exporter, live 2026-10-21: merged with the
+    # copier, gates and health check wired, but no Image line).
+    local svc
+    for svc in "${UNITS[@]}"; do
+        grep -q '^Image=' "$UNIT_DIR/$svc.container" \
+            || { echo "$svc.container has no Image= - quadlet will not generate it"; return 1; }
+    done
+}
+
 @test "changes under shared/blackbox/ plan the prober" {
     run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
         --paths shared/blackbox/blackbox.yml
@@ -716,7 +728,7 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names),
         copy_blackbox_config
     )
     [ -f "$sandbox/blackbox.yml" ]
-    grep -q "listen_address: 127.0.0.1:9115" "$sandbox/blackbox.yml"
+    grep -q '^  http_2xx:' "$sandbox/blackbox.yml"
 }
 
 @test "the alloy copier refuses to install a config the engine rejects" {