fix(blackbox): Exec carries flags only (image has ENTRYPOINT) and the exporter binds loopback+LAN

Three stacked lessons, each proved live 2026-10-21:
- the image's ENTRYPOINT is /bin/blackbox_exporter, so Exec= repeating the
  binary made it a positional arg: 'unexpected /bin/blackbox_exporter'.
- the podman gateway 169.254.1.2 is NOT bindable from the host netns - it
  exists only inside each rootless netns: 'bind: cannot assign requested
  address'. The scrape path does not need it bound: a bridge container
  dialing the gateway lands on the host LOOPBACK (proved 200 from the
  prometheus container), so the exporter binds 127.0.0.1 + LAN.
- HealthCmd shell form ('/bin/sh -c "..."') is passed as one quoted token
  and quadlet folds the following --health-interval/--health-retries (and
  the image) into the command string; the plain exec form the sibling units
  use is correct.

Prometheus.md bind law corrected (gateway is a forward-to-loopback, not a
bind address); deploy-remote gate stays loopback, which is now honest.

dankclaw/observability/quadlet/blackbox-exporter.container

@@ -1,29 +1,50 @@
-# Quadlet unit: blackbox exporter (rootless, boot-persistent).
+# Quadlet unit: blackbox exporter (rootless, host network, boot-persistent).
 #
 # Install path on dankclaw:
 #   ~/.config/containers/systemd/blackbox-exporter.container
 #
 # WHY Network=host. The probes it runs are (a) the host's own LAN IP
-# (192.168.0.143:<port>) and (b) container names on dankclaw_default (comfyui).
+# (192.168.0.143:8081 SearXNG, :3005 OpenWebUI — host-process/portmap binds a
-# A bridge container cannot reach the host's LAN IP at an unpublished port
+# bridge container cannot reach; netavark DNATs published ports only, see
-# (netavark DNATs published ports only — see Container Host Reachability in the
+# Container Host Reachability in the vault) and (b) ComfyUI's published port
-# vault); a host-netns process reaches both. Prometheus (bridge) reaches THIS
+# via 127.0.0.1 (rootless --network=host shares the real host netns, so the
-# container the same way it reaches the signal relay: the quadlet-published
+# 0.0.0.0:8188 publish answers on loopback). A bridge-networked prober could
-# host.containers.internal address (169.254.1.2), mapped via HostAddr= on the
+# reach neither class reliably.
-# prometheus unit.
 #
-# The 9115 metrics port publishes on loopback only — it is scraped by the
+# WHY the listener binds 127.0.0.1 + LAN. A bridge container CANNOT dial
-# prometheus container through the gateway, and never opened on the LAN.
+# its own host's LAN IP (netavark hairpin gap, measured 2026-10-21), but a
+# bridge container dialing the podman gateway 169.254.1.2 lands on the host
+# LOOPBACK — that is the scrape path, and it needs this exporter to be
+# LISTENING ON LOOPBACK (proved: 200 from the prometheus container to
+# http://169.254.1.2:9115/-/healthy with a loopback bind). The gateway
+# address itself is NOT bindable from the host netns — it exists only
+# inside each rootless netns; binding it dies "bind: cannot assign
+# requested address" (live 2026-10-21). The LAN bind stays for diagnostics.
+# No public DNS name, no guard chain: internal net only, per the exposure
+# policy. The listen flag is --web.listen-address and it is REPEATABLE, one
+# flag per bind; plain --listen-address does not exist and the exporter
+# exits in 100 ms with "unknown long flag" (live 2026-10-21).
 
 [Unit]
 Description=Blackbox exporter (HTTP probe prober)
 After=network-online.target
 
 [Container]
+Image=docker.io/prom/blackbox-exporter@sha256:ea1c7aa3ef3f307cd03e85f34abb604478ccb90b567fc690da0cd9776da76b13
 ContainerName=blackbox-exporter
-Exec=/bin/blackbox_exporter --config.file=/etc/blackbox_exporter/config.yml
+Network=host
+# The image's ENTRYPOINT is /bin/blackbox_exporter itself (proved live
+# 2026-10-21: `podman run IMAGE sh` dies "unexpected sh"), so Exec= carries
+# FLAGS ONLY — repeating the binary makes it a positional arg and kingpin
+# exits "unexpected /bin/blackbox_exporter".
+Exec=--config.file=/etc/blackbox_exporter/config.yml --web.listen-address=127.0.0.1:9115 --web.listen-address=192.168.0.143:9115
 Volume=/opt/blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
-HealthCmd=/bin/sh -c "wget -q -O /dev/null http://127.0.0.1:9115/-/healthy || exit 1"
+# Plain exec form like every sibling unit. Shell form (`/bin/sh -c "…"`)
+# is passed as one quoted token and quadlet then folds the following
+# --health-interval/--health-retries (and the image name) into the command
+# string — unit died twice that way, live 2026-10-21. wget only; this
+# busybox variant has no curl, and host-network wget answers on loopback.
+HealthCmd=wget -q -O /dev/null http://127.0.0.1:9115/-/healthy
 HealthInterval=30s
 HealthRetries=3
 

dankclaw/observability/quadlet/prometheus.container

@@ -45,10 +45,11 @@ Wants=network-online.target
 ContainerName=prometheus
 Image=docker.io/prom/prometheus:latest
 Network=dankclaw_default
-# Name the podman gateway so scrape jobs can reach host-netns processes (the
+# Name the podman gateway so scrape jobs can reach same-host targets (the
-# blackbox exporter listens on the host's loopback; a bridge container reaches
+# netavark hairpin gap bars this container from dialing its own host's LAN
-# it at this address, never at 127.0.0.1). Same mechanism the hermes Caddy
+# IP; the cadvisor/gpu jobs and the blackbox scrape all ride this name).
-# upstream learned the hard way (Container Host Reachability in the vault).
+# Same mechanism the hermes Caddy upstream learned the hard way (Container
+# Host Reachability in the vault).
 HostAddr=host.containers.internal=169.254.1.2
 User=0:0
 Exec=--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=30d --web.enable-lifecycle

scripts/deploy-remote.sh

@@ -86,7 +86,7 @@ service_is_allowed() {
         dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
         dankclaw:comfyui|dankclaw:hermes|\
         dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
-        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:alertmanager|\
+        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:alertmanager|dankclaw:blackbox-exporter|\
         dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
         dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
@@ -617,6 +617,8 @@ deploy_observability() {
         loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
         node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
         cadvisor) wait_for_health cadvisor http://127.0.0.1:8080/healthz 90 ;;
+        # Loopback: the exporter binds 127.0.0.1 + LAN (the gateway address
+        # is not bindable from the host netns — see the unit header).
         blackbox-exporter) wait_for_health blackbox-exporter http://127.0.0.1:9115/-/healthy 60 ;;
     esac
 }
@@ -2086,7 +2088,7 @@ deploy_service() {
         plex) deploy_plex ;;
         tautulli) deploy_tautulli ;;
         alloy) deploy_alloy ;;
-        prometheus|grafana|loki|node-exporter|cadvisor|alertmanager) deploy_observability "$service" ;;
+        prometheus|grafana|loki|node-exporter|cadvisor|alertmanager|blackbox-exporter) deploy_observability "$service" ;;
         gpu-exporter) deploy_gpu_exporter ;;
         comfyui) deploy_comfyui ;;
         *) deploy_compose_service "$service" ;;

scripts/deploy.sh

@@ -140,7 +140,7 @@ valid_service() {
         dankclaw:gitea-server|dankclaw:homepage|\
         dankclaw:comfyui|dankclaw:hermes|dankclaw:hermes-dashboard|\
         dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|dankclaw:loki|dankclaw:alloy|\
-        dankclaw:alertmanager|\
+        dankclaw:alertmanager|dankclaw:blackbox-exporter|\
         dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|dankclaw:seerr|\
         dankclaw:open-notebook|\

serverAppConfigs-docs/Infrastructure/Observability Provisioning.md

@@ -21,7 +21,7 @@ committed.
 | `shared/prometheus/rules/llm-fleet.yml` | alert rules + recording rules |
 | `shared/grafana/provisioning/` | datasources + dashboards (provisioned tree) |
 | `shared/alloy/config.alloy` | the log pipeline: journal source → label promotion → Loki push |
-| `shared/blackbox/blackbox.yml` | prober modules (`http_2xx`) + the loopback bind |
+| `shared/blackbox/blackbox.yml` | prober modules (`http_2xx`); the listen bind lives on the unit's `Exec` |
 
 Targets and datasources reference **container names** (`grafana:3000`
 `loki:3100` — `shared/prometheus/prometheus.yml:52-61`), never

serverAppConfigs-docs/Infrastructure/Observability.md

@@ -163,6 +163,6 @@ verify it through the APIs.
 - `scripts/gpu-exporter.py`, `scripts/gpu-exporter.service` (per-GPU host exporter)
 - `shared/prometheus/prometheus.yml`, `shared/prometheus/alertmanager.yml`, `shared/prometheus/rules/llm-fleet.yml`
 - `shared/grafana/provisioning/`, `shared/loki/loki-config.yaml`, `shared/alloy/config.alloy`
-- `shared/blackbox/blackbox.yml` (prober modules + loopback bind)
+- `shared/blackbox/blackbox.yml` (prober modules; the listen bind is on the unit's `Exec`)
 - `tests/observability_quadlet.bats` (boot persistence, no wildcard publish, probe addresses, delivery contract)
 - Host evidence 2026-10-18: `ss -ltnp`, `podman ps`, `systemctl --user is-enabled|is-active`, `curl` on `127.0.0.1:9090/api/v1/{targets,rules,alerts}`, `127.0.0.1:3100/loki/api/v1/labels`, `127.0.0.1:12345/metrics`, `127.0.0.1:3002/api/health`, `192.168.0.143:18093/health`, `journalctl --user -u signal-relay`

serverAppConfigs-docs/Infrastructure/Prometheus.md

@@ -48,16 +48,18 @@ retired 2026-10-21; census re-run after each deploy):
 | Job | Target | how |
 |---|---|---|
 | `gitea` | `192.168.0.143:3000/metrics` | metrics enabled in the quadlet (`GITEA__metrics__ENABLED=true`, `ENABLED_HOST=0.0.0.0`) — the page once said "stays off until `[metrics] ENABLED = true` in its own config"; the env form is what the deployer ships |
-| `services` | `http://192.168.0.143:8081` (SearXNG), `:3005` (OpenWebUI), `http://comfyui:8188` | **blackbox exporter** probe, module `http_2xx` — the comfyui metrics job is retired, one owner per service |
+| `services` | `http://192.168.0.143:8081` (SearXNG), `:3005` (OpenWebUI), `http://127.0.0.1:8188` (ComfyUI) | **blackbox exporter** probe, module `http_2xx` — the comfyui metrics job is retired, one owner per service. The prober runs in the host netns, so comfyui is probed over loopback (its published port); bridge container names do not resolve there |
 
 Scraping a web root is not a health check: an HTML page fails the scrape
 protocol (`unsupported Content-Type`), which is what kept those targets red
 beside healthy services. The classification rule and the honesty law live in
 [[Stale Signal Audit]]. The prober is `blackbox-exporter.container` (host
-netns, config `shared/blackbox/blackbox.yml`, bound 127.0.0.1:9115; Prometheus
+netns, config `shared/blackbox/blackbox.yml`); it binds 9115 on loopback and
-reaches it at the `host.containers.internal` gateway name). Plain targets stay
+the LAN, and Prometheus scrapes it at `host.containers.internal` — traffic to
-`host:port`, never URLs — the probe job's URL targets are the documented
+that gateway lands on the host's loopback listener. The gateway address is
-exception, relabeled onto the prober.
+not itself bindable from the host netns.
+Plain targets stay `host:port`, never URLs — the probe job's URL targets are
+the documented exception, relabeled onto the prober.
 
 **`frank-zealot` reads up since 2026-10-18, and did not before.** llama.cpp only answers `/metrics` when started with `--metrics`, so `up{job="frank-zealot"}` was 0 across the whole retained history (the oldest sample, 2026-10-11, is already 0) while the service answered `/health` 200 the whole time; the flag and its opt-out are on [[frank-zealot]]. Read `lastError` before believing an outage: a target can read down beside a healthy service.
 

shared/blackbox/blackbox.yml

@@ -3,11 +3,10 @@
 # 200 that fails the scrape protocol), which is why the old `services` and
 # `comfyui` jobs sat permanently red on `unsupported Content-Type`.
 # See serverAppConfigs-docs/Architecture/Stale Signal Audit.md §1.
-# Loopback bind: the image's default 0.0.0.0:9115 would ride the LAN on this
+# The listen address is NOT set here: the quadlet unit's Exec passes
-# Network=host container. Prometheus (bridge) still reaches it — its traffic
+# --listen-address (LAN bind — see the unit header for why loopback is
-# arrives on the podman gateway interface, which is not loopback.
+# unreachable from the bridge-networked Prometheus). Keep this file to
-network:
+# modules only, so the two sources cannot disagree.
-  listen_address: 127.0.0.1:9115
 
 modules:
   http_2xx:

shared/prometheus/prometheus.yml

@@ -64,11 +64,13 @@ scrape_configs:
   # web root is NOT a health check — an HTML front page fails the scrape
   # protocol (`unsupported Content-Type`), which is exactly why the old
   # `services` and `comfyui` jobs were red forever beside healthy services
-  # (Stale Signal Audit §1). The prober is a host-netns container reached at
+  # (Stale Signal Audit §1). The prober is a host-netns container listening on
-  # the podman gateway name (HostAddr= on the prometheus unit); it in turn
+  # the podman gateway (the scrape address below - the netavark hairpin gap
-  # reaches both the host LAN IP and bridge container names. comfyui's UI probe
+  # bars this container from its own host's LAN IP, cadvisor job note) and on
-  # lives here now — its own metrics-scrape job is retired, one owner per
+  # the LAN; from that netns it reaches the host LAN IP, and published ports
-  # service.
+  # via 127.0.0.1 - bridge container names do NOT resolve in the host netns,
+  # which is why comfyui's probe is a loopback URL. comfyui's own
+  # metrics-scrape job is retired, one owner per service.
   - job_name: "services"
     metrics_path: /probe
     params:
@@ -77,7 +79,7 @@ scrape_configs:
       - targets:
           - "http://192.168.0.143:8081"   # SearXNG (via the Gluetun tunnel)
           - "http://192.168.0.143:3005"   # OpenWebUI
-          - "http://comfyui:8188"         # ComfyUI UI (gfx1151 generator)
+          - "http://127.0.0.1:8188"        # ComfyUI UI (published 0.0.0.0:8188; probed from the prober's host netns)
     relabel_configs:
       - source_labels: [__address__]
         target_label: __param_target

tests/deployment_flow.bats

@@ -150,3 +150,23 @@ load helpers
         grep -q "$pair" "$BATS_TEST_TMPDIR/cli" || { echo "client gate missing $pair"; return 1; }
     done
 }
+
+@test "every gated dankclaw service is dispatched" {
+    # A service can clear both allowlists and still die inside the deployer:
+    # dankclaw has no compose file, and deploy_compose_service (the case
+    # catch-all) hard-fails with "no quadlet handler" since 2026-09-16. So
+    # every name the remote gate allows for dankclaw MUST be named by a
+    # deploy_service pattern — blackbox-exporter (2026-10-21) shipped with
+    # both gates and the health gate wired but no dispatch case, and only
+    # the deploy attempt found it.
+    run env OUT="$BATS_TEST_TMPDIR" bash -c '
+        grep -oE "dankclaw:[a-z0-9-]+" \
+            <(sed -n "/^service_is_allowed()/,/^}/p" "$1/scripts/deploy-remote.sh") \
+            | cut -d: -f2 | sort -u > "$OUT/gated"
+        sed -n "/^deploy_service()/,/^}/p" "$1/scripts/deploy-remote.sh" \
+            | grep -oE "^ +[a-z][a-z0-9|-]*\)" | tr -d " )" | tr "|" "\n" \
+            | sort -u > "$OUT/dispatched"
+        comm -23 "$OUT/gated" "$OUT/dispatched"
+    ' _ "$REPO_ROOT"
+    [ -z "$output" ] || { echo "gated dankclaw services with no deploy_service case:"; echo "$output"; return 1; }
+}

tests/observability_quadlet.bats

@@ -669,6 +669,18 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names),
     [ -f "$sandbox/opt/grafana/provisioning/datasources/prometheus.yml" ]
 }
 
+@test "every observability unit declares an Image" {
+    # Quadlet refuses to GENERATE a service for a .container without Image=
+    # - the deployer then dies at `systemctl restart` with "Unit not found"
+    # and no diagnosis (blackbox-exporter, live 2026-10-21: merged with the
+    # copier, gates and health check wired, but no Image line).
+    local svc
+    for svc in "${UNITS[@]}"; do
+        grep -q '^Image=' "$UNIT_DIR/$svc.container" \
+            || { echo "$svc.container has no Image= - quadlet will not generate it"; return 1; }
+    done
+}
+
 @test "changes under shared/blackbox/ plan the prober" {
     run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
         --paths shared/blackbox/blackbox.yml
@@ -716,7 +728,7 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names),
         copy_blackbox_config
     )
     [ -f "$sandbox/blackbox.yml" ]
-    grep -q "listen_address: 127.0.0.1:9115" "$sandbox/blackbox.yml"
+    grep -q '^  http_2xx:' "$sandbox/blackbox.yml"
 }
 
 @test "the alloy copier refuses to install a config the engine rejects" {