fix(blackbox): Exec carries flags only (image has ENTRYPOINT) and the exporter binds loopback+LAN

Three stacked lessons, each proved live 2026-10-21:
- the image's ENTRYPOINT is /bin/blackbox_exporter, so Exec= repeating the
  binary made it a positional arg: 'unexpected /bin/blackbox_exporter'.
- the podman gateway 169.254.1.2 is NOT bindable from the host netns - it
  exists only inside each rootless netns: 'bind: cannot assign requested
  address'. The scrape path does not need it bound: a bridge container
  dialing the gateway lands on the host LOOPBACK (proved 200 from the
  prometheus container), so the exporter binds 127.0.0.1 + LAN.
- HealthCmd shell form ('/bin/sh -c "..."') is passed as one quoted token
  and quadlet folds the following --health-interval/--health-retries (and
  the image) into the command string; the plain exec form the sibling units
  use is correct.

Prometheus.md bind law corrected (gateway is a forward-to-loopback, not a
bind address); deploy-remote gate stays loopback, which is now honest.

dankclaw/observability/quadlet/blackbox-exporter.container

@@ -1,29 +1,50 @@
-# Quadlet unit: blackbox exporter (rootless, boot-persistent).
+# Quadlet unit: blackbox exporter (rootless, host network, boot-persistent).
 #
 # Install path on dankclaw:
 #   ~/.config/containers/systemd/blackbox-exporter.container
 #
 # WHY Network=host. The probes it runs are (a) the host's own LAN IP
-# (192.168.0.143:<port>) and (b) container names on dankclaw_default (comfyui).
-# A bridge container cannot reach the host's LAN IP at an unpublished port
-# (netavark DNATs published ports only — see Container Host Reachability in the
-# vault); a host-netns process reaches both. Prometheus (bridge) reaches THIS
-# container the same way it reaches the signal relay: the quadlet-published
-# host.containers.internal address (169.254.1.2), mapped via HostAddr= on the
-# prometheus unit.
+# (192.168.0.143:8081 SearXNG, :3005 OpenWebUI — host-process/portmap binds a
+# bridge container cannot reach; netavark DNATs published ports only, see
+# Container Host Reachability in the vault) and (b) ComfyUI's published port
+# via 127.0.0.1 (rootless --network=host shares the real host netns, so the
+# 0.0.0.0:8188 publish answers on loopback). A bridge-networked prober could
+# reach neither class reliably.
 #
-# The 9115 metrics port publishes on loopback only — it is scraped by the
-# prometheus container through the gateway, and never opened on the LAN.
+# WHY the listener binds 127.0.0.1 + LAN. A bridge container CANNOT dial
+# its own host's LAN IP (netavark hairpin gap, measured 2026-10-21), but a
+# bridge container dialing the podman gateway 169.254.1.2 lands on the host
+# LOOPBACK — that is the scrape path, and it needs this exporter to be
+# LISTENING ON LOOPBACK (proved: 200 from the prometheus container to
+# http://169.254.1.2:9115/-/healthy with a loopback bind). The gateway
+# address itself is NOT bindable from the host netns — it exists only
+# inside each rootless netns; binding it dies "bind: cannot assign
+# requested address" (live 2026-10-21). The LAN bind stays for diagnostics.
+# No public DNS name, no guard chain: internal net only, per the exposure
+# policy. The listen flag is --web.listen-address and it is REPEATABLE, one
+# flag per bind; plain --listen-address does not exist and the exporter
+# exits in 100 ms with "unknown long flag" (live 2026-10-21).
 
 [Unit]
 Description=Blackbox exporter (HTTP probe prober)
 After=network-online.target
 
 [Container]
+Image=docker.io/prom/blackbox-exporter@sha256:ea1c7aa3ef3f307cd03e85f34abb604478ccb90b567fc690da0cd9776da76b13
 ContainerName=blackbox-exporter
-Exec=/bin/blackbox_exporter --config.file=/etc/blackbox_exporter/config.yml
+Network=host
+# The image's ENTRYPOINT is /bin/blackbox_exporter itself (proved live
+# 2026-10-21: `podman run IMAGE sh` dies "unexpected sh"), so Exec= carries
+# FLAGS ONLY — repeating the binary makes it a positional arg and kingpin
+# exits "unexpected /bin/blackbox_exporter".
+Exec=--config.file=/etc/blackbox_exporter/config.yml --web.listen-address=127.0.0.1:9115 --web.listen-address=192.168.0.143:9115
 Volume=/opt/blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
-HealthCmd=/bin/sh -c "wget -q -O /dev/null http://127.0.0.1:9115/-/healthy || exit 1"
+# Plain exec form like every sibling unit. Shell form (`/bin/sh -c "…"`)
+# is passed as one quoted token and quadlet then folds the following
+# --health-interval/--health-retries (and the image name) into the command
+# string — unit died twice that way, live 2026-10-21. wget only; this
+# busybox variant has no curl, and host-network wget answers on loopback.
+HealthCmd=wget -q -O /dev/null http://127.0.0.1:9115/-/healthy
 HealthInterval=30s
 HealthRetries=3
 

dankclaw/observability/quadlet/prometheus.container

@@ -45,10 +45,11 @@ Wants=network-online.target
 ContainerName=prometheus
 Image=docker.io/prom/prometheus:latest
 Network=dankclaw_default
-# Name the podman gateway so scrape jobs can reach host-netns processes (the
-# blackbox exporter listens on the host's loopback; a bridge container reaches
-# it at this address, never at 127.0.0.1). Same mechanism the hermes Caddy
-# upstream learned the hard way (Container Host Reachability in the vault).
+# Name the podman gateway so scrape jobs can reach same-host targets (the
+# netavark hairpin gap bars this container from dialing its own host's LAN
+# IP; the cadvisor/gpu jobs and the blackbox scrape all ride this name).
+# Same mechanism the hermes Caddy upstream learned the hard way (Container
+# Host Reachability in the vault).
 HostAddr=host.containers.internal=169.254.1.2
 User=0:0
 Exec=--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=30d --web.enable-lifecycle

scripts/deploy-remote.sh

@@ -617,6 +617,8 @@ deploy_observability() {
         loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
         node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
         cadvisor) wait_for_health cadvisor http://127.0.0.1:8080/healthz 90 ;;
+        # Loopback: the exporter binds 127.0.0.1 + LAN (the gateway address
+        # is not bindable from the host netns — see the unit header).
         blackbox-exporter) wait_for_health blackbox-exporter http://127.0.0.1:9115/-/healthy 60 ;;
     esac
 }

serverAppConfigs-docs/Infrastructure/Observability Provisioning.md

@@ -21,7 +21,7 @@ committed.
 | `shared/prometheus/rules/llm-fleet.yml` | alert rules + recording rules |
 | `shared/grafana/provisioning/` | datasources + dashboards (provisioned tree) |
 | `shared/alloy/config.alloy` | the log pipeline: journal source → label promotion → Loki push |
-| `shared/blackbox/blackbox.yml` | prober modules (`http_2xx`) + the loopback bind |
+| `shared/blackbox/blackbox.yml` | prober modules (`http_2xx`); the listen bind lives on the unit's `Exec` |
 
 Targets and datasources reference **container names** (`grafana:3000`
 `loki:3100` — `shared/prometheus/prometheus.yml:52-61`), never

serverAppConfigs-docs/Infrastructure/Observability.md

@@ -163,6 +163,6 @@ verify it through the APIs.
 - `scripts/gpu-exporter.py`, `scripts/gpu-exporter.service` (per-GPU host exporter)
 - `shared/prometheus/prometheus.yml`, `shared/prometheus/alertmanager.yml`, `shared/prometheus/rules/llm-fleet.yml`
 - `shared/grafana/provisioning/`, `shared/loki/loki-config.yaml`, `shared/alloy/config.alloy`
-- `shared/blackbox/blackbox.yml` (prober modules + loopback bind)
+- `shared/blackbox/blackbox.yml` (prober modules; the listen bind is on the unit's `Exec`)
 - `tests/observability_quadlet.bats` (boot persistence, no wildcard publish, probe addresses, delivery contract)
 - Host evidence 2026-10-18: `ss -ltnp`, `podman ps`, `systemctl --user is-enabled|is-active`, `curl` on `127.0.0.1:9090/api/v1/{targets,rules,alerts}`, `127.0.0.1:3100/loki/api/v1/labels`, `127.0.0.1:12345/metrics`, `127.0.0.1:3002/api/health`, `192.168.0.143:18093/health`, `journalctl --user -u signal-relay`

serverAppConfigs-docs/Infrastructure/Prometheus.md

@@ -48,16 +48,18 @@ retired 2026-10-21; census re-run after each deploy):
 | Job | Target | how |
 |---|---|---|
 | `gitea` | `192.168.0.143:3000/metrics` | metrics enabled in the quadlet (`GITEA__metrics__ENABLED=true`, `ENABLED_HOST=0.0.0.0`) — the page once said "stays off until `[metrics] ENABLED = true` in its own config"; the env form is what the deployer ships |
-| `services` | `http://192.168.0.143:8081` (SearXNG), `:3005` (OpenWebUI), `http://comfyui:8188` | **blackbox exporter** probe, module `http_2xx` — the comfyui metrics job is retired, one owner per service |
+| `services` | `http://192.168.0.143:8081` (SearXNG), `:3005` (OpenWebUI), `http://127.0.0.1:8188` (ComfyUI) | **blackbox exporter** probe, module `http_2xx` — the comfyui metrics job is retired, one owner per service. The prober runs in the host netns, so comfyui is probed over loopback (its published port); bridge container names do not resolve there |
 
 Scraping a web root is not a health check: an HTML page fails the scrape
 protocol (`unsupported Content-Type`), which is what kept those targets red
 beside healthy services. The classification rule and the honesty law live in
 [[Stale Signal Audit]]. The prober is `blackbox-exporter.container` (host
-netns, config `shared/blackbox/blackbox.yml`, bound 127.0.0.1:9115; Prometheus
-reaches it at the `host.containers.internal` gateway name). Plain targets stay
-`host:port`, never URLs — the probe job's URL targets are the documented
-exception, relabeled onto the prober.
+netns, config `shared/blackbox/blackbox.yml`); it binds 9115 on loopback and
+the LAN, and Prometheus scrapes it at `host.containers.internal` — traffic to
+that gateway lands on the host's loopback listener. The gateway address is
+not itself bindable from the host netns.
+Plain targets stay `host:port`, never URLs — the probe job's URL targets are
+the documented exception, relabeled onto the prober.
 
 **`frank-zealot` reads up since 2026-10-18, and did not before.** llama.cpp only answers `/metrics` when started with `--metrics`, so `up{job="frank-zealot"}` was 0 across the whole retained history (the oldest sample, 2026-10-11, is already 0) while the service answered `/health` 200 the whole time; the flag and its opt-out are on [[frank-zealot]]. Read `lastError` before believing an outage: a target can read down beside a healthy service.
 

shared/blackbox/blackbox.yml

@@ -3,11 +3,10 @@
 # 200 that fails the scrape protocol), which is why the old `services` and
 # `comfyui` jobs sat permanently red on `unsupported Content-Type`.
 # See serverAppConfigs-docs/Architecture/Stale Signal Audit.md §1.
-# Loopback bind: the image's default 0.0.0.0:9115 would ride the LAN on this
-# Network=host container. Prometheus (bridge) still reaches it — its traffic
-# arrives on the podman gateway interface, which is not loopback.
-network:
-  listen_address: 127.0.0.1:9115
+# The listen address is NOT set here: the quadlet unit's Exec passes
+# --listen-address (LAN bind — see the unit header for why loopback is
+# unreachable from the bridge-networked Prometheus). Keep this file to
+# modules only, so the two sources cannot disagree.
 
 modules:
   http_2xx:

shared/prometheus/prometheus.yml

@@ -64,11 +64,13 @@ scrape_configs:
   # web root is NOT a health check — an HTML front page fails the scrape
   # protocol (`unsupported Content-Type`), which is exactly why the old
   # `services` and `comfyui` jobs were red forever beside healthy services
-  # (Stale Signal Audit §1). The prober is a host-netns container reached at
-  # the podman gateway name (HostAddr= on the prometheus unit); it in turn
-  # reaches both the host LAN IP and bridge container names. comfyui's UI probe
-  # lives here now — its own metrics-scrape job is retired, one owner per
-  # service.
+  # (Stale Signal Audit §1). The prober is a host-netns container listening on
+  # the podman gateway (the scrape address below - the netavark hairpin gap
+  # bars this container from its own host's LAN IP, cadvisor job note) and on
+  # the LAN; from that netns it reaches the host LAN IP, and published ports
+  # via 127.0.0.1 - bridge container names do NOT resolve in the host netns,
+  # which is why comfyui's probe is a loopback URL. comfyui's own
+  # metrics-scrape job is retired, one owner per service.
   - job_name: "services"
     metrics_path: /probe
     params:
@@ -77,7 +79,7 @@ scrape_configs:
       - targets:
           - "http://192.168.0.143:8081"   # SearXNG (via the Gluetun tunnel)
           - "http://192.168.0.143:3005"   # OpenWebUI
-          - "http://comfyui:8188"         # ComfyUI UI (gfx1151 generator)
+          - "http://127.0.0.1:8188"        # ComfyUI UI (published 0.0.0.0:8188; probed from the prober's host netns)
     relabel_configs:
       - source_labels: [__address__]
         target_label: __param_target

tests/observability_quadlet.bats

@@ -669,6 +669,18 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names),
     [ -f "$sandbox/opt/grafana/provisioning/datasources/prometheus.yml" ]
 }
 
+@test "every observability unit declares an Image" {
+    # Quadlet refuses to GENERATE a service for a .container without Image=
+    # - the deployer then dies at `systemctl restart` with "Unit not found"
+    # and no diagnosis (blackbox-exporter, live 2026-10-21: merged with the
+    # copier, gates and health check wired, but no Image line).
+    local svc
+    for svc in "${UNITS[@]}"; do
+        grep -q '^Image=' "$UNIT_DIR/$svc.container" \
+            || { echo "$svc.container has no Image= - quadlet will not generate it"; return 1; }
+    done
+}
+
 @test "changes under shared/blackbox/ plan the prober" {
     run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
         --paths shared/blackbox/blackbox.yml
@@ -716,7 +728,7 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names),
         copy_blackbox_config
     )
     [ -f "$sandbox/blackbox.yml" ]
-    grep -q "listen_address: 127.0.0.1:9115" "$sandbox/blackbox.yml"
+    grep -q '^  http_2xx:' "$sandbox/blackbox.yml"
 }
 
 @test "the alloy copier refuses to install a config the engine rejects" {