From f073b91033860f2666ff4e068a296e45bc67dfbe Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:32:58 -0700 Subject: [PATCH] feat(observability): blackbox exporter replaces the HTML-probe jobs Prometheus 3 refuses HTML scrape bodies, so the comfyui/services probe jobs were structurally dead targets reading as outages. HTTP liveness now rides a blackbox exporter (shared/blackbox/blackbox.yml, loopback bind, http_2xx module); prometheus.yml probes SearXNG, OpenWebUI and ComfyUI via __param_target relabeling. The unit is a boot-persistent, self-healing root quadlet. --- .../quadlet/blackbox-exporter.container | 35 +++++++++++++++++++ .../quadlet/prometheus.container | 5 +++ shared/blackbox/blackbox.yml | 18 ++++++++++ shared/prometheus/prometheus.yml | 33 +++++++++++------ 4 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 dankclaw/observability/quadlet/blackbox-exporter.container create mode 100644 shared/blackbox/blackbox.yml diff --git a/dankclaw/observability/quadlet/blackbox-exporter.container b/dankclaw/observability/quadlet/blackbox-exporter.container new file mode 100644 index 0000000..636b763 --- /dev/null +++ b/dankclaw/observability/quadlet/blackbox-exporter.container @@ -0,0 +1,35 @@ +# Quadlet unit: blackbox exporter (rootless, boot-persistent). +# +# Install path on dankclaw: +# ~/.config/containers/systemd/blackbox-exporter.container +# +# WHY Network=host. The probes it runs are (a) the host's own LAN IP +# (192.168.0.143:) and (b) container names on dankclaw_default (comfyui). +# A bridge container cannot reach the host's LAN IP at an unpublished port +# (netavark DNATs published ports only — see Container Host Reachability in the +# vault); a host-netns process reaches both. Prometheus (bridge) reaches THIS +# container the same way it reaches the signal relay: the quadlet-published +# host.containers.internal address (169.254.1.2), mapped via HostAddr= on the +# prometheus unit. +# +# The 9115 metrics port publishes on loopback only — it is scraped by the +# prometheus container through the gateway, and never opened on the LAN. + +[Unit] +Description=Blackbox exporter (HTTP probe prober) +After=network-online.target + +[Container] +ContainerName=blackbox-exporter +Exec=/bin/blackbox_exporter --config.file=/etc/blackbox_exporter/config.yml +Volume=/opt/blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro +HealthCmd=/bin/sh -c "wget -q -O /dev/null http://127.0.0.1:9115/-/healthy || exit 1" +HealthInterval=30s +HealthRetries=3 + +[Service] +Restart=always +RestartSec=5 + +[Install] +WantedBy=default.target diff --git a/dankclaw/observability/quadlet/prometheus.container b/dankclaw/observability/quadlet/prometheus.container index 2fb30a7..e6905d9 100644 --- a/dankclaw/observability/quadlet/prometheus.container +++ b/dankclaw/observability/quadlet/prometheus.container @@ -45,6 +45,11 @@ Wants=network-online.target ContainerName=prometheus Image=docker.io/prom/prometheus:latest Network=dankclaw_default +# Name the podman gateway so scrape jobs can reach host-netns processes (the +# blackbox exporter listens on the host's loopback; a bridge container reaches +# it at this address, never at 127.0.0.1). Same mechanism the hermes Caddy +# upstream learned the hard way (Container Host Reachability in the vault). +HostAddr=host.containers.internal=169.254.1.2 User=0:0 Exec=--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=30d --web.enable-lifecycle PublishPort=127.0.0.1:9090:9090 diff --git a/shared/blackbox/blackbox.yml b/shared/blackbox/blackbox.yml new file mode 100644 index 0000000..dbf44e3 --- /dev/null +++ b/shared/blackbox/blackbox.yml @@ -0,0 +1,18 @@ +# Blackbox exporter modules — the honest answer to "is this HTTP service +# answering?". A Prometheus scrape cannot express it (an HTML front page is a +# 200 that fails the scrape protocol), which is why the old `services` and +# `comfyui` jobs sat permanently red on `unsupported Content-Type`. +# See serverAppConfigs-docs/Architecture/Stale Signal Audit.md §1. +# Loopback bind: the image's default 0.0.0.0:9115 would ride the LAN on this +# Network=host container. Prometheus (bridge) still reaches it — its traffic +# arrives on the podman gateway interface, which is not loopback. +network: + listen_address: 127.0.0.1:9115 + +modules: + http_2xx: + prober: http + timeout: 10s + http: + preferred_ip_protocol: ip4 + valid_status_codes: [] # default: 2xx + 3xx diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index 94a7c49..85f05a4 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -60,20 +60,31 @@ scrape_configs: - targets: ["loki:3100"] metrics_path: "/metrics" - # ComfyUI (no native /metrics; probe the UI) - - job_name: "comfyui" - metrics_path: "/" - static_configs: - - targets: ["comfyui:8188"] - - # Blackbox-style probes for service up/down (published LAN ports; - # gitea and comfyui are covered by their own jobs above) + # HTTP liveness, probed honestly through the blackbox exporter. Scraping a + # web root is NOT a health check — an HTML front page fails the scrape + # protocol (`unsupported Content-Type`), which is exactly why the old + # `services` and `comfyui` jobs were red forever beside healthy services + # (Stale Signal Audit §1). The prober is a host-netns container reached at + # the podman gateway name (HostAddr= on the prometheus unit); it in turn + # reaches both the host LAN IP and bridge container names. comfyui's UI probe + # lives here now — its own metrics-scrape job is retired, one owner per + # service. - job_name: "services" - metrics_path: "/" + metrics_path: /probe + params: + module: [http_2xx] static_configs: - targets: - - "192.168.0.143:8081" # SearXNG (via the Gluetun tunnel) - - "192.168.0.143:3005" # OpenWebUI + - "http://192.168.0.143:8081" # SearXNG (via the Gluetun tunnel) + - "http://192.168.0.143:3005" # OpenWebUI + - "http://comfyui:8188" # ComfyUI UI (gfx1151 generator) + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: "host.containers.internal:9115" # LLM fleet. Scraped on the LAN addresses where the endpoint is reachable from # this container, not the guarded vhosts: these are machine clients and