From f073b91033860f2666ff4e068a296e45bc67dfbe Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:32:58 -0700 Subject: [PATCH 1/8] feat(observability): blackbox exporter replaces the HTML-probe jobs Prometheus 3 refuses HTML scrape bodies, so the comfyui/services probe jobs were structurally dead targets reading as outages. HTTP liveness now rides a blackbox exporter (shared/blackbox/blackbox.yml, loopback bind, http_2xx module); prometheus.yml probes SearXNG, OpenWebUI and ComfyUI via __param_target relabeling. The unit is a boot-persistent, self-healing root quadlet. --- .../quadlet/blackbox-exporter.container | 35 +++++++++++++++++++ .../quadlet/prometheus.container | 5 +++ shared/blackbox/blackbox.yml | 18 ++++++++++ shared/prometheus/prometheus.yml | 33 +++++++++++------ 4 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 dankclaw/observability/quadlet/blackbox-exporter.container create mode 100644 shared/blackbox/blackbox.yml diff --git a/dankclaw/observability/quadlet/blackbox-exporter.container b/dankclaw/observability/quadlet/blackbox-exporter.container new file mode 100644 index 0000000..636b763 --- /dev/null +++ b/dankclaw/observability/quadlet/blackbox-exporter.container @@ -0,0 +1,35 @@ +# Quadlet unit: blackbox exporter (rootless, boot-persistent). +# +# Install path on dankclaw: +# ~/.config/containers/systemd/blackbox-exporter.container +# +# WHY Network=host. The probes it runs are (a) the host's own LAN IP +# (192.168.0.143:) and (b) container names on dankclaw_default (comfyui). +# A bridge container cannot reach the host's LAN IP at an unpublished port +# (netavark DNATs published ports only — see Container Host Reachability in the +# vault); a host-netns process reaches both. Prometheus (bridge) reaches THIS +# container the same way it reaches the signal relay: the quadlet-published +# host.containers.internal address (169.254.1.2), mapped via HostAddr= on the +# prometheus unit. +# +# The 9115 metrics port publishes on loopback only — it is scraped by the +# prometheus container through the gateway, and never opened on the LAN. + +[Unit] +Description=Blackbox exporter (HTTP probe prober) +After=network-online.target + +[Container] +ContainerName=blackbox-exporter +Exec=/bin/blackbox_exporter --config.file=/etc/blackbox_exporter/config.yml +Volume=/opt/blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro +HealthCmd=/bin/sh -c "wget -q -O /dev/null http://127.0.0.1:9115/-/healthy || exit 1" +HealthInterval=30s +HealthRetries=3 + +[Service] +Restart=always +RestartSec=5 + +[Install] +WantedBy=default.target diff --git a/dankclaw/observability/quadlet/prometheus.container b/dankclaw/observability/quadlet/prometheus.container index 2fb30a7..e6905d9 100644 --- a/dankclaw/observability/quadlet/prometheus.container +++ b/dankclaw/observability/quadlet/prometheus.container @@ -45,6 +45,11 @@ Wants=network-online.target ContainerName=prometheus Image=docker.io/prom/prometheus:latest Network=dankclaw_default +# Name the podman gateway so scrape jobs can reach host-netns processes (the +# blackbox exporter listens on the host's loopback; a bridge container reaches +# it at this address, never at 127.0.0.1). Same mechanism the hermes Caddy +# upstream learned the hard way (Container Host Reachability in the vault). +HostAddr=host.containers.internal=169.254.1.2 User=0:0 Exec=--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=30d --web.enable-lifecycle PublishPort=127.0.0.1:9090:9090 diff --git a/shared/blackbox/blackbox.yml b/shared/blackbox/blackbox.yml new file mode 100644 index 0000000..dbf44e3 --- /dev/null +++ b/shared/blackbox/blackbox.yml @@ -0,0 +1,18 @@ +# Blackbox exporter modules — the honest answer to "is this HTTP service +# answering?". A Prometheus scrape cannot express it (an HTML front page is a +# 200 that fails the scrape protocol), which is why the old `services` and +# `comfyui` jobs sat permanently red on `unsupported Content-Type`. +# See serverAppConfigs-docs/Architecture/Stale Signal Audit.md §1. +# Loopback bind: the image's default 0.0.0.0:9115 would ride the LAN on this +# Network=host container. Prometheus (bridge) still reaches it — its traffic +# arrives on the podman gateway interface, which is not loopback. +network: + listen_address: 127.0.0.1:9115 + +modules: + http_2xx: + prober: http + timeout: 10s + http: + preferred_ip_protocol: ip4 + valid_status_codes: [] # default: 2xx + 3xx diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index 94a7c49..85f05a4 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -60,20 +60,31 @@ scrape_configs: - targets: ["loki:3100"] metrics_path: "/metrics" - # ComfyUI (no native /metrics; probe the UI) - - job_name: "comfyui" - metrics_path: "/" - static_configs: - - targets: ["comfyui:8188"] - - # Blackbox-style probes for service up/down (published LAN ports; - # gitea and comfyui are covered by their own jobs above) + # HTTP liveness, probed honestly through the blackbox exporter. Scraping a + # web root is NOT a health check — an HTML front page fails the scrape + # protocol (`unsupported Content-Type`), which is exactly why the old + # `services` and `comfyui` jobs were red forever beside healthy services + # (Stale Signal Audit §1). The prober is a host-netns container reached at + # the podman gateway name (HostAddr= on the prometheus unit); it in turn + # reaches both the host LAN IP and bridge container names. comfyui's UI probe + # lives here now — its own metrics-scrape job is retired, one owner per + # service. - job_name: "services" - metrics_path: "/" + metrics_path: /probe + params: + module: [http_2xx] static_configs: - targets: - - "192.168.0.143:8081" # SearXNG (via the Gluetun tunnel) - - "192.168.0.143:3005" # OpenWebUI + - "http://192.168.0.143:8081" # SearXNG (via the Gluetun tunnel) + - "http://192.168.0.143:3005" # OpenWebUI + - "http://comfyui:8188" # ComfyUI UI (gfx1151 generator) + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: "host.containers.internal:9115" # LLM fleet. Scraped on the LAN addresses where the endpoint is reachable from # this container, not the guarded vhosts: these are machine clients and -- 2.54.0 From fbe54ca1ad22d7fb085c507bbb84da2ab6652930 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:32:58 -0700 Subject: [PATCH 2/8] feat(gitea): enable /metrics for Prometheus scraping Gitea shipped metrics disabled, so its scrape target was a permanent structural down beside a healthy service. --- dankclaw/gitea/quadlet/gitea-server.container | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dankclaw/gitea/quadlet/gitea-server.container b/dankclaw/gitea/quadlet/gitea-server.container index 940cc3b..0179ff0 100644 --- a/dankclaw/gitea/quadlet/gitea-server.container +++ b/dankclaw/gitea/quadlet/gitea-server.container @@ -29,6 +29,13 @@ Environment=DATABASE_PASSWD=${GITEA_DB_PASSWORD} Environment=DATABASE_PORT=5432 Environment=USER_UID=1000 Environment=USER_GID=1000 +# Prometheus scrapes /metrics over the published LAN port (job "gitea" in +# shared/prometheus/prometheus.yml). Off, /metrics is a 404 and the target is +# permanently red (the audit's "disabled metrics" class). 0.0.0.0 binds inside +# the container; the only route in is this host's published 3000, and the +# exposed surface is counters only — same posture as the LLM /metrics. +Environment=GITEA__metrics__ENABLED=true +Environment=GITEA__metrics__ENABLED_HOST=0.0.0.0 PublishPort=0.0.0.0:3000:3000 PublishPort=2222:22 Volume=/home/dank/Containers/gitea/data:/data -- 2.54.0 From 2af2e6ac56481372cdf9878a2af0b0dbe492985d Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:32:58 -0700 Subject: [PATCH 3/8] fix(grafana): observability and logs-overview boards read real data observability.json: stale 'structural downs' panel description fixed, target census matches the post-blackbox job set. logs-overview.json: panels grouped by the promoted container label instead of the absent container_name, so the board stops reading empty beside live traffic. --- .../dashboards/logs-overview.json | 242 +++++++++-- .../dashboards/observability.json | 407 +++++++++++++++--- 2 files changed, 553 insertions(+), 96 deletions(-) diff --git a/shared/grafana/provisioning/dashboards/logs-overview.json b/shared/grafana/provisioning/dashboards/logs-overview.json index 2a1fd15..2ca5321 100644 --- a/shared/grafana/provisioning/dashboards/logs-overview.json +++ b/shared/grafana/provisioning/dashboards/logs-overview.json @@ -2,74 +2,242 @@ "uid": "logs-overview", "title": "Logs \u2014 Loki (systemd journal)", "description": "Everything in Loki arrives as the journal (job=systemd-journal); the container is a JSON field parsed at query time, not a stream label. Starter breadth \u2014 the volume/error panels group by the parsed container_name. Loki keeps ~7 days; wide ranges are cheap counts, not the raw log panel.", - "tags": ["logs", "loki"], + "tags": [ + "logs", + "loki" + ], "timezone": "browser", "schemaVersion": 39, "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "templating": { "list": [ { "name": "container", "label": "container", "type": "query", - "datasource": { "type": "loki", "uid": "loki" }, - "query": { "query": "{job=\"systemd-journal\"} | json", "label": "container_name", "refId": "A" }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": { + "query": "label_values({job=\"systemd-journal\"}, container)", + "refId": "LokiVariableQueryEditor-VariableQuery" + }, "includeAll": true, "multi": true, "refresh": 2, - "current": { "text": "All", "value": "$__all" } + "current": { + "text": "All", + "value": "$__all" + } } ] }, "panels": [ { - "id": 1, "type": "timeseries", "title": "Log volume by container (lines/s)", - "description": "Parse the journal JSON, count lines per container. Spikes = a chatty or crashing unit.", - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 }, - "datasource": { "type": "loki", "uid": "loki" }, - "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } } }, "overrides": [] }, - "targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }] + "id": 1, + "type": "timeseries", + "title": "Log volume by container (lines/s)", + "description": "Per-container journal line rate. The `container` label is promoted at ingestion by Alloy (loki.process) from podman's CONTAINER_NAME journald field - Loki metrics can only group by indexed stream labels.", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "fillOpacity": 20, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (container) (count_over_time({job=\"systemd-journal\", container!=\"\"} [$__auto]))", + "legendFormat": "{{container_name}}", + "refId": "A" + } + ] }, { - "id": 2, "type": "timeseries", "title": "Total log rate", + "id": 2, + "type": "timeseries", + "title": "Total log rate", "description": "All journal lines per second.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "datasource": { "type": "loki", "uid": "loki" }, - "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, - "targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))", "legendFormat": "all lines/s", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))", + "legendFormat": "all lines/s", + "refId": "A" + } + ] }, { - "id": 3, "type": "timeseries", "title": "Error-level line rate", + "id": 3, + "type": "timeseries", + "title": "Error-ish line rate (text match)", "description": "Lines matching error/panic/fatal/fail. A blunt text match \u2014 the journal carries no structured level for all units.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, - "datasource": { "type": "loki", "uid": "loki" }, - "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "fixed", "fixedColor": "red" } }, "overrides": [] }, - "targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))", "legendFormat": "error lines/s", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "fixed", + "fixedColor": "red" + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))", + "legendFormat": "error lines/s", + "refId": "A" + } + ] }, { - "id": 4, "type": "timeseries", "title": "Errors by container", + "id": 4, + "type": "timeseries", + "title": "Errors by container", "description": "Which unit is throwing. Rank by the top offenders.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, - "datasource": { "type": "loki", "uid": "loki" }, - "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, - "targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (container) (count_over_time({job=\"systemd-journal\", container!=\"\"} |~ \"(?i)(error|panic|fatal)\" [$__auto]))", + "legendFormat": "{{container_name}}", + "refId": "A" + } + ] }, { - "id": 5, "type": "stat", "title": "Error lines (last 15m)", + "id": 5, + "type": "stat", + "title": "Error lines (last 15m)", "description": "Absolute count over the window, not a rate.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, - "datasource": { "type": "loki", "uid": "loki" }, - "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 500 }] } }, "overrides": [] }, - "targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))", "instant": true, "legendFormat": "errors/15m", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 500 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))", + "instant": true, + "legendFormat": "errors/15m", + "refId": "A" + } + ] }, { - "id": 6, "type": "logs", "title": "Journal (filtered)", + "id": 6, + "type": "logs", + "title": "Journal (filtered)", "description": "Raw lines for the selected containers. Set the container picker; keep the window tight.", - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 }, - "datasource": { "type": "loki", "uid": "loki" }, - "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true }, - "targets": [{ "expr": "{job=\"systemd-journal\"} | json | container_name=~\"$container\"", "refId": "A" }] + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 25 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true + }, + "targets": [ + { + "expr": "{job=\"systemd-journal\", container=~\"$container\"}", + "refId": "A" + } + ] } ] -} +} \ No newline at end of file diff --git a/shared/grafana/provisioning/dashboards/observability.json b/shared/grafana/provisioning/dashboards/observability.json index d3a6fb2..ad3da1d 100644 --- a/shared/grafana/provisioning/dashboards/observability.json +++ b/shared/grafana/provisioning/dashboards/observability.json @@ -2,95 +2,384 @@ "uid": "observability", "title": "Observability \u2014 self-monitoring", "description": "The health of the stack that watches the fleet: Prometheus scrape/TSDB, Loki ingest, Grafana. If this is red the dashboards lie. Note: Alertmanager is running but NOT scraped by Prometheus (no job) \u2014 its metrics are absent by design gap, not fault.", - "tags": ["observability", "meta"], + "tags": [ + "observability", + "meta" + ], "timezone": "browser", "schemaVersion": 39, "refresh": "30s", - "time": { "from": "now-6h", "to": "now" }, + "time": { + "from": "now-6h", + "to": "now" + }, "panels": [ { - "id": 1, "type": "stat", "title": "Targets up", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] }, - "targets": [{ "expr": "count(up == 1)", "instant": true, "refId": "A" }] + "id": 1, + "type": "stat", + "title": "Targets up", + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "count(up == 1)", + "instant": true, + "refId": "A" + } + ] }, { - "id": 2, "type": "stat", "title": "Targets down", - "description": "Known structural downs: comfyui (no metrics endpoint), gitea (metrics off), and the homepage 'services' probes. Real regressions ride up from here too \u2014 check the table before dismissing.", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] } }, "overrides": [] }, - "targets": [{ "expr": "count(up == 0)", "instant": true, "refId": "A" }] + "id": 2, + "type": "stat", + "title": "Targets down", + "description": "Every scrape target that is down. ZERO IS THE HEALTHY STATE: the HTML-probe jobs and the disabled Gitea metrics were retired 2026-10-21 (HTTP liveness now rides the blackbox exporter, job \"services\"). A non-zero count here is a real regression - read /api/v1/targets lastError before triaging.", + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "count(up == 0)", + "instant": true, + "refId": "A" + } + ] }, { - "id": 3, "type": "stat", "title": "TSDB head series", + "id": 3, + "type": "stat", + "title": "TSDB head series", "description": "Active series in memory. A jump = a new exporter or a label-cardinality leak.", - "gridPos": { "h": 4, "w": 5, "x": 8, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, - "targets": [{ "expr": "prometheus_tsdb_head_series", "instant": true, "refId": "A" }] + "gridPos": { + "h": 4, + "w": 5, + "x": 8, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "prometheus_tsdb_head_series", + "instant": true, + "refId": "A" + } + ] }, { - "id": 4, "type": "stat", "title": "Discovered targets", - "gridPos": { "h": 4, "w": 5, "x": 13, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, - "targets": [{ "expr": "sum(prometheus_sd_discovered_targets)", "instant": true, "refId": "A" }] + "id": 4, + "type": "stat", + "title": "Discovered targets", + "gridPos": { + "h": 4, + "w": 5, + "x": 13, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets)", + "instant": true, + "refId": "A" + } + ] }, { - "id": 5, "type": "stat", "title": "Prometheus mem", - "gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, - "targets": [{ "expr": "process_resident_memory_bytes{job=\"prometheus\"}", "instant": true, "refId": "A" }] + "id": 5, + "type": "stat", + "title": "Prometheus mem", + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"prometheus\"}", + "instant": true, + "refId": "A" + } + ] }, { - "id": 6, "type": "stat", "title": "Grafana up", - "gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "mappings": [{ "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }], "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] }, - "targets": [{ "expr": "up{job=\"grafana\"}", "instant": true, "refId": "A" }] + "id": 6, + "type": "stat", + "title": "Grafana up", + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "1": { + "text": "UP", + "color": "green" + }, + "0": { + "text": "DOWN", + "color": "red" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "up{job=\"grafana\"}", + "instant": true, + "refId": "A" + } + ] }, { - "id": 7, "type": "table", "title": "Target health", + "id": 7, + "type": "table", + "title": "Target health", "description": "Every scrape target and its current up. Sort by Value to surface regressions.", - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 4 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "options": { "sortBy": [{ "displayName": "Value", "sortOrder": "Asc" }] }, - "targets": [{ "expr": "up", "format": "table", "instant": true, "refId": "A" }] + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "options": { + "sortBy": [ + { + "displayName": "Value", + "sortOrder": "Asc" + } + ] + }, + "targets": [ + { + "expr": "up", + "format": "table", + "instant": true, + "refId": "A" + } + ] }, { - "id": 8, "type": "timeseries", "title": "Scrape interval drift (p99 actual)", + "id": 8, + "type": "timeseries", + "title": "Scrape interval drift (p99 actual)", "description": "Actual seconds between scrapes (summary quantile). Above the 15s scrape_interval = Prometheus is falling behind its own cadence (slow endpoints or overload).", - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 4 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, - "targets": [{ "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}", "legendFormat": "{{instance}}", "refId": "A" }] + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] }, { - "id": 9, "type": "timeseries", "title": "Scrape samples/s", + "id": 9, + "type": "timeseries", + "title": "Scrape samples/s", "description": "Total samples ingested across all jobs.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, - "targets": [{ "expr": "sum(rate(scrape_samples_scraped[5m]))", "legendFormat": "samples/s", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(rate(scrape_samples_scraped[5m]))", + "legendFormat": "samples/s", + "refId": "A" + } + ] }, { - "id": 10, "type": "timeseries", "title": "Loki request rate", + "id": 10, + "type": "timeseries", + "title": "Loki request rate", "description": "Loki served requests/s (push, query). A drop to zero means ingest or query has stopped.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, - "targets": [{ "expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))", "legendFormat": "{{route}}", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))", + "legendFormat": "{{route}}", + "refId": "A" + } + ] }, { - "id": 11, "type": "timeseries", "title": "Exporter process CPU", + "id": 11, + "type": "timeseries", + "title": "Exporter process CPU", "description": "CPU by monitored process \u2014 spot a runaway exporter or the Prometheus process itself.", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, - "targets": [{ "expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))", "legendFormat": "{{job}}", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))", + "legendFormat": "{{job}}", + "refId": "A" + } + ] } ] -} +} \ No newline at end of file -- 2.54.0 From 999c4fe4405e96093589f9defb333c9a31e99c94 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:33:03 -0700 Subject: [PATCH 4/8] fix(alloy): promote CONTAINER_NAME to an indexed Loki stream label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loki metrics queries group only by indexed stream labels, so the journal-as-JSON stream made every per-container view structurally empty (measured 6.5k lines/5min flowing while the board read zero). A loki.process relabel stage now extracts CONTAINER_NAME into a 'container' label. alloy 1.21's json stage has no on_error attribute (validated against the pinned image) — format_as_json guarantees the line shape, so no error branch exists. --- shared/alloy/config.alloy | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/shared/alloy/config.alloy b/shared/alloy/config.alloy index 8727b58..568d43d 100644 --- a/shared/alloy/config.alloy +++ b/shared/alloy/config.alloy @@ -17,6 +17,9 @@ logging { // Loki is published on the host's loopback (127.0.0.1:3100). Alloy shares the // host network, so it pushes by loopback, never by container name — a root // container is not on the rootless dankclaw_default network. +// GRAFANA_HOST_LABEL is replaced with this host's name by +// copy_alloy_config at deploy time; it must not appear verbatim in the +// installed config. loki.write "default" { endpoint { url = "http://127.0.0.1:3100/loki/api/v1/push" @@ -32,7 +35,34 @@ loki.write "default" { loki.source.journal "journal" { format_as_json = true labels = { - job = "systemd-journal", + job = "systemd-journal", + host = "GRAFANA_HOST_LABEL", + } + forward_to = [loki.process.relabel.receiver] +} + +// Promote the fields that ARE the point of the log stream to real Loki labels. +// Without this the journal arrives as one opaque JSON line per entry and every +// per-container view is structurally impossible: Loki metrics queries can only +// group by INDEXED stream labels, so `sum by (container_name)` returns a single +// anonymous series and the logs-overview board read empty beside 6.5k +// lines/5min of live traffic (measured 2026-10-21). CONTAINER_NAME is what +// podman's journald logging writes (uppercase); host units lack it and stay on +// the host-only stream. `format_as_json = true` above guarantees every line +// reaching this stage is a JSON object, so there is no non-JSON branch to +// protect — alloy 1.21's json stage has no `on_error` attribute (validated +// against the pinned image); a malformed line would drop, which is the +// correct fate for a line Loki could not index anyway. +loki.process "relabel" { + stage.json { + expressions = { + container = "CONTAINER_NAME", + } + } + stage.labels { + values = { + container = "container", + } } forward_to = [loki.write.default.receiver] } -- 2.54.0 From 765a8cb7de1c3c6b5f422469aba1a6eafaf35113 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:33:03 -0700 Subject: [PATCH 5/8] feat(deploy): blackbox copier, alloy engine-validation gate, planner wiring copy_alloy_config now validates the rendered config with the very alloy image that will run it, before installing: a misplaced attribute crash-loops the unit behind a green deploy, and no textual seam can see it. copy_blackbox_config ships the prober config; the planner maps the new unit and shared/blackbox/** to the blackbox-exporter service, ordered before prometheus. --- scripts/deploy-plan.py | 9 ++++++-- scripts/deploy-remote.sh | 45 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index a03231c..009fb72 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -34,7 +34,6 @@ SERVICE_ORDER = ( "hermes", "hermes-dashboard", "hermes-webui", - "prometheus", "grafana", "loki", "alloy", @@ -42,6 +41,9 @@ SERVICE_ORDER = ( "cadvisor", "gpu-exporter", "alertmanager", + # The prober must answer before prometheus re-runs the services job. + "blackbox-exporter", + "prometheus", "authentik-postgres", "authentik-server", "authentik-worker", @@ -97,6 +99,7 @@ DANKCLAW_SERVICES = { "cadvisor", "gpu-exporter", "alertmanager", + "blackbox-exporter", "authentik-postgres", "authentik-server", "authentik-worker", @@ -277,6 +280,8 @@ def services_for_path(path: str, target: str) -> set[str]: # Provisioned datasources and dashboards: git is the source of # truth, so a change here must restart the service that reads them. _add(services, "grafana") + elif group == "blackbox": + _add(services, "blackbox-exporter") elif group == "homepage": # Homepage is a Dankclaw-owned deployment. Shared homepage files # are intentionally not copied during a Dankserver-only run. @@ -380,7 +385,7 @@ def services_for_path(path: str, target: str) -> set[str]: # consumer (prometheus before grafana, loki before alloy). if len(parts) >= 4 and parts[2] == "quadlet": name = parts[3].removesuffix(".container") - if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "cadvisor", "alertmanager"}: + if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "cadvisor", "alertmanager", "blackbox-exporter"}: _add(services, name) return services if len(parts) >= 2 and parts[1] == "gitea": diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index c715ba6..560b765 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -311,8 +311,45 @@ copy_loki_config() { } copy_alloy_config() { - run_privileged mkdir -p /opt/alloy/config - run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy + # The repo config carries the token GRAFANA_HOST_LABEL; the installed copy + # carries the host's name, so Loki streams say WHICH machine's journal a + # line came from without a second config file per host. + # *_CONF_ROOT are test seams: the bats cases execute these copiers for + # real against a sandbox with a stubbed run_privileged. + local root="${ALLOY_CONF_ROOT:-/opt/alloy}" tmp + tmp="$(mktemp)" + sed "s/GRAFANA_HOST_LABEL/$TARGET/g" shared/alloy/config.alloy > "$tmp" + if ! grep -q "host = \"$TARGET\"," "$tmp" || grep -q "GRAFANA_HOST_LABEL" "$tmp"; then + echo "alloy config: host-label substitution failed" >&2 + rm -f "$tmp" + return 1 + fi + # Syntax gate: alloy's config language has no schema — a misplaced + # attribute (e.g. `on_error` outside the json stage) parses only at RUN + # time, and the unit then crash-loops with a healthy-looking deploy. + # Validate against the very image that will run it, before installing. + # Skipped only in the bats sandbox (no image there). + if [ "$root" = "/opt/alloy" ] || podman image exists localhost/franksdank/alloy:latest 2>/dev/null; then + if ! podman run --rm -v "$tmp:/c.alloy:ro" \ + --entrypoint /usr/bin/alloy localhost/franksdank/alloy:latest \ + validate /c.alloy; then + echo "alloy config: validate FAILED against the pinned image" >&2 + rm -f "$tmp" + return 1 + fi + fi + run_privileged mkdir -p "$root/config" + run_privileged cp "$tmp" "$root/config/config.alloy" + rm -f "$tmp" +} + +copy_blackbox_config() { + # Single-file mount target: the unit binds this exact path into + # /etc/blackbox_exporter/config.yml. cp (not cp -r) keeps it one file. + # BLACKBOX_CONF_ROOT is the same test seam. + local root="${BLACKBOX_CONF_ROOT:-/opt/blackbox}" + run_privileged mkdir -p "$root" + run_privileged cp shared/blackbox/blackbox.yml "$root/blackbox.yml" } copy_grafana_provisioning() { @@ -563,6 +600,9 @@ deploy_observability() { ;; cadvisor) ;; + blackbox-exporter) + copy_blackbox_config + ;; *) echo "unknown observability service: $service" >&2 return 2 @@ -577,6 +617,7 @@ deploy_observability() { loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;; node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;; cadvisor) wait_for_health cadvisor http://127.0.0.1:8080/healthz 90 ;; + blackbox-exporter) wait_for_health blackbox-exporter http://127.0.0.1:9115/-/healthy 60 ;; esac } -- 2.54.0 From 078b550bd8e547ed6116a44a7ea3c97b00573b16 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:33:03 -0700 Subject: [PATCH 6/8] test(observability): copier sandboxes, probe-target seam exception, cop gates - caddyfile seam learns the documented exception: URL-shaped targets are legal only inside a prober job (state is per-job; the first cut leaked it and passed a planted URL). - observability seams: blackbox unit + planner entries, both copiers executed for real in sandboxes, alloy engine-validation refusal (image-gated), dashboard panel invariants. - AGENTS.md: structural target downs retired. --- AGENTS.md | 11 ++-- tests/caddyfile.bats | 13 ++++- tests/observability_quadlet.bats | 90 +++++++++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 02873d4..04717a3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,11 +176,12 @@ alertmanager.container`, loopback :9093) → host-side Signal relay (`scripts/signal-relay.py`, user unit, 192.168.0.143:18093 — the host LAN IP, the one bind a container can reach as `host.containers.internal`; iptables admits loopback, the host and the podman bridge subnet only) → signal-cli account 318311. -Prometheus targets read down for structural reasons (HTML-scraped "probes", -Gitea metrics off); the remedy is a blackbox exporter. `frank-zealot` was down for -a different reason until 2026-10-18: llama.cpp will not serve `/metrics` at all -without `--metrics`. See -`serverAppConfigs-docs/Infrastructure/Observability.md`. +The former structural target downs (HTML-scraped "probes", Gitea metrics off) +are retired as of 2026-10-21: HTTP liveness rides a blackbox exporter +(`shared/blackbox/blackbox.yml`), Gitea serves `/metrics` from its quadlet, and +every Prometheus job reports up. `frank-zealot` was down for a different reason +until 2026-10-18: llama.cpp will not serve `/metrics` at all without +`--metrics`. See `serverAppConfigs-docs/Infrastructure/Observability.md`. ## Testing diff --git a/tests/caddyfile.bats b/tests/caddyfile.bats index 28b8f2a..fdc44df 100644 --- a/tests/caddyfile.bats +++ b/tests/caddyfile.bats @@ -193,7 +193,18 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile" } @test "prometheus targets are host:port (no URL schemes)" { - ! grep -nE '^\s*-\s*"?http' "$REPO_ROOT/shared/prometheus/prometheus.yml" + # Plain scrape targets are host:port; a URL target is only meaningful for + # the blackbox prober job, whose relabel_configs copy it to __param_target + # and point __address__ at the prober. Anywhere else a URL target is the + # old HTML-probe defect wearing a new hat. + awk ' + /job_name:/ { job=$0; sub(/.*"/, "", job); sub(/".*/, "", job); probe = 0 } + /metrics_path: \/probe/ { probe = 1 } + /^[[:space:]]*-[[:space:]]*"http/ && !probe { + print FILENAME ":" FNR ": URL target outside a prober job: " $0; bad = 1 + } + END { exit bad } + ' "$REPO_ROOT/shared/prometheus/prometheus.yml" } @test "observability containers run as container root (rootless = host user)" { diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index fb4fe9b..d360cb1 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -21,7 +21,7 @@ load helpers UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager) +UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager blackbox-exporter) # container_section — the [Container] key/value block only container_section() { @@ -33,7 +33,7 @@ service_section() { awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1" } -@test "all seven observability units exist under dankclaw/observability/quadlet/" { +@test "all eight observability units exist under dankclaw/observability/quadlet/" { local svc for svc in "${UNITS[@]}"; do [ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; } @@ -668,3 +668,89 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names), [ -f "$sandbox/opt/grafana/provisioning/dashboards/provider.yml" ] [ -f "$sandbox/opt/grafana/provisioning/datasources/prometheus.yml" ] } + +@test "changes under shared/blackbox/ plan the prober" { + run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \ + --paths shared/blackbox/blackbox.yml + [ "$status" -eq 0 ] + [ "$output" = "blackbox-exporter" ] +} + +@test "the copier runs the same code path the deployer runs" { + # Textually pinned so a refactor cannot quietly route the deployer around + # the copy_* functions the seams execute. + grep -q ' prometheus)' "$REMOTE_DEPLOY" + grep -q 'copy_prometheus_config' "$REMOTE_DEPLOY" + grep -q 'copy_alertmanager_config' "$REMOTE_DEPLOY" + grep -q 'copy_loki_config' "$REMOTE_DEPLOY" + grep -q ' blackbox-exporter)' "$REMOTE_DEPLOY" + grep -q 'copy_blackbox_config' "$REMOTE_DEPLOY" +} + +@test "the alloy copier installs the host-labeled config, token-free" { + # Executes copy_alloy_config for real against a sandbox (stubbed + # run_privileged). A surviving GRAFANA_HOST_LABEL token would ship a + # config whose host label is a literal placeholder — the exact class of + # silent-label defect this tranche retired the board for. + local sandbox="$BATS_TEST_TMPDIR/opt" + ( + cd "$REPO_ROOT" + eval "$(sed -n '/^copy_alloy_config()/,/^}/p' scripts/deploy-remote.sh)" + run_privileged() { "$@"; } + # NOTE: VAR=x func does not bind for shell functions — export. + export TARGET=dankclaw ALLOY_CONF_ROOT="$sandbox" + copy_alloy_config + ) + [ -f "$sandbox/config/config.alloy" ] + grep -q 'host = "dankclaw",' "$sandbox/config/config.alloy" + ! grep -q "GRAFANA_HOST_LABEL" "$sandbox/config/config.alloy" +} + +@test "the blackbox copier installs the prober config" { + local sandbox="$BATS_TEST_TMPDIR/blackbox" + ( + cd "$REPO_ROOT" + eval "$(sed -n '/^copy_blackbox_config()/,/^}/p' scripts/deploy-remote.sh)" + run_privileged() { "$@"; } + export BLACKBOX_CONF_ROOT="$sandbox" + copy_blackbox_config + ) + [ -f "$sandbox/blackbox.yml" ] + grep -q "listen_address: 127.0.0.1:9115" "$sandbox/blackbox.yml" +} + +@test "the alloy copier refuses to install a config the engine rejects" { + # The syntax gate: alloy attributes are position-sensitive and a + # misplaced one only explodes at run time (crash-loop behind a green + # deploy). Prove the copier refuses a config the pinned image won't + # validate, and installs nothing. Needs the alloy image; skipped + # where it is absent (the copier's gate is likewise image-gated). + if ! podman image exists localhost/franksdank/alloy:latest; then + skip "franksdank/alloy image not present on this box" + fi + local sandbox="$BATS_TEST_TMPDIR/opt" cfg="$BATS_TEST_TMPDIR/broken.alloy" + sed 's/GRAFANA_HOST_LABEL/dankclaw/g' shared/alloy/config.alloy > "$cfg" + printf 'bogus_attr = "x"\n' >> "$cfg" + ! ( + cd "$REPO_ROOT" + eval "$(sed -n '/^copy_alloy_config()/,/^}/p' scripts/deploy-remote.sh)" + run_privileged() { "$@"; } + cp() { if [ "$1" = "$cfg" ]; then return 1; fi; command cp "$@"; } + export TARGET=dankclaw ALLOY_CONF_ROOT="$sandbox" + copy_alloy_config + ) + [ ! -f "$sandbox/config/config.alloy" ] +} + +@test "the installed alloy config is engine-valid" { + # Runs the pinned image's own validator over the shipped config. The + # 2026-10-21 defect (`on_error` on the labels stage, which alloy 1.21 + # does not define) passed every textual seam and only failed at boot. + if ! podman image exists localhost/franksdank/alloy:latest; then + skip "franksdank/alloy image not present on this box" + fi + local cfg="$BATS_TEST_TMPDIR/rendered.alloy" + sed 's/GRAFANA_HOST_LABEL/dankclaw/g' shared/alloy/config.alloy > "$cfg" + podman run --rm -v "$cfg:/c.alloy:ro" --entrypoint /usr/bin/alloy \ + localhost/franksdank/alloy:latest validate /c.alloy +} -- 2.54.0 From cb86ad8fd7a8acf6222977541c64fe3cc8403bfe Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:33:03 -0700 Subject: [PATCH 7/8] docs(observability): post-blackbox state, copier law, label-promotion why Prometheus/Observability/Provisioning pages: every job up after the blackbox swap, the copier list learns copy_blackbox_config and the alloy validate gate, the config table gains alloy + blackbox rows, and the logs bullet states the stream-label law instead of the retired 'awaiting deploy' note. --- .../Observability Provisioning.md | 13 ++++--- .../Infrastructure/Observability.md | 37 ++++++++++--------- .../Infrastructure/Prometheus.md | 30 +++++++-------- 3 files changed, 41 insertions(+), 39 deletions(-) diff --git a/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md b/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md index 2771bbf..a59771a 100644 --- a/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md +++ b/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md @@ -20,6 +20,8 @@ committed. | `shared/prometheus/prometheus.yml` | scrape jobs (global + per-host) | | `shared/prometheus/rules/llm-fleet.yml` | alert rules + recording rules | | `shared/grafana/provisioning/` | datasources + dashboards (provisioned tree) | +| `shared/alloy/config.alloy` | the log pipeline: journal source → label promotion → Loki push | +| `shared/blackbox/blackbox.yml` | prober modules (`http_2xx`) + the loopback bind | Targets and datasources reference **container names** (`grafana:3000` `loki:3100` — `shared/prometheus/prometheus.yml:52-61`), never @@ -31,8 +33,8 @@ scraped as `host.containers.internal:` because a bridge container cannot reach a host-netns bind at the host's LAN IP (`prometheus.yml:89-99`), and only the endpoint on another machine (`192.168.0.235:7770`) is addressed by LAN IP (`prometheus.yml:128-131`). Prometheus 3 refuses HTML responses as metrics — -HTML-scraped "probes" are structurally inert and need a blackbox exporter (see -[[Stale Signal Audit]]). +HTTP liveness rides the **blackbox exporter** (`shared/blackbox/blackbox.yml`, +deployed 2026-10-21; see [[Stale Signal Audit]]). ## The deploy path @@ -43,7 +45,8 @@ from `deploy_observability`: - `copy_alertmanager_config` → `/opt/alertmanager/config/alertmanager.yml` (`:232-235`) - `install_signal_relay` → the user unit + iptables + the health wait on `192.168.0.143:18093/health` (`:243-274`) - `copy_loki_config` → `/opt/loki/config/loki-config.yaml` (`:276-279`) -- `copy_alloy_config` → `/opt/alloy/config/config.alloy` (`:281-284`) +- `copy_alloy_config` → `/opt/alloy/config/config.alloy` — substitutes the `GRAFANA_HOST_LABEL` token with the deploy target's name and **refuses to install** if the token survives; `ALLOY_CONF_ROOT` is the bats sandbox seam +- `copy_blackbox_config` → `/opt/blackbox/blackbox.yml` (single-file mount of the prober); `BLACKBOX_CONF_ROOT` is its sandbox seam - `copy_grafana_provisioning` → `/opt/grafana/provisioning/` — datasources, provider, and **every** `dashboards/*.json` (a glob, not a hand-listed subset; `GRAFANA_PROV_ROOT` is the test seam the bats case uses to run the copier into a sandbox) `scripts/deploy-plan.py` maps `shared/prometheus/**` to `prometheus` (plus @@ -109,8 +112,8 @@ curl -s "http://127.0.0.1:9090/api/v1/targets?state=active" | \ The two Prometheus queries were run on 2026-10-18 and are what every number on this page comes from: 4 alerting + 2 recording rules, and `up` for `prometheus`, `node-dankclaw`, `node-dankserver`, `grafana`, `loki` -`frank-zealot`, `frank-dragoon`, `frank-archon` with the four structural downs -listed in [[Prometheus]]. The Grafana proof is the browser session above, run +`frank-zealot`, `frank-dragoon`, `frank-archon`. The four structural downs +that census recorded are retired (2026-10-21, [[Prometheus]]). The Grafana proof is the browser session above, run 2026-10-21: it found a provisioned-file gap the filesystem listing hid. The filesystem side (`ls /opt/grafana/provisioning/*`) is proof of installation only — see *Add a dashboard* for why a repo JSON is not a live board. diff --git a/serverAppConfigs-docs/Infrastructure/Observability.md b/serverAppConfigs-docs/Infrastructure/Observability.md index 615404f..0660283 100644 --- a/serverAppConfigs-docs/Infrastructure/Observability.md +++ b/serverAppConfigs-docs/Infrastructure/Observability.md @@ -95,13 +95,14 @@ health endpoint (`/prometheus`: `/-/healthy`, `/alertmanager`: `/-/healthy` [[Observability Provisioning]] for what is provisioned as code and how to verify it through the APIs. -## Current state (2026-10-20) +## Current state (2026-10-21) * **Metrics: working.** `curl -s http://127.0.0.1:9090/api/v1/targets` reports - eight up targets — the three LLM endpoints (frank-archon on `vllm:*`, - frank-dragoon and frank-zealot on `llamacpp:*`), grafana, loki, prometheus - and both node exporters. The four downs are structural (the `comfyui` and - `services` HTML probes, and Gitea's disabled metrics endpoint), not outages. + every job up — the three LLM endpoints (frank-archon on `vllm:*`, + frank-dragoon and frank-zealot on `llamacpp:*`), grafana, loki, prometheus, + both node exporters, cadvisor, gitea (metrics enabled in its quadlet) and + the blackbox-probed `services` (SearXNG, OpenWebUI, ComfyUI). The structural + downs are retired — see [[Prometheus]] → Targets. Two rule groups in `shared/prometheus/rules/llm-fleet.yml`: `llm-fleet` (2 recording + 4 alert, `llamacpp:*`) and `vllm-fleet` (1 recording + 4 alert, `vllm:*` — archon queueing, prefix collapse, preemption, p95 @@ -142,26 +143,26 @@ verify it through the APIs. `power1_average`, so they carry no `gpu_power_watts` line at all (only NVIDIA and the Dankclaw 8060S iGPU report power); and it binds loopback + the host LAN, like node-exporter, never the wildcard. The dashboard graphs util/VRAM/VRAM-%/temp/power. -* **Logs: fixed in code, awaiting deploy.** The pipeline shipped nothing because - a rootless Alloy cannot read the `0640` journal files, and the container-log - globs point at a root-storage layout that does not exist here. The fix is built - and tested in-repo (2026-10-20): Alloy is now a **root system service on - `Network=host`**, reading `system.journal` + `user-*.journal` and pushing to - Loki's `127.0.0.1:3100` loopback publish — a throwaway root probe read 31,825 - lines and pushed 26,966 with zero drops. The **live** instance is still the old - rootless unit until `deploy.sh dankclaw --service alloy` runs. Detail in - [[Alloy]] → Why root; Loki's store state in [[Loki]] → Contents. -* **Not configured:** no blackbox exporter (the structural downs have no real - uptime signal) and no Alertmanager vhost. The log pipeline's fix is in code but - not yet deployed (see *Logs* above). +* **Logs: flowing; container labels land with the 2026-10-21 deploy.** The + pipeline once shipped nothing (rootless Alloy cannot read the `0640` journal; + the container-log globs pointed at a root-storage layout that does not exist + here) — fixed 2026-10-20 by making Alloy a **root system service on + `Network=host`**. Journal lines now reach Loki (measured ~6.5k lines/5min), + but the stream carried only `job`/`service_name`, so every per-container + view was structurally empty: Loki metrics queries group only by INDEXED + stream labels. Alloy now promotes podman's `CONTAINER_NAME` to a `container` + label (plus a `host` label) at ingestion — `loki.process` in + `shared/alloy/config.alloy`; detail in [[Alloy]] → Why root, and the + purpose-test law in [[Observability Provisioning]]. ## Citations -- `dankclaw/observability/quadlet/{prometheus,alertmanager,grafana,loki,alloy,node-exporter,cadvisor}.container` +- `dankclaw/observability/quadlet/{prometheus,alertmanager,grafana,loki,alloy,node-exporter,cadvisor,blackbox-exporter}.container` - `scripts/deploy-remote.sh:219-300` (copiers + relay), `:521-536` (health waits), `:477-486` (grafana env seed) - `scripts/signal-relay.py:26-49`, `scripts/signal-relay.service` - `scripts/gpu-exporter.py`, `scripts/gpu-exporter.service` (per-GPU host exporter) - `shared/prometheus/prometheus.yml`, `shared/prometheus/alertmanager.yml`, `shared/prometheus/rules/llm-fleet.yml` - `shared/grafana/provisioning/`, `shared/loki/loki-config.yaml`, `shared/alloy/config.alloy` +- `shared/blackbox/blackbox.yml` (prober modules + loopback bind) - `tests/observability_quadlet.bats` (boot persistence, no wildcard publish, probe addresses, delivery contract) - Host evidence 2026-10-18: `ss -ltnp`, `podman ps`, `systemctl --user is-enabled|is-active`, `curl` on `127.0.0.1:9090/api/v1/{targets,rules,alerts}`, `127.0.0.1:3100/loki/api/v1/labels`, `127.0.0.1:12345/metrics`, `127.0.0.1:3002/api/health`, `192.168.0.143:18093/health`, `journalctl --user -u signal-relay` diff --git a/serverAppConfigs-docs/Infrastructure/Prometheus.md b/serverAppConfigs-docs/Infrastructure/Prometheus.md index 9b51a7a..fffed02 100644 --- a/serverAppConfigs-docs/Infrastructure/Prometheus.md +++ b/serverAppConfigs-docs/Infrastructure/Prometheus.md @@ -42,29 +42,27 @@ Authentik; post-auth the Caddy vhost routes straight to `prometheus:9090`) — The `frank-medium` job is **retired**: no `frank-medium` job exists in the config or in `/api/v1/targets` (2026-10-18), and the endpoint itself left the fleet — its history is on [[frank-archon]], which holds the port today. -**Three jobs read down today, and they are not outages** (re-audited against `curl -s http://127.0.0.1:9090/api/v1/targets`, 2026-10-18; first audited 2026-09-16): +**No job reads down structurally any more** (the three permanent-reds were +retired 2026-10-21; census re-run after each deploy): -| Job | Target | `health` | `lastError`, verbatim | -|-----|--------|----------|------------------------| -| `gitea` | `http://192.168.0.143:3000/metrics` | down | `server returned HTTP status 404 Not Found` | -| `comfyui` | `http://comfyui:8188/` | down | `received unsupported Content-Type "text/html" and no fallback_scrape_protocol specified for target` | -| `services` | `http://192.168.0.143:8081/` | down | `received unsupported Content-Type "text/html; charset=utf-8" and no fallback_scrape_protocol specified for target` | -| `services` | `http://192.168.0.143:3005/` | down | same `unsupported Content-Type "text/html; charset=utf-8"` error | +| Job | Target | how | +|---|---|---| +| `gitea` | `192.168.0.143:3000/metrics` | metrics enabled in the quadlet (`GITEA__metrics__ENABLED=true`, `ENABLED_HOST=0.0.0.0`) — the page once said "stays off until `[metrics] ENABLED = true` in its own config"; the env form is what the deployer ships | +| `services` | `http://192.168.0.143:8081` (SearXNG), `:3005` (OpenWebUI), `http://comfyui:8188` | **blackbox exporter** probe, module `http_2xx` — the comfyui metrics job is retired, one owner per service | -Gitea's exporter stays off until `[metrics] ENABLED = true` in its own config (no metrics env is set in the container, and `curl -s http://127.0.0.1:3000/metrics` → 404, 2026-10-18); ComfyUI and the two `services` probes are HTML pages, which Prometheus 3 refuses as metrics. Everything else is green: `prometheus`, `node-dankclaw`, `node-dankserver`, `grafana`, `loki`, `frank-zealot`, `frank-dragoon`, `frank-archon`. +Scraping a web root is not a health check: an HTML page fails the scrape +protocol (`unsupported Content-Type`), which is what kept those targets red +beside healthy services. The classification rule and the honesty law live in +[[Stale Signal Audit]]. The prober is `blackbox-exporter.container` (host +netns, config `shared/blackbox/blackbox.yml`, bound 127.0.0.1:9115; Prometheus +reaches it at the `host.containers.internal` gateway name). Plain targets stay +`host:port`, never URLs — the probe job's URL targets are the documented +exception, relabeled onto the prober. **`frank-zealot` reads up since 2026-10-18, and did not before.** llama.cpp only answers `/metrics` when started with `--metrics`, so `up{job="frank-zealot"}` was 0 across the whole retained history (the oldest sample, 2026-10-11, is already 0) while the service answered `/health` 200 the whole time; the flag and its opt-out are on [[frank-zealot]]. Read `lastError` before believing an outage: a target can read down beside a healthy service. Alert state is a live property, not a page fact: `curl -s http://127.0.0.1:9090/api/v1/alerts` returned no alerts at the 2026-10-18 morning census, one firing `LlmPrefixCacheCollapse` (info, `frank-zealot`, `activeAt` 11:10:48Z) minutes later, and none again by 11:56Z. -The remedy for a probe-shaped question is the **blackbox exporter**; the -classification rule and the honesty law for it live in [[Stale Signal Audit]]. -Until the exporter is deployed, these three stay red for a structural reason: -deleting them would -hide a real service, and fixing them by publishing databases or enabling metrics -would trade a wrong signal for a real exposure. Targets are `host:port`, never -URLs. - ## Volumes | Container path | Host path | -- 2.54.0 From e6fcf217b44e4052a7b372434ddde43d7ff2a2fc Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:35:50 -0700 Subject: [PATCH 8/8] docs(comfyui): the GTT aperture is not free RAM beside frank-dragoon Measured 2026-10-21 with frank-dragoon live: MemAvailable 15.3 GiB, iGPU busy 1.75 GiB. The Qwen-Image weight sets (13.3-14.2 GB) do not fit that budget, and the page's 122 GB /system_stats figure is the aperture, not headroom. Adds the fit table and the stop-mask-render-verify procedure (the 3090 bench law, applied to the APU); ComfyUI.md gotcha links it. --- .../Dankclaw/ComfyUI Qwen-Image Workflows.md | 36 ++++++++++++++++--- serverAppConfigs-docs/Dankclaw/ComfyUI.md | 6 ++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/serverAppConfigs-docs/Dankclaw/ComfyUI Qwen-Image Workflows.md b/serverAppConfigs-docs/Dankclaw/ComfyUI Qwen-Image Workflows.md index e36aebe..2924a7b 100644 --- a/serverAppConfigs-docs/Dankclaw/ComfyUI Qwen-Image Workflows.md +++ b/serverAppConfigs-docs/Dankclaw/ComfyUI Qwen-Image Workflows.md @@ -110,9 +110,9 @@ only `models`, `input`, `output`, `custom_nodes` and the healthcheck file ## Recommended additions -Quantized preferred, sized for the 8060S iGPU (GTT-backed — `/system_stats` -reports 122.4 GB of device VRAM on `cuda:0`, `curl http://127.0.0.1:8188/system_stats` -2026-10-18; the RTX 3090 is frank-zealot's). All repos verified 2026-10-05 and +Quantized preferred. The iGPU's GTT aperture is not free RAM beside +frank-dragoon — size every pick against *What actually fits beside +frank-dragoon* below (the RTX 3090 is frank-zealot's). All repos verified 2026-10-05 and all four still resolve today (`huggingface.co/lightx2v/Qwen-Image-Lightning` `huggingface.co/unsloth/Qwen-Image-Edit-2511-GGUF` `huggingface.co/Comfy-Org/Qwen-Image-DiffSynth-ControlNets` @@ -133,8 +133,34 @@ sha256 checks) or a plain `hf download` with the token in the environment — never on the command line. Skip list: the original `city96/Qwen-Image-gguf` conversions (superseded by -the 2.1 files already on disk), and any BF16 of #2/#4 (the iGPU does not need -them at GTT capacity, and they double the disk bill). +the 2.1 files already on disk), and any BF16 of #2/#4 — they double the disk +bill, and see *What actually fits beside frank-dragoon* for why raw GTT size +is not the real budget. + +## What actually fits beside frank-dragoon + +The 122 GB `/system_stats` figure is the GTT **aperture**, not free memory: on +an APU the aperture and `MemAvailable` are the same physical RAM, and +frank-dragoon pins tens of GiB of it. Measured 2026-10-21 with frank-dragoon +live: `MemAvailable` 15.3 GiB, iGPU busy 1.75 GiB — ComfyUI itself idles at +~10 MB, so the budget for a generation run is roughly what `free` says. + +Weights must land inside that budget with slack for compute buffers and +activations (the mmproj-class lesson: price a model at ~1.5× its file set): + +| Pipeline (weights) | Fits beside frank-dragoon? | +|---|---| +| Qwen-Image 2.1 int8_convrot + w4a8 + bf16 VAE (~14.2 GB) — the two proven workflows | **No** — thrash-or-fail; do not queue it while the endpoint serves | +| GGUF Q8_0 + w4a8 + small VAE (~13.3 GB) | No, same reason | +| Any BF16 combination | No | +| SDXL-Turbo / SD 1.5-class (~2–7 GB) | Yes, with real headroom | + +**The law: one process owns the memory — same rule as the 3090 bench law.** +For a Qwen-Image render: stop and mask frank-dragoon +(`sudo systemctl stop frank-dragoon` + `mask` — the mask stops +`Restart=always` from firing mid-render), generate, unmask, verify +`curl 192.168.0.143:7770/health`. The proven workflows above then run with +the whole machine. Do not treat `/system_stats`'s 122 GB as an invitation. ## See Also diff --git a/serverAppConfigs-docs/Dankclaw/ComfyUI.md b/serverAppConfigs-docs/Dankclaw/ComfyUI.md index b2dfd25..e946d86 100644 --- a/serverAppConfigs-docs/Dankclaw/ComfyUI.md +++ b/serverAppConfigs-docs/Dankclaw/ComfyUI.md @@ -129,6 +129,12 @@ bind-mounted to `/usr/local/bin/comfyui-healthcheck.py` and run as `HealthCmd` - Do not let pip reinstall the torch family. The Containerfile filters it out on purpose. - The base image is about 32 GiB (`podman image inspect localhost/comfyui-rocm:gfx1151` → 33,765,227,471 B ≈ 31.4 GiB, 2026-10-18). A code update means a rebuild, because the code is baked in. - The empty-models note from the 2026-09-16 era is retired: `/home/dank/comfyui-data/models` now holds the Qwen-Image 2.1 payload — three `diffusion_models`, three `text_encoders`, two VAEs (~35 GB, `ls -l` 2026-10-18), listed in [[ComfyUI Qwen-Image Workflows]]. New model classes (LoRA, ControlNet) still need their directories created first: `models/lora` and `models/controlnet` do not exist yet. +- **The 122 GB `/system_stats` VRAM figure is the GTT aperture, not free + memory.** On the APU it is system RAM, and frank-dragoon pins most of it + (`MemAvailable` 15.3 GiB with the endpoint live, measured 2026-10-21). A + Qwen-Image-class render beside the running endpoint is thrash-or-fail; the + stop-mask-render-verify procedure and the fit table are in + [[ComfyUI Qwen-Image Workflows]] → What actually fits beside frank-dragoon. ## See Also -- 2.54.0