From c96d0228a4c813b2612d9a136a7775149f3a3e1b Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 02:45:57 -0700 Subject: [PATCH] =?UTF-8?q?fix(grafana):=20dashboard=20census=20fixes=20?= =?UTF-8?q?=E2=80=94=20every=20panel=20now=20receives=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Census: all 5 provisioned dashboards, 56 targets, every expression re-evaluated against the live Prometheus/Loki with template vars expanded (queries must run from the host: both APIs bind 127.0.0.1; from the dev box that is connection-refused, which is NOT a dashboard defect). - host-system 'Load 5m / core' was STRUCTURALLY EMPTY: bare '/' drops non-matching pairs and the right side collapses to {job} while the left carries {job,instance} -> 0 series forever. Fixed with / on(job); proven live (2 series). - containers 'Memory vs limit (%)' was STRUCTURALLY EMPTY: every container_spec_memory_limit_bytes on this fleet is 0 (no cgroup limits), so the '> 0' filter can never match. Rewritten to share-of-host-total via label_replace join of the node exporter's MemTotal by host (the parens form is a PromQL parse error: nothing may follow group_left); 62 series live. Panel unit percentunit kept -> expression returns a fraction. - llm-fleet 'Firing fleet alerts' is legitimately empty when healthy (ALERTS carries only firing/pending series); description now says so, per the Stale Signal rule that 0-series-on-ALERTS is not a defect. logs-overview and observability: all targets already returning data. --- shared/grafana/provisioning/dashboards/containers.json | 6 +++--- shared/grafana/provisioning/dashboards/host-system.json | 2 +- shared/grafana/provisioning/dashboards/llm-fleet.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/shared/grafana/provisioning/dashboards/containers.json b/shared/grafana/provisioning/dashboards/containers.json index c324b97..6bb13c0 100644 --- a/shared/grafana/provisioning/dashboards/containers.json +++ b/shared/grafana/provisioning/dashboards/containers.json @@ -104,14 +104,14 @@ { "id": 6, "type": "timeseries", - "title": "Memory vs limit (%)", - "description": "Working set over the cgroup memory limit (container_spec_memory_limit_bytes). Only containers that carry a limit are shown; approaching 1.0 means an OOM is near. Containers with no limit are omitted.", + "title": "Memory vs host total (%)", + "description": "Working set over the host's total RAM. The fleet sets NO cgroup memory limits (every container_spec_memory_limit_bytes is 0 — a limit-share panel is structurally empty here), so the honest pressure view is share of the host. The join labels the node exporter's MemTotal onto the cadvisor series by host.", "gridPos": { "h": 9, "w": 12, "x": 12, "y": 11 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] }, "targets": [ { - "expr": "(container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} / on (job, host, container) group_left container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}) and (container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} > 0)", + "expr": "container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\", container!=\"\"} / on(host) group_left label_replace(node_memory_MemTotal_bytes{job=~\"node-.*\"}, \"host\", \"$1\", \"job\", \"node-(.*)\")", "legendFormat": "{{host}} / {{container}}", "refId": "A" } diff --git a/shared/grafana/provisioning/dashboards/host-system.json b/shared/grafana/provisioning/dashboards/host-system.json index 487a712..c9b14fc 100644 --- a/shared/grafana/provisioning/dashboards/host-system.json +++ b/shared/grafana/provisioning/dashboards/host-system.json @@ -54,7 +54,7 @@ "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 1 }] } }, "overrides": [] }, - "targets": [{ "expr": "node_load5{job=~\"$node\"} / count by (job) (node_cpu_seconds_total{job=~\"$node\",mode=\"idle\"})", "legendFormat": "{{job}}", "refId": "A" }] + "targets": [{ "expr": "node_load5{job=~\"$node\"} / on(job) count by (job) (node_cpu_seconds_total{job=~\"$node\",mode=\"idle\"})", "legendFormat": "{{job}}", "refId": "A" }] }, { "id": 4, "type": "stat", "title": "Open file handles", diff --git a/shared/grafana/provisioning/dashboards/llm-fleet.json b/shared/grafana/provisioning/dashboards/llm-fleet.json index c2bf94c..fff5af4 100644 --- a/shared/grafana/provisioning/dashboards/llm-fleet.json +++ b/shared/grafana/provisioning/dashboards/llm-fleet.json @@ -194,7 +194,7 @@ "id": 6, "type": "timeseries", "title": "Firing fleet alerts", - "description": "The LLM-fleet rules from shared/prometheus/rules/llm-fleet.yml. A firing alert is delivered to Signal: Prometheus -> Alertmanager (dankclaw quadlet) -> the host-side signal-relay user unit -> signal-cli, since 2026-09-17.", + "description": "The LLM-fleet rules from shared/prometheus/rules/llm-fleet.yml. EMPTY IS THE HEALTHY STATE: zero series means nothing is firing (ALERTS only carries active/pending alert series — do not read this panel as broken). A firing alert is delivered to Signal: Prometheus -> Alertmanager (dankclaw quadlet) -> the host-side signal-relay user unit -> signal-cli, since 2026-09-17.", "gridPos": { "h": 8, "w": 12,