From 1006e96312d1e39028a03acba1882b194ef2a737 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 20 Oct 2026 21:10:06 -0700 Subject: [PATCH] feat(grafana): starter dashboards for host/system, Loki logs, and observability self-monitoring llm-fleet.json already owns LLM capacity+spec-decode. These three cover the domains that had NO dashboard: - host-system: node_exporter CPU/Load/Mem/Disk/Net/Uptime/FS/FDs for both Dankclaw and Dankserver (templated by job node-dankclaw|node-dankserver) - logs-overview: Loki volume + per-container error rate + a sample log panel; container is parsed from the JSON line at query time (journal streams have no container_name stream label) - observability: the stack's own health - Prometheus target up/down + scrape interval drift + samples/s + TSDB, Loki ingest/discard, Grafana CPU/mem. Panel queries verified live against Prometheus/Loki; provisioned dashboards confirmed via /api/search. Alertmanager is running but NOT scraped by Prometheus - noted in the dashboard, no empty panels built on it. --- .../provisioning/dashboards/host-system.json | 144 ++++++++++++++++++ .../dashboards/logs-overview.json | 75 +++++++++ .../dashboards/observability.json | 96 ++++++++++++ 3 files changed, 315 insertions(+) create mode 100644 shared/grafana/provisioning/dashboards/host-system.json create mode 100644 shared/grafana/provisioning/dashboards/logs-overview.json create mode 100644 shared/grafana/provisioning/dashboards/observability.json diff --git a/shared/grafana/provisioning/dashboards/host-system.json b/shared/grafana/provisioning/dashboards/host-system.json new file mode 100644 index 0000000..2c7fb19 --- /dev/null +++ b/shared/grafana/provisioning/dashboards/host-system.json @@ -0,0 +1,144 @@ +{ + "uid": "host-system", + "title": "Hosts \u2014 Dankclaw & Dankserver", + "description": "node_exporter for both servers. Starter set \u2014 throw-at-the-wall breadth; refine the windows and drop the noise. The picker selects the host; CPU/load/memory/filesystem/network are per-node.", + "tags": ["host", "system"], + "timezone": "browser", + "schemaVersion": 39, + "refresh": "30s", + "time": { "from": "now-6h", "to": "now" }, + "templating": { + "list": [ + { + "name": "node", + "label": "host", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": { "query": "label_values(up{job=~\"node-.*\"}, job)", "refId": "A" }, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 1, "type": "stat", "title": "Uptime", + "description": "Time since boot. A drop means a reboot.", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, + "targets": [{ "expr": "time() - node_boot_time_seconds{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }] + }, + { + "id": 2, "type": "stat", "title": "Memory used", + "description": "1 - MemAvailable/MemTotal. The honest \"in use\" figure (excludes reclaimable cache).", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.8 }, { "color": "red", "value": 0.92 }] } }, "overrides": [] }, + "targets": [{ "expr": "1 - node_memory_MemAvailable_bytes{job=~\"$node\"} / node_memory_MemTotal_bytes{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }] + }, + { + "id": 3, "type": "stat", "title": "Load 5m / core", + "description": "Per-core load5. >1 sustained means runnable tasks are queuing behind cores.", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 1 }] } }, "overrides": [] }, + "targets": [{ "expr": "node_load5{job=~\"$node\"} / count by (job) (node_cpu_seconds_total{job=~\"$node\",mode=\"idle\"})", "legendFormat": "{{job}}", "refId": "A" }] + }, + { + "id": 4, "type": "stat", "title": "Open file handles", + "description": "FDs allocated vs the kernel maximum. Rising toward the max precedes EMFILE across the host.", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percentunit", "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 }] } }, "overrides": [] }, + "targets": [{ "expr": "node_filefd_allocated{job=~\"$node\"} / node_filefd_maximum{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }] + }, + { + "id": 5, "type": "timeseries", "title": "CPU busy %", + "description": "CPU seconds/s in each mode = cores busy per mode; the stack sums to total cores in use (16/host here). A user or system spike points at what is loading the box; steal > 0 means the hypervisor is throttling.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] }, + "targets": [ + { "expr": "sum by (job,mode) (rate(node_cpu_seconds_total{job=~\"$node\",mode!=\"idle\",mode!=\"iowait\"}[5m]))", "legendFormat": "{{job}} {{mode}}", "refId": "A" } + ] + }, + { + "id": 6, "type": "timeseries", "title": "Load average", + "description": "load1/5/15 per host.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "targets": [ + { "expr": "node_load1{job=~\"$node\"}", "legendFormat": "{{job}} load1", "refId": "A" }, + { "expr": "node_load5{job=~\"$node\"}", "legendFormat": "{{job}} load5", "refId": "B" }, + { "expr": "node_load15{job=~\"$node\"}", "legendFormat": "{{job}} load15", "refId": "C" } + ] + }, + { + "id": 7, "type": "timeseries", "title": "Memory breakdown", + "description": "Total, free, buffers+cache. Reclaimable cache is not pressure; MemAvailable is the real headroom.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, + "targets": [ + { "expr": "node_memory_MemTotal_bytes{job=~\"$node\"} - node_memory_MemAvailable_bytes{job=~\"$node\"}", "legendFormat": "{{job}} used", "refId": "A" }, + { "expr": "node_memory_MemAvailable_bytes{job=~\"$node\"}", "legendFormat": "{{job}} available", "refId": "B" } + ] + }, + { + "id": 8, "type": "timeseries", "title": "Filesystem free %", + "description": "Per mountpoint. The media/library stacks fill disks.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.15 }] } }, "overrides": [] }, + "targets": [ + { "expr": "node_filesystem_avail_bytes{job=~\"$node\",fstype!~\"tmpfs|overlay|squashfs\"} / node_filesystem_size_bytes{job=~\"$node\",fstype!~\"tmpfs|overlay|squashfs\"}", "legendFormat": "{{job}} {{mountpoint}}", "refId": "A" } + ] + }, + { + "id": 9, "type": "timeseries", "title": "Disk throughput", + "description": "Read/write bytes/s per device.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, + "targets": [ + { "expr": "sum by (job,device) (rate(node_disk_read_bytes_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} R", "refId": "A" }, + { "expr": "-sum by (job,device) (rate(node_disk_written_bytes_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} W", "refId": "B" } + ] + }, + { + "id": 10, "type": "timeseries", "title": "Disk busy %", + "description": "Fraction of time the device had I/O in flight. Near 1 = the device is the bottleneck.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] }, + "targets": [ + { "expr": "rate(node_disk_io_time_seconds_total{job=~\"$node\"}[5m])", "legendFormat": "{{job}} {{device}}", "refId": "A" } + ] + }, + { + "id": 11, "type": "timeseries", "title": "Network throughput", + "description": "Receive (up) / transmit (down) bytes/s per interface.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, + "targets": [ + { "expr": "sum by (job,device) (rate(node_network_receive_bytes_total{job=~\"$node\",device!~\"lo|veth.*|br.*|docker.*\"}[5m]))", "legendFormat": "{{job}} {{device}} RX", "refId": "A" }, + { "expr": "-sum by (job,device) (rate(node_network_transmit_bytes_total{job=~\"$node\",device!~\"lo|veth.*|br.*|docker.*\"}[5m]))", "legendFormat": "{{job}} {{device}} TX", "refId": "B" } + ] + }, + { + "id": 12, "type": "timeseries", "title": "Network errors / drops", + "description": "Interface errors and drops \u2014 the signature of a saturated link or a bad NIC/cable.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, + "targets": [ + { "expr": "sum by (job,device) (rate(node_network_receive_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} RX err", "refId": "A" }, + { "expr": "sum by (job,device) (rate(node_network_transmit_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} TX err", "refId": "B" } + ] + } + ] +} diff --git a/shared/grafana/provisioning/dashboards/logs-overview.json b/shared/grafana/provisioning/dashboards/logs-overview.json new file mode 100644 index 0000000..2a1fd15 --- /dev/null +++ b/shared/grafana/provisioning/dashboards/logs-overview.json @@ -0,0 +1,75 @@ +{ + "uid": "logs-overview", + "title": "Logs \u2014 Loki (systemd journal)", + "description": "Everything in Loki arrives as the journal (job=systemd-journal); the container is a JSON field parsed at query time, not a stream label. Starter breadth \u2014 the volume/error panels group by the parsed container_name. Loki keeps ~7 days; wide ranges are cheap counts, not the raw log panel.", + "tags": ["logs", "loki"], + "timezone": "browser", + "schemaVersion": 39, + "refresh": "30s", + "time": { "from": "now-1h", "to": "now" }, + "templating": { + "list": [ + { + "name": "container", + "label": "container", + "type": "query", + "datasource": { "type": "loki", "uid": "loki" }, + "query": { "query": "{job=\"systemd-journal\"} | json", "label": "container_name", "refId": "A" }, + "includeAll": true, + "multi": true, + "refresh": 2, + "current": { "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 1, "type": "timeseries", "title": "Log volume by container (lines/s)", + "description": "Parse the journal JSON, count lines per container. Spikes = a chatty or crashing unit.", + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 }, + "datasource": { "type": "loki", "uid": "loki" }, + "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } } }, "overrides": [] }, + "targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }] + }, + { + "id": 2, "type": "timeseries", "title": "Total log rate", + "description": "All journal lines per second.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, + "datasource": { "type": "loki", "uid": "loki" }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))", "legendFormat": "all lines/s", "refId": "A" }] + }, + { + "id": 3, "type": "timeseries", "title": "Error-level line rate", + "description": "Lines matching error/panic/fatal/fail. A blunt text match \u2014 the journal carries no structured level for all units.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, + "datasource": { "type": "loki", "uid": "loki" }, + "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "fixed", "fixedColor": "red" } }, "overrides": [] }, + "targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))", "legendFormat": "error lines/s", "refId": "A" }] + }, + { + "id": 4, "type": "timeseries", "title": "Errors by container", + "description": "Which unit is throwing. Rank by the top offenders.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, + "datasource": { "type": "loki", "uid": "loki" }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }] + }, + { + "id": 5, "type": "stat", "title": "Error lines (last 15m)", + "description": "Absolute count over the window, not a rate.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, + "datasource": { "type": "loki", "uid": "loki" }, + "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 500 }] } }, "overrides": [] }, + "targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))", "instant": true, "legendFormat": "errors/15m", "refId": "A" }] + }, + { + "id": 6, "type": "logs", "title": "Journal (filtered)", + "description": "Raw lines for the selected containers. Set the container picker; keep the window tight.", + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 }, + "datasource": { "type": "loki", "uid": "loki" }, + "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true }, + "targets": [{ "expr": "{job=\"systemd-journal\"} | json | container_name=~\"$container\"", "refId": "A" }] + } + ] +} diff --git a/shared/grafana/provisioning/dashboards/observability.json b/shared/grafana/provisioning/dashboards/observability.json new file mode 100644 index 0000000..d3a6fb2 --- /dev/null +++ b/shared/grafana/provisioning/dashboards/observability.json @@ -0,0 +1,96 @@ +{ + "uid": "observability", + "title": "Observability \u2014 self-monitoring", + "description": "The health of the stack that watches the fleet: Prometheus scrape/TSDB, Loki ingest, Grafana. If this is red the dashboards lie. Note: Alertmanager is running but NOT scraped by Prometheus (no job) \u2014 its metrics are absent by design gap, not fault.", + "tags": ["observability", "meta"], + "timezone": "browser", + "schemaVersion": 39, + "refresh": "30s", + "time": { "from": "now-6h", "to": "now" }, + "panels": [ + { + "id": 1, "type": "stat", "title": "Targets up", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] }, + "targets": [{ "expr": "count(up == 1)", "instant": true, "refId": "A" }] + }, + { + "id": 2, "type": "stat", "title": "Targets down", + "description": "Known structural downs: comfyui (no metrics endpoint), gitea (metrics off), and the homepage 'services' probes. Real regressions ride up from here too \u2014 check the table before dismissing.", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] } }, "overrides": [] }, + "targets": [{ "expr": "count(up == 0)", "instant": true, "refId": "A" }] + }, + { + "id": 3, "type": "stat", "title": "TSDB head series", + "description": "Active series in memory. A jump = a new exporter or a label-cardinality leak.", + "gridPos": { "h": 4, "w": 5, "x": 8, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "targets": [{ "expr": "prometheus_tsdb_head_series", "instant": true, "refId": "A" }] + }, + { + "id": 4, "type": "stat", "title": "Discovered targets", + "gridPos": { "h": 4, "w": 5, "x": 13, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "targets": [{ "expr": "sum(prometheus_sd_discovered_targets)", "instant": true, "refId": "A" }] + }, + { + "id": 5, "type": "stat", "title": "Prometheus mem", + "gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, + "targets": [{ "expr": "process_resident_memory_bytes{job=\"prometheus\"}", "instant": true, "refId": "A" }] + }, + { + "id": 6, "type": "stat", "title": "Grafana up", + "gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "mappings": [{ "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }], "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] }, + "targets": [{ "expr": "up{job=\"grafana\"}", "instant": true, "refId": "A" }] + }, + { + "id": 7, "type": "table", "title": "Target health", + "description": "Every scrape target and its current up. Sort by Value to surface regressions.", + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "options": { "sortBy": [{ "displayName": "Value", "sortOrder": "Asc" }] }, + "targets": [{ "expr": "up", "format": "table", "instant": true, "refId": "A" }] + }, + { + "id": 8, "type": "timeseries", "title": "Scrape interval drift (p99 actual)", + "description": "Actual seconds between scrapes (summary quantile). Above the 15s scrape_interval = Prometheus is falling behind its own cadence (slow endpoints or overload).", + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, + "targets": [{ "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}", "legendFormat": "{{instance}}", "refId": "A" }] + }, + { + "id": 9, "type": "timeseries", "title": "Scrape samples/s", + "description": "Total samples ingested across all jobs.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, + "targets": [{ "expr": "sum(rate(scrape_samples_scraped[5m]))", "legendFormat": "samples/s", "refId": "A" }] + }, + { + "id": 10, "type": "timeseries", "title": "Loki request rate", + "description": "Loki served requests/s (push, query). A drop to zero means ingest or query has stopped.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "targets": [{ "expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))", "legendFormat": "{{route}}", "refId": "A" }] + }, + { + "id": 11, "type": "timeseries", "title": "Exporter process CPU", + "description": "CPU by monitored process \u2014 spot a runaway exporter or the Prometheus process itself.", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 21 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, + "targets": [{ "expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))", "legendFormat": "{{job}}", "refId": "A" }] + } + ] +} -- 2.54.0