diff --git a/shared/grafana/provisioning/dashboards/logs-overview.json b/shared/grafana/provisioning/dashboards/logs-overview.json index 2a1fd15..2ca5321 100644 --- a/shared/grafana/provisioning/dashboards/logs-overview.json +++ b/shared/grafana/provisioning/dashboards/logs-overview.json @@ -2,74 +2,242 @@ "uid": "logs-overview", "title": "Logs \u2014 Loki (systemd journal)", "description": "Everything in Loki arrives as the journal (job=systemd-journal); the container is a JSON field parsed at query time, not a stream label. Starter breadth \u2014 the volume/error panels group by the parsed container_name. Loki keeps ~7 days; wide ranges are cheap counts, not the raw log panel.", - "tags": ["logs", "loki"], + "tags": [ + "logs", + "loki" + ], "timezone": "browser", "schemaVersion": 39, "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "templating": { "list": [ { "name": "container", "label": "container", "type": "query", - "datasource": { "type": "loki", "uid": "loki" }, - "query": { "query": "{job=\"systemd-journal\"} | json", "label": "container_name", "refId": "A" }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": { + "query": "label_values({job=\"systemd-journal\"}, container)", + "refId": "LokiVariableQueryEditor-VariableQuery" + }, "includeAll": true, "multi": true, "refresh": 2, - "current": { "text": "All", "value": "$__all" } + "current": { + "text": "All", + "value": "$__all" + } } ] }, "panels": [ { - "id": 1, "type": "timeseries", "title": "Log volume by container (lines/s)", - "description": "Parse the journal JSON, count lines per container. Spikes = a chatty or crashing unit.", - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 }, - "datasource": { "type": "loki", "uid": "loki" }, - "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } } }, "overrides": [] }, - "targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }] + "id": 1, + "type": "timeseries", + "title": "Log volume by container (lines/s)", + "description": "Per-container journal line rate. The `container` label is promoted at ingestion by Alloy (loki.process) from podman's CONTAINER_NAME journald field - Loki metrics can only group by indexed stream labels.", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "fillOpacity": 20, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (container) (count_over_time({job=\"systemd-journal\", container!=\"\"} [$__auto]))", + "legendFormat": "{{container_name}}", + "refId": "A" + } + ] }, { - "id": 2, "type": "timeseries", "title": "Total log rate", + "id": 2, + "type": "timeseries", + "title": "Total log rate", "description": "All journal lines per second.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "datasource": { "type": "loki", "uid": "loki" }, - "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, - "targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))", "legendFormat": "all lines/s", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))", + "legendFormat": "all lines/s", + "refId": "A" + } + ] }, { - "id": 3, "type": "timeseries", "title": "Error-level line rate", + "id": 3, + "type": "timeseries", + "title": "Error-ish line rate (text match)", "description": "Lines matching error/panic/fatal/fail. A blunt text match \u2014 the journal carries no structured level for all units.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, - "datasource": { "type": "loki", "uid": "loki" }, - "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "fixed", "fixedColor": "red" } }, "overrides": [] }, - "targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))", "legendFormat": "error lines/s", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "fixed", + "fixedColor": "red" + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))", + "legendFormat": "error lines/s", + "refId": "A" + } + ] }, { - "id": 4, "type": "timeseries", "title": "Errors by container", + "id": 4, + "type": "timeseries", + "title": "Errors by container", "description": "Which unit is throwing. Rank by the top offenders.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, - "datasource": { "type": "loki", "uid": "loki" }, - "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, - "targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (container) (count_over_time({job=\"systemd-journal\", container!=\"\"} |~ \"(?i)(error|panic|fatal)\" [$__auto]))", + "legendFormat": "{{container_name}}", + "refId": "A" + } + ] }, { - "id": 5, "type": "stat", "title": "Error lines (last 15m)", + "id": 5, + "type": "stat", + "title": "Error lines (last 15m)", "description": "Absolute count over the window, not a rate.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, - "datasource": { "type": "loki", "uid": "loki" }, - "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 500 }] } }, "overrides": [] }, - "targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))", "instant": true, "legendFormat": "errors/15m", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 500 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))", + "instant": true, + "legendFormat": "errors/15m", + "refId": "A" + } + ] }, { - "id": 6, "type": "logs", "title": "Journal (filtered)", + "id": 6, + "type": "logs", + "title": "Journal (filtered)", "description": "Raw lines for the selected containers. Set the container picker; keep the window tight.", - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 }, - "datasource": { "type": "loki", "uid": "loki" }, - "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true }, - "targets": [{ "expr": "{job=\"systemd-journal\"} | json | container_name=~\"$container\"", "refId": "A" }] + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 25 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending", + "enableLogDetails": true + }, + "targets": [ + { + "expr": "{job=\"systemd-journal\", container=~\"$container\"}", + "refId": "A" + } + ] } ] -} +} \ No newline at end of file diff --git a/shared/grafana/provisioning/dashboards/observability.json b/shared/grafana/provisioning/dashboards/observability.json index d3a6fb2..ad3da1d 100644 --- a/shared/grafana/provisioning/dashboards/observability.json +++ b/shared/grafana/provisioning/dashboards/observability.json @@ -2,95 +2,384 @@ "uid": "observability", "title": "Observability \u2014 self-monitoring", "description": "The health of the stack that watches the fleet: Prometheus scrape/TSDB, Loki ingest, Grafana. If this is red the dashboards lie. Note: Alertmanager is running but NOT scraped by Prometheus (no job) \u2014 its metrics are absent by design gap, not fault.", - "tags": ["observability", "meta"], + "tags": [ + "observability", + "meta" + ], "timezone": "browser", "schemaVersion": 39, "refresh": "30s", - "time": { "from": "now-6h", "to": "now" }, + "time": { + "from": "now-6h", + "to": "now" + }, "panels": [ { - "id": 1, "type": "stat", "title": "Targets up", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] }, - "targets": [{ "expr": "count(up == 1)", "instant": true, "refId": "A" }] + "id": 1, + "type": "stat", + "title": "Targets up", + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "count(up == 1)", + "instant": true, + "refId": "A" + } + ] }, { - "id": 2, "type": "stat", "title": "Targets down", - "description": "Known structural downs: comfyui (no metrics endpoint), gitea (metrics off), and the homepage 'services' probes. Real regressions ride up from here too \u2014 check the table before dismissing.", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] } }, "overrides": [] }, - "targets": [{ "expr": "count(up == 0)", "instant": true, "refId": "A" }] + "id": 2, + "type": "stat", + "title": "Targets down", + "description": "Every scrape target that is down. ZERO IS THE HEALTHY STATE: the HTML-probe jobs and the disabled Gitea metrics were retired 2026-10-21 (HTTP liveness now rides the blackbox exporter, job \"services\"). A non-zero count here is a real regression - read /api/v1/targets lastError before triaging.", + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "count(up == 0)", + "instant": true, + "refId": "A" + } + ] }, { - "id": 3, "type": "stat", "title": "TSDB head series", + "id": 3, + "type": "stat", + "title": "TSDB head series", "description": "Active series in memory. A jump = a new exporter or a label-cardinality leak.", - "gridPos": { "h": 4, "w": 5, "x": 8, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, - "targets": [{ "expr": "prometheus_tsdb_head_series", "instant": true, "refId": "A" }] + "gridPos": { + "h": 4, + "w": 5, + "x": 8, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "prometheus_tsdb_head_series", + "instant": true, + "refId": "A" + } + ] }, { - "id": 4, "type": "stat", "title": "Discovered targets", - "gridPos": { "h": 4, "w": 5, "x": 13, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, - "targets": [{ "expr": "sum(prometheus_sd_discovered_targets)", "instant": true, "refId": "A" }] + "id": 4, + "type": "stat", + "title": "Discovered targets", + "gridPos": { + "h": 4, + "w": 5, + "x": 13, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets)", + "instant": true, + "refId": "A" + } + ] }, { - "id": 5, "type": "stat", "title": "Prometheus mem", - "gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, - "targets": [{ "expr": "process_resident_memory_bytes{job=\"prometheus\"}", "instant": true, "refId": "A" }] + "id": 5, + "type": "stat", + "title": "Prometheus mem", + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"prometheus\"}", + "instant": true, + "refId": "A" + } + ] }, { - "id": 6, "type": "stat", "title": "Grafana up", - "gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "mappings": [{ "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }], "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] }, - "targets": [{ "expr": "up{job=\"grafana\"}", "instant": true, "refId": "A" }] + "id": 6, + "type": "stat", + "title": "Grafana up", + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "1": { + "text": "UP", + "color": "green" + }, + "0": { + "text": "DOWN", + "color": "red" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "up{job=\"grafana\"}", + "instant": true, + "refId": "A" + } + ] }, { - "id": 7, "type": "table", "title": "Target health", + "id": 7, + "type": "table", + "title": "Target health", "description": "Every scrape target and its current up. Sort by Value to surface regressions.", - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 4 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "options": { "sortBy": [{ "displayName": "Value", "sortOrder": "Asc" }] }, - "targets": [{ "expr": "up", "format": "table", "instant": true, "refId": "A" }] + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "options": { + "sortBy": [ + { + "displayName": "Value", + "sortOrder": "Asc" + } + ] + }, + "targets": [ + { + "expr": "up", + "format": "table", + "instant": true, + "refId": "A" + } + ] }, { - "id": 8, "type": "timeseries", "title": "Scrape interval drift (p99 actual)", + "id": 8, + "type": "timeseries", + "title": "Scrape interval drift (p99 actual)", "description": "Actual seconds between scrapes (summary quantile). Above the 15s scrape_interval = Prometheus is falling behind its own cadence (slow endpoints or overload).", - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 4 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, - "targets": [{ "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}", "legendFormat": "{{instance}}", "refId": "A" }] + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] }, { - "id": 9, "type": "timeseries", "title": "Scrape samples/s", + "id": 9, + "type": "timeseries", + "title": "Scrape samples/s", "description": "Total samples ingested across all jobs.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, - "targets": [{ "expr": "sum(rate(scrape_samples_scraped[5m]))", "legendFormat": "samples/s", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(rate(scrape_samples_scraped[5m]))", + "legendFormat": "samples/s", + "refId": "A" + } + ] }, { - "id": 10, "type": "timeseries", "title": "Loki request rate", + "id": 10, + "type": "timeseries", + "title": "Loki request rate", "description": "Loki served requests/s (push, query). A drop to zero means ingest or query has stopped.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, - "targets": [{ "expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))", "legendFormat": "{{route}}", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))", + "legendFormat": "{{route}}", + "refId": "A" + } + ] }, { - "id": 11, "type": "timeseries", "title": "Exporter process CPU", + "id": 11, + "type": "timeseries", + "title": "Exporter process CPU", "description": "CPU by monitored process \u2014 spot a runaway exporter or the Prometheus process itself.", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, - "targets": [{ "expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))", "legendFormat": "{{job}}", "refId": "A" }] + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))", + "legendFormat": "{{job}}", + "refId": "A" + } + ] } ] -} +} \ No newline at end of file