dank/serverAppConfigs
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(grafana): observability and logs-overview boards read real data

Browse Source 
observability.json: stale 'structural downs' panel description fixed,
target census matches the post-blackbox job set. logs-overview.json:
panels grouped by the promoted container label instead of the absent
container_name, so the board stops reading empty beside live traffic.
This commit is contained in:
Frank
2026-10-21 13:32:58 -07:00
parent fbe54ca1ad
commit 2af2e6ac56
2 changed files with 553 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
shared/grafana/provisioning/dashboards/logs-overview.json
+205 -37
View File
@@ -2,74 +2,242 @@
"uid": "logs-overview",
"title": "Logs \u2014 Loki (systemd journal)",
"description": "Everything in Loki arrives as the journal (job=systemd-journal); the container is a JSON field parsed at query time, not a stream label. Starter breadth \u2014 the volume/error panels group by the parsed container_name. Loki keeps ~7 days; wide ranges are cheap counts, not the raw log panel.",
"tags": ["logs", "loki"],
"tags": [
"logs",
"loki"
],
"timezone": "browser",
"schemaVersion": 39,
"refresh": "30s",
"time": { "from": "now-1h", "to": "now" },
"time": {
"from": "now-1h",
"to": "now"
},
"templating": {
"list": [
{
"name": "container",
"label": "container",
"type": "query",
"datasource": { "type": "loki", "uid": "loki" },
"query": { "query": "{job=\"systemd-journal\"} | json", "label": "container_name", "refId": "A" },
"datasource": {
"type": "loki",
"uid": "loki"
},
"query": {
"query": "label_values({job=\"systemd-journal\"}, container)",
"refId": "LokiVariableQueryEditor-VariableQuery"
},
"includeAll": true,
"multi": true,
"refresh": 2,
"current": { "text": "All", "value": "$__all" }
"current": {
"text": "All",
"value": "$__all"
}
}
]
},
"panels": [
{
"id": 1, "type": "timeseries", "title": "Log volume by container (lines/s)",
"description": "Parse the journal JSON, count lines per container. Spikes = a chatty or crashing unit.",
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 },
"datasource": { "type": "loki", "uid": "loki" },
"fieldConfig": { "defaults": { "unit": "reqps", "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } } }, "overrides": [] },
"targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }]
"id": 1,
"type": "timeseries",
"title": "Log volume by container (lines/s)",
"description": "Per-container journal line rate. The `container` label is promoted at ingestion by Alloy (loki.process) from podman's CONTAINER_NAME journald field - Loki metrics can only group by indexed stream labels.",
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 0
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"fillOpacity": 20,
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
},
"targets": [
{
"expr": "sum by (container) (count_over_time({job=\"systemd-journal\", container!=\"\"} [$__auto]))",
"legendFormat": "{{container_name}}",
"refId": "A"
}
]
},
{
"id": 2, "type": "timeseries", "title": "Total log rate",
"id": 2,
"type": "timeseries",
"title": "Total log rate",
"description": "All journal lines per second.",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
"datasource": { "type": "loki", "uid": "loki" },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))", "legendFormat": "all lines/s", "refId": "A" }]
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 9
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"targets": [
{
"expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))",
"legendFormat": "all lines/s",
"refId": "A"
}
]
},
{
"id": 3, "type": "timeseries", "title": "Error-level line rate",
"id": 3,
"type": "timeseries",
"title": "Error-ish line rate (text match)",
"description": "Lines matching error/panic/fatal/fail. A blunt text match \u2014 the journal carries no structured level for all units.",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
"datasource": { "type": "loki", "uid": "loki" },
"fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "fixed", "fixedColor": "red" } }, "overrides": [] },
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))", "legendFormat": "error lines/s", "refId": "A" }]
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 9
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"color": {
"mode": "fixed",
"fixedColor": "red"
}
},
"overrides": []
},
"targets": [
{
"expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))",
"legendFormat": "error lines/s",
"refId": "A"
}
]
},
{
"id": 4, "type": "timeseries", "title": "Errors by container",
"id": 4,
"type": "timeseries",
"title": "Errors by container",
"description": "Which unit is throwing. Rank by the top offenders.",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
"datasource": { "type": "loki", "uid": "loki" },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }]
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 17
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"targets": [
{
"expr": "sum by (container) (count_over_time({job=\"systemd-journal\", container!=\"\"} |~ \"(?i)(error|panic|fatal)\" [$__auto]))",
"legendFormat": "{{container_name}}",
"refId": "A"
}
]
},
{
"id": 5, "type": "stat", "title": "Error lines (last 15m)",
"id": 5,
"type": "stat",
"title": "Error lines (last 15m)",
"description": "Absolute count over the window, not a rate.",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
"datasource": { "type": "loki", "uid": "loki" },
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 500 }] } }, "overrides": [] },
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))", "instant": true, "legendFormat": "errors/15m", "refId": "A" }]
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 17
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 50
},
{
"color": "red",
"value": 500
}
]
}
},
"overrides": []
},
"targets": [
{
"expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))",
"instant": true,
"legendFormat": "errors/15m",
"refId": "A"
}
]
},
{
"id": 6, "type": "logs", "title": "Journal (filtered)",
"id": 6,
"type": "logs",
"title": "Journal (filtered)",
"description": "Raw lines for the selected containers. Set the container picker; keep the window tight.",
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 },
"datasource": { "type": "loki", "uid": "loki" },
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
"targets": [{ "expr": "{job=\"systemd-journal\"} | json | container_name=~\"$container\"", "refId": "A" }]
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 25
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"options": {
"showTime": true,
"wrapLogMessage": true,
"sortOrder": "Descending",
"enableLogDetails": true
},
"targets": [
{
"expr": "{job=\"systemd-journal\", container=~\"$container\"}",
"refId": "A"
}
]
}
]
}
}
shared/grafana/provisioning/dashboards/observability.json
+348 -59
View File
@@ -2,95 +2,384 @@
"uid": "observability",
"title": "Observability \u2014 self-monitoring",
"description": "The health of the stack that watches the fleet: Prometheus scrape/TSDB, Loki ingest, Grafana. If this is red the dashboards lie. Note: Alertmanager is running but NOT scraped by Prometheus (no job) \u2014 its metrics are absent by design gap, not fault.",
"tags": ["observability", "meta"],
"tags": [
"observability",
"meta"
],
"timezone": "browser",
"schemaVersion": 39,
"refresh": "30s",
"time": { "from": "now-6h", "to": "now" },
"time": {
"from": "now-6h",
"to": "now"
},
"panels": [
{
"id": 1, "type": "stat", "title": "Targets up",
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] },
"targets": [{ "expr": "count(up == 1)", "instant": true, "refId": "A" }]
"id": 1,
"type": "stat",
"title": "Targets up",
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 0
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"targets": [
{
"expr": "count(up == 1)",
"instant": true,
"refId": "A"
}
]
},
{
"id": 2, "type": "stat", "title": "Targets down",
"description": "Known structural downs: comfyui (no metrics endpoint), gitea (metrics off), and the homepage 'services' probes. Real regressions ride up from here too \u2014 check the table before dismissing.",
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] } }, "overrides": [] },
"targets": [{ "expr": "count(up == 0)", "instant": true, "refId": "A" }]
"id": 2,
"type": "stat",
"title": "Targets down",
"description": "Every scrape target that is down. ZERO IS THE HEALTHY STATE: the HTML-probe jobs and the disabled Gitea metrics were retired 2026-10-21 (HTTP liveness now rides the blackbox exporter, job \"services\"). A non-zero count here is a real regression - read /api/v1/targets lastError before triaging.",
"gridPos": {
"h": 4,
"w": 4,
"x": 4,
"y": 0
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"targets": [
{
"expr": "count(up == 0)",
"instant": true,
"refId": "A"
}
]
},
{
"id": 3, "type": "stat", "title": "TSDB head series",
"id": 3,
"type": "stat",
"title": "TSDB head series",
"description": "Active series in memory. A jump = a new exporter or a label-cardinality leak.",
"gridPos": { "h": 4, "w": 5, "x": 8, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
"targets": [{ "expr": "prometheus_tsdb_head_series", "instant": true, "refId": "A" }]
"gridPos": {
"h": 4,
"w": 5,
"x": 8,
"y": 0
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"expr": "prometheus_tsdb_head_series",
"instant": true,
"refId": "A"
}
]
},
{
"id": 4, "type": "stat", "title": "Discovered targets",
"gridPos": { "h": 4, "w": 5, "x": 13, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
"targets": [{ "expr": "sum(prometheus_sd_discovered_targets)", "instant": true, "refId": "A" }]
"id": 4,
"type": "stat",
"title": "Discovered targets",
"gridPos": {
"h": 4,
"w": 5,
"x": 13,
"y": 0
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"expr": "sum(prometheus_sd_discovered_targets)",
"instant": true,
"refId": "A"
}
]
},
{
"id": 5, "type": "stat", "title": "Prometheus mem",
"gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
"targets": [{ "expr": "process_resident_memory_bytes{job=\"prometheus\"}", "instant": true, "refId": "A" }]
"id": 5,
"type": "stat",
"title": "Prometheus mem",
"gridPos": {
"h": 4,
"w": 3,
"x": 18,
"y": 0
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"targets": [
{
"expr": "process_resident_memory_bytes{job=\"prometheus\"}",
"instant": true,
"refId": "A"
}
]
},
{
"id": 6, "type": "stat", "title": "Grafana up",
"gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "mappings": [{ "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }], "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] },
"targets": [{ "expr": "up{job=\"grafana\"}", "instant": true, "refId": "A" }]
"id": 6,
"type": "stat",
"title": "Grafana up",
"gridPos": {
"h": 4,
"w": 3,
"x": 21,
"y": 0
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"mappings": [
{
"type": "value",
"options": {
"1": {
"text": "UP",
"color": "green"
},
"0": {
"text": "DOWN",
"color": "red"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"targets": [
{
"expr": "up{job=\"grafana\"}",
"instant": true,
"refId": "A"
}
]
},
{
"id": 7, "type": "table", "title": "Target health",
"id": 7,
"type": "table",
"title": "Target health",
"description": "Every scrape target and its current up. Sort by Value to surface regressions.",
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 4 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"options": { "sortBy": [{ "displayName": "Value", "sortOrder": "Asc" }] },
"targets": [{ "expr": "up", "format": "table", "instant": true, "refId": "A" }]
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 4
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"options": {
"sortBy": [
{
"displayName": "Value",
"sortOrder": "Asc"
}
]
},
"targets": [
{
"expr": "up",
"format": "table",
"instant": true,
"refId": "A"
}
]
},
{
"id": 8, "type": "timeseries", "title": "Scrape interval drift (p99 actual)",
"id": 8,
"type": "timeseries",
"title": "Scrape interval drift (p99 actual)",
"description": "Actual seconds between scrapes (summary quantile). Above the 15s scrape_interval = Prometheus is falling behind its own cadence (slow endpoints or overload).",
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 4 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
"targets": [{ "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}", "legendFormat": "{{instance}}", "refId": "A" }]
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 4
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
},
{
"id": 9, "type": "timeseries", "title": "Scrape samples/s",
"id": 9,
"type": "timeseries",
"title": "Scrape samples/s",
"description": "Total samples ingested across all jobs.",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
"targets": [{ "expr": "sum(rate(scrape_samples_scraped[5m]))", "legendFormat": "samples/s", "refId": "A" }]
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 13
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"expr": "sum(rate(scrape_samples_scraped[5m]))",
"legendFormat": "samples/s",
"refId": "A"
}
]
},
{
"id": 10, "type": "timeseries", "title": "Loki request rate",
"id": 10,
"type": "timeseries",
"title": "Loki request rate",
"description": "Loki served requests/s (push, query). A drop to zero means ingest or query has stopped.",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"targets": [{ "expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))", "legendFormat": "{{route}}", "refId": "A" }]
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 13
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"targets": [
{
"expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))",
"legendFormat": "{{route}}",
"refId": "A"
}
]
},
{
"id": 11, "type": "timeseries", "title": "Exporter process CPU",
"id": 11,
"type": "timeseries",
"title": "Exporter process CPU",
"description": "CPU by monitored process \u2014 spot a runaway exporter or the Prometheus process itself.",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 21 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] },
"targets": [{ "expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))", "legendFormat": "{{job}}", "refId": "A" }]
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 21
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"targets": [
{
"expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))",
"legendFormat": "{{job}}",
"refId": "A"
}
]
}
]
}
}