fix(grafana): observability and logs-overview boards read real data
observability.json: stale 'structural downs' panel description fixed, target census matches the post-blackbox job set. logs-overview.json: panels grouped by the promoted container label instead of the absent container_name, so the board stops reading empty beside live traffic.
This commit is contained in:
Frank committed
1 parent
fbe54ca1ad
commit
2af2e6ac56
2 files changed
+553
-96
No files matched your search
@@ -2,74 +2,242 @@
|
||||
"uid": "logs-overview",
|
||||
"title": "Logs \u2014 Loki (systemd journal)",
|
||||
"description": "Everything in Loki arrives as the journal (job=systemd-journal); the container is a JSON field parsed at query time, not a stream label. Starter breadth \u2014 the volume/error panels group by the parsed container_name. Loki keeps ~7 days; wide ranges are cheap counts, not the raw log panel.",
|
||||
"tags": ["logs", "loki"],
|
||||
"tags": [
|
||||
"logs",
|
||||
"loki"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "container",
|
||||
"label": "container",
|
||||
"type": "query",
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"query": { "query": "{job=\"systemd-journal\"} | json", "label": "container_name", "refId": "A" },
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"query": {
|
||||
"query": "label_values({job=\"systemd-journal\"}, container)",
|
||||
"refId": "LokiVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"refresh": 2,
|
||||
"current": { "text": "All", "value": "$__all" }
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "type": "timeseries", "title": "Log volume by container (lines/s)",
|
||||
"description": "Parse the journal JSON, count lines per container. Spikes = a chatty or crashing unit.",
|
||||
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps", "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }]
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "Log volume by container (lines/s)",
|
||||
"description": "Per-container journal line rate. The `container` label is promoted at ingestion by Alloy (loki.process) from podman's CONTAINER_NAME journald field - Loki metrics can only group by indexed stream labels.",
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"custom": {
|
||||
"fillOpacity": 20,
|
||||
"stacking": {
|
||||
"mode": "normal"
|
||||
}
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (container) (count_over_time({job=\"systemd-journal\", container!=\"\"} [$__auto]))",
|
||||
"legendFormat": "{{container_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2, "type": "timeseries", "title": "Total log rate",
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Total log rate",
|
||||
"description": "All journal lines per second.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))", "legendFormat": "all lines/s", "refId": "A" }]
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))",
|
||||
"legendFormat": "all lines/s",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3, "type": "timeseries", "title": "Error-level line rate",
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "Error-ish line rate (text match)",
|
||||
"description": "Lines matching error/panic/fatal/fail. A blunt text match \u2014 the journal carries no structured level for all units.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "fixed", "fixedColor": "red" } }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))", "legendFormat": "error lines/s", "refId": "A" }]
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "red"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))",
|
||||
"legendFormat": "error lines/s",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4, "type": "timeseries", "title": "Errors by container",
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Errors by container",
|
||||
"description": "Which unit is throwing. Rank by the top offenders.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }]
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 17
|
||||
},
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (container) (count_over_time({job=\"systemd-journal\", container!=\"\"} |~ \"(?i)(error|panic|fatal)\" [$__auto]))",
|
||||
"legendFormat": "{{container_name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5, "type": "stat", "title": "Error lines (last 15m)",
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Error lines (last 15m)",
|
||||
"description": "Absolute count over the window, not a rate.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 500 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))", "instant": true, "legendFormat": "errors/15m", "refId": "A" }]
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 17
|
||||
},
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 50
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 500
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))",
|
||||
"instant": true,
|
||||
"legendFormat": "errors/15m",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6, "type": "logs", "title": "Journal (filtered)",
|
||||
"id": 6,
|
||||
"type": "logs",
|
||||
"title": "Journal (filtered)",
|
||||
"description": "Raw lines for the selected containers. Set the container picker; keep the window tight.",
|
||||
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
|
||||
"targets": [{ "expr": "{job=\"systemd-journal\"} | json | container_name=~\"$container\"", "refId": "A" }]
|
||||
"gridPos": {
|
||||
"h": 12,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 25
|
||||
},
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"wrapLogMessage": true,
|
||||
"sortOrder": "Descending",
|
||||
"enableLogDetails": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{job=\"systemd-journal\", container=~\"$container\"}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user