Frank
1006e96312
llm-fleet.json already owns LLM capacity+spec-decode. These three cover the domains that had NO dashboard: - host-system: node_exporter CPU/Load/Mem/Disk/Net/Uptime/FS/FDs for both Dankclaw and Dankserver (templated by job node-dankclaw|node-dankserver) - logs-overview: Loki volume + per-container error rate + a sample log panel; container is parsed from the JSON line at query time (journal streams have no container_name stream label) - observability: the stack's own health - Prometheus target up/down + scrape interval drift + samples/s + TSDB, Loki ingest/discard, Grafana CPU/mem. Panel queries verified live against Prometheus/Loki; provisioned dashboards confirmed via /api/search. Alertmanager is running but NOT scraped by Prometheus - noted in the dashboard, no empty panels built on it.
76 lines
4.3 KiB
JSON
76 lines
4.3 KiB
JSON
{
|
|
"uid": "logs-overview",
|
|
"title": "Logs \u2014 Loki (systemd journal)",
|
|
"description": "Everything in Loki arrives as the journal (job=systemd-journal); the container is a JSON field parsed at query time, not a stream label. Starter breadth \u2014 the volume/error panels group by the parsed container_name. Loki keeps ~7 days; wide ranges are cheap counts, not the raw log panel.",
|
|
"tags": ["logs", "loki"],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"refresh": "30s",
|
|
"time": { "from": "now-1h", "to": "now" },
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "container",
|
|
"label": "container",
|
|
"type": "query",
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"query": { "query": "{job=\"systemd-journal\"} | json", "label": "container_name", "refId": "A" },
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"refresh": 2,
|
|
"current": { "text": "All", "value": "$__all" }
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1, "type": "timeseries", "title": "Log volume by container (lines/s)",
|
|
"description": "Parse the journal JSON, count lines per container. Spikes = a chatty or crashing unit.",
|
|
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"fieldConfig": { "defaults": { "unit": "reqps", "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
|
"targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 2, "type": "timeseries", "title": "Total log rate",
|
|
"description": "All journal lines per second.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
|
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))", "legendFormat": "all lines/s", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 3, "type": "timeseries", "title": "Error-level line rate",
|
|
"description": "Lines matching error/panic/fatal/fail. A blunt text match \u2014 the journal carries no structured level for all units.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "fixed", "fixedColor": "red" } }, "overrides": [] },
|
|
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))", "legendFormat": "error lines/s", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 4, "type": "timeseries", "title": "Errors by container",
|
|
"description": "Which unit is throwing. Rank by the top offenders.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
|
"targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 5, "type": "stat", "title": "Error lines (last 15m)",
|
|
"description": "Absolute count over the window, not a rate.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 500 }] } }, "overrides": [] },
|
|
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))", "instant": true, "legendFormat": "errors/15m", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 6, "type": "logs", "title": "Journal (filtered)",
|
|
"description": "Raw lines for the selected containers. Set the container picker; keep the window tight.",
|
|
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
|
|
"targets": [{ "expr": "{job=\"systemd-journal\"} | json | container_name=~\"$container\"", "refId": "A" }]
|
|
}
|
|
]
|
|
}
|