Frank
1006e96312
llm-fleet.json already owns LLM capacity+spec-decode. These three cover the domains that had NO dashboard: - host-system: node_exporter CPU/Load/Mem/Disk/Net/Uptime/FS/FDs for both Dankclaw and Dankserver (templated by job node-dankclaw|node-dankserver) - logs-overview: Loki volume + per-container error rate + a sample log panel; container is parsed from the JSON line at query time (journal streams have no container_name stream label) - observability: the stack's own health - Prometheus target up/down + scrape interval drift + samples/s + TSDB, Loki ingest/discard, Grafana CPU/mem. Panel queries verified live against Prometheus/Loki; provisioned dashboards confirmed via /api/search. Alertmanager is running but NOT scraped by Prometheus - noted in the dashboard, no empty panels built on it.
145 lines
8.9 KiB
JSON
145 lines
8.9 KiB
JSON
{
|
|
"uid": "host-system",
|
|
"title": "Hosts \u2014 Dankclaw & Dankserver",
|
|
"description": "node_exporter for both servers. Starter set \u2014 throw-at-the-wall breadth; refine the windows and drop the noise. The picker selects the host; CPU/load/memory/filesystem/network are per-node.",
|
|
"tags": ["host", "system"],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"refresh": "30s",
|
|
"time": { "from": "now-6h", "to": "now" },
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "node",
|
|
"label": "host",
|
|
"type": "query",
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"query": { "query": "label_values(up{job=~\"node-.*\"}, job)", "refId": "A" },
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"current": { "text": "All", "value": "$__all" }
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1, "type": "stat", "title": "Uptime",
|
|
"description": "Time since boot. A drop means a reboot.",
|
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
|
|
"targets": [{ "expr": "time() - node_boot_time_seconds{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 2, "type": "stat", "title": "Memory used",
|
|
"description": "1 - MemAvailable/MemTotal. The honest \"in use\" figure (excludes reclaimable cache).",
|
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.8 }, { "color": "red", "value": 0.92 }] } }, "overrides": [] },
|
|
"targets": [{ "expr": "1 - node_memory_MemAvailable_bytes{job=~\"$node\"} / node_memory_MemTotal_bytes{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 3, "type": "stat", "title": "Load 5m / core",
|
|
"description": "Per-core load5. >1 sustained means runnable tasks are queuing behind cores.",
|
|
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 1 }] } }, "overrides": [] },
|
|
"targets": [{ "expr": "node_load5{job=~\"$node\"} / count by (job) (node_cpu_seconds_total{job=~\"$node\",mode=\"idle\"})", "legendFormat": "{{job}}", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 4, "type": "stat", "title": "Open file handles",
|
|
"description": "FDs allocated vs the kernel maximum. Rising toward the max precedes EMFILE across the host.",
|
|
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 }] } }, "overrides": [] },
|
|
"targets": [{ "expr": "node_filefd_allocated{job=~\"$node\"} / node_filefd_maximum{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 5, "type": "timeseries", "title": "CPU busy %",
|
|
"description": "CPU seconds/s in each mode = cores busy per mode; the stack sums to total cores in use (16/host here). A user or system spike points at what is loading the box; steal > 0 means the hypervisor is throttling.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "sum by (job,mode) (rate(node_cpu_seconds_total{job=~\"$node\",mode!=\"idle\",mode!=\"iowait\"}[5m]))", "legendFormat": "{{job}} {{mode}}", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 6, "type": "timeseries", "title": "Load average",
|
|
"description": "load1/5/15 per host.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "node_load1{job=~\"$node\"}", "legendFormat": "{{job}} load1", "refId": "A" },
|
|
{ "expr": "node_load5{job=~\"$node\"}", "legendFormat": "{{job}} load5", "refId": "B" },
|
|
{ "expr": "node_load15{job=~\"$node\"}", "legendFormat": "{{job}} load15", "refId": "C" }
|
|
]
|
|
},
|
|
{
|
|
"id": 7, "type": "timeseries", "title": "Memory breakdown",
|
|
"description": "Total, free, buffers+cache. Reclaimable cache is not pressure; MemAvailable is the real headroom.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "node_memory_MemTotal_bytes{job=~\"$node\"} - node_memory_MemAvailable_bytes{job=~\"$node\"}", "legendFormat": "{{job}} used", "refId": "A" },
|
|
{ "expr": "node_memory_MemAvailable_bytes{job=~\"$node\"}", "legendFormat": "{{job}} available", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 8, "type": "timeseries", "title": "Filesystem free %",
|
|
"description": "Per mountpoint. The media/library stacks fill disks.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.15 }] } }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "node_filesystem_avail_bytes{job=~\"$node\",fstype!~\"tmpfs|overlay|squashfs\"} / node_filesystem_size_bytes{job=~\"$node\",fstype!~\"tmpfs|overlay|squashfs\"}", "legendFormat": "{{job}} {{mountpoint}}", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 9, "type": "timeseries", "title": "Disk throughput",
|
|
"description": "Read/write bytes/s per device.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "sum by (job,device) (rate(node_disk_read_bytes_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} R", "refId": "A" },
|
|
{ "expr": "-sum by (job,device) (rate(node_disk_written_bytes_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} W", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 10, "type": "timeseries", "title": "Disk busy %",
|
|
"description": "Fraction of time the device had I/O in flight. Near 1 = the device is the bottleneck.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "rate(node_disk_io_time_seconds_total{job=~\"$node\"}[5m])", "legendFormat": "{{job}} {{device}}", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 11, "type": "timeseries", "title": "Network throughput",
|
|
"description": "Receive (up) / transmit (down) bytes/s per interface.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "sum by (job,device) (rate(node_network_receive_bytes_total{job=~\"$node\",device!~\"lo|veth.*|br.*|docker.*\"}[5m]))", "legendFormat": "{{job}} {{device}} RX", "refId": "A" },
|
|
{ "expr": "-sum by (job,device) (rate(node_network_transmit_bytes_total{job=~\"$node\",device!~\"lo|veth.*|br.*|docker.*\"}[5m]))", "legendFormat": "{{job}} {{device}} TX", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 12, "type": "timeseries", "title": "Network errors / drops",
|
|
"description": "Interface errors and drops \u2014 the signature of a saturated link or a bad NIC/cable.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "sum by (job,device) (rate(node_network_receive_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} RX err", "refId": "A" },
|
|
{ "expr": "sum by (job,device) (rate(node_network_transmit_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} TX err", "refId": "B" }
|
|
]
|
|
}
|
|
]
|
|
}
|