Frank
b02647b992
Add a host-process GPU exporter, scrape it on both servers, and graph it on the host-system dashboard. The fleet has four GPUs across three vendors of interest (2x R9700 on Dankserver, the 8060S iGPU + RTX 3090 on Dankclaw) and nothing exposed them until now. scripts/gpu-exporter.py reads each AMD card's amdgpu sysfs (gpu_busy_percent / mem_info_vram_* / hwmon temp1_input / power1_average) plus one nvidia-smi --query-gpu CSV, and emits unified gpu_* Prometheus families (utilization_percent, memory_used/total_bytes, temperature_celsius, power_watts) labelled gpu/vendor/model. Absent hardware yields ABSENT series, never a fabricated 0: the R9700s expose no sysfs power, so they simply carry no gpu_power_watts line. Both sources are injectable, so the seam runs on a GPU-less dev box; live-verified on Dankserver's real sysfs. It runs as a plain user unit (not a quadlet): it must read the host's own DRM sysfs and shell to nvidia-smi, which a container cannot do without GPU passthrough. Same posture as signal-relay. The base unit binds loopback; the deploy drop-in appends the host LAN address, never a wildcard (matching node-exporter/cadvisor). deploy_gpu_exporter installs/enables/health-gates it on both hosts and is wired into the planner (both targets), the test-changed router, and two Prometheus scrape jobs using the node/cadvisor reach rules (gateway on Dankclaw, LAN on Dankserver).
205 lines
12 KiB
JSON
205 lines
12 KiB
JSON
{
|
|
"uid": "host-system",
|
|
"title": "Hosts \u2014 Dankclaw & Dankserver",
|
|
"description": "node_exporter for both servers. Starter set \u2014 throw-at-the-wall breadth; refine the windows and drop the noise. The picker selects the host; CPU/load/memory/filesystem/network are per-node.",
|
|
"tags": ["host", "system"],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"refresh": "30s",
|
|
"time": { "from": "now-6h", "to": "now" },
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "node",
|
|
"label": "host",
|
|
"type": "query",
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"query": { "query": "label_values(up{job=~\"node-.*\"}, job)", "refId": "A" },
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"current": { "text": "All", "value": "$__all" }
|
|
},
|
|
{
|
|
"name": "gpuhost",
|
|
"label": "gpu host",
|
|
"type": "query",
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"query": { "query": "label_values(gpu_utilization_percent, host)", "refId": "A" },
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"current": { "text": "All", "value": "$__all" }
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1, "type": "stat", "title": "Uptime",
|
|
"description": "Time since boot. A drop means a reboot.",
|
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
|
|
"targets": [{ "expr": "time() - node_boot_time_seconds{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 2, "type": "stat", "title": "Memory used",
|
|
"description": "1 - MemAvailable/MemTotal. The honest \"in use\" figure (excludes reclaimable cache).",
|
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.8 }, { "color": "red", "value": 0.92 }] } }, "overrides": [] },
|
|
"targets": [{ "expr": "1 - node_memory_MemAvailable_bytes{job=~\"$node\"} / node_memory_MemTotal_bytes{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 3, "type": "stat", "title": "Load 5m / core",
|
|
"description": "Per-core load5. >1 sustained means runnable tasks are queuing behind cores.",
|
|
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 1 }] } }, "overrides": [] },
|
|
"targets": [{ "expr": "node_load5{job=~\"$node\"} / count by (job) (node_cpu_seconds_total{job=~\"$node\",mode=\"idle\"})", "legendFormat": "{{job}}", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 4, "type": "stat", "title": "Open file handles",
|
|
"description": "FDs allocated vs the kernel maximum. Rising toward the max precedes EMFILE across the host.",
|
|
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 }] } }, "overrides": [] },
|
|
"targets": [{ "expr": "node_filefd_allocated{job=~\"$node\"} / node_filefd_maximum{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
|
},
|
|
{
|
|
"id": 5, "type": "timeseries", "title": "CPU busy %",
|
|
"description": "CPU seconds/s in each mode = cores busy per mode; the stack sums to total cores in use (16/host here). A user or system spike points at what is loading the box; steal > 0 means the hypervisor is throttling.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "sum by (job,mode) (rate(node_cpu_seconds_total{job=~\"$node\",mode!=\"idle\",mode!=\"iowait\"}[5m]))", "legendFormat": "{{job}} {{mode}}", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 6, "type": "timeseries", "title": "Load average",
|
|
"description": "load1/5/15 per host.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "node_load1{job=~\"$node\"}", "legendFormat": "{{job}} load1", "refId": "A" },
|
|
{ "expr": "node_load5{job=~\"$node\"}", "legendFormat": "{{job}} load5", "refId": "B" },
|
|
{ "expr": "node_load15{job=~\"$node\"}", "legendFormat": "{{job}} load15", "refId": "C" }
|
|
]
|
|
},
|
|
{
|
|
"id": 7, "type": "timeseries", "title": "Memory breakdown",
|
|
"description": "Total, free, buffers+cache. Reclaimable cache is not pressure; MemAvailable is the real headroom.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "node_memory_MemTotal_bytes{job=~\"$node\"} - node_memory_MemAvailable_bytes{job=~\"$node\"}", "legendFormat": "{{job}} used", "refId": "A" },
|
|
{ "expr": "node_memory_MemAvailable_bytes{job=~\"$node\"}", "legendFormat": "{{job}} available", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 8, "type": "timeseries", "title": "Filesystem free %",
|
|
"description": "Per mountpoint. The media/library stacks fill disks.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.15 }] } }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "node_filesystem_avail_bytes{job=~\"$node\",fstype!~\"tmpfs|overlay|squashfs\"} / node_filesystem_size_bytes{job=~\"$node\",fstype!~\"tmpfs|overlay|squashfs\"}", "legendFormat": "{{job}} {{mountpoint}}", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 9, "type": "timeseries", "title": "Disk throughput",
|
|
"description": "Read/write bytes/s per device.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "sum by (job,device) (rate(node_disk_read_bytes_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} R", "refId": "A" },
|
|
{ "expr": "-sum by (job,device) (rate(node_disk_written_bytes_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} W", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 10, "type": "timeseries", "title": "Disk busy %",
|
|
"description": "Fraction of time the device had I/O in flight. Near 1 = the device is the bottleneck.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "rate(node_disk_io_time_seconds_total{job=~\"$node\"}[5m])", "legendFormat": "{{job}} {{device}}", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 11, "type": "timeseries", "title": "Network throughput",
|
|
"description": "Receive (up) / transmit (down) bytes/s per interface.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "sum by (job,device) (rate(node_network_receive_bytes_total{job=~\"$node\",device!~\"lo|veth.*|br.*|docker.*\"}[5m]))", "legendFormat": "{{job}} {{device}} RX", "refId": "A" },
|
|
{ "expr": "-sum by (job,device) (rate(node_network_transmit_bytes_total{job=~\"$node\",device!~\"lo|veth.*|br.*|docker.*\"}[5m]))", "legendFormat": "{{job}} {{device}} TX", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 12, "type": "timeseries", "title": "Network errors / drops",
|
|
"description": "Interface errors and drops \u2014 the signature of a saturated link or a bad NIC/cable.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "sum by (job,device) (rate(node_network_receive_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} RX err", "refId": "A" },
|
|
{ "expr": "sum by (job,device) (rate(node_network_transmit_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} TX err", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 13, "type": "timeseries", "title": "GPU busy %",
|
|
"description": "Per-GPU utilization from gpu-exporter (amdgpu gpu_busy_percent / nvidia-smi). The picker above selects the host; each series is one card.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "gpu_utilization_percent{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 14, "type": "timeseries", "title": "GPU VRAM used",
|
|
"description": "Device memory in use. On the R9700s (frank-archon) and the 3090 (frank-zealot) this is the loaded model's footprint.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "bytes", "min": 0 }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "gpu_memory_used_bytes{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 15, "type": "timeseries", "title": "GPU VRAM used %",
|
|
"description": "Used / total device memory. Near 100% is the pre-OOM wall.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "100 * gpu_memory_used_bytes{host=~\"$gpuhost\"} / gpu_memory_total_bytes{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 16, "type": "timeseries", "title": "GPU temperature",
|
|
"description": "Package temperature. AMD edge temp (hwmon temp1_input); NVIDIA gpu temp.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "celsius", "min": 0 }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "gpu_temperature_celsius{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 17, "type": "timeseries", "title": "GPU power draw",
|
|
"description": "Board power draw. Emitted only where readable: NVIDIA and the Dankclaw iGPU. The R9700s have no sysfs power sensor, so they are simply absent here — never a fabricated 0.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "watt", "min": 0 }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "gpu_power_watts{host=~\"$gpuhost\"}", "legendFormat": "{{host}} {{gpu}} ({{model}})", "refId": "A" }
|
|
]
|
|
}
|
|
]
|
|
}
|