feat(observability): per-container metrics via cadvisor + Containers dashboard
Per-container CPU/memory was invisible for the quadlet app stack — Grafana could only see the host (node_exporter) and the LLM engines. This adds a rootless cadvisor quadlet on each host and a `containers` dashboard. cadvisor works on this rootless-Podman / cgroup-v2 host on one condition: --cgroupns=host. Podman starts every container in a private cgroup namespace, so without it cadvisor sees only its own cgroup root and exports a single series (measured 2026-10-20: 1 series off, 996 on). The libpod socket does not name the rootless app containers (it resolved 2 of 34), so naming comes from elsewhere: the quadlet unit name is carried in the cgroup path (app.slice/<name>.service/libpod-payload-<hex>), and Prometheus relabels it into a `container` label. No podman socket needed. cadvisor cannot bind per-interface like node_exporter, so each unit is a bridge container publishing exactly 127.0.0.1:8080 and the host LAN address — never a wildcard. Prometheus (a bridge container) reaches a host-published port at the LAN address, so cadvisor-dankclaw/dankserver scrape 192.168.0.143:8080 / 192.168.0.235:8080. Only the families cadvisor actually exports for these containers are graphed — CPU usage, memory working-set/limit, OOM — not network or filesystem, which are absent under the host-PID layout. Wired through the deployer (allowlist, deploy_observability case + /healthz gate, dispatcher), the planner (both targets), and observability_quadlet.bats.
This commit is contained in:
Frank
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"uid": "containers",
|
||||
"title": "Containers — Dankclaw & Dankserver",
|
||||
"description": "Per-container CPU and memory for the quadlet app stacks, from cadvisor (a rootless quadlet on each host). cadvisor keys each series by the cgroup path, and Prometheus relabels the quadlet unit name (app.slice/<name>.service) into a `container` label. Panels use only the metric families cadvisor actually exports for these rootless containers: CPU usage and memory working-set/limit/OOM. Network and filesystem are NOT exported for them (no netns/classid attribution under the rootless+host-PID layout), so no panels pretend otherwise. See serverAppConfigs-docs/Infrastructure/Observability.md.",
|
||||
"tags": ["fleet", "cadvisor", "containers"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "host",
|
||||
"label": "Host",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\"}, host)", "refId": "host_var" },
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": { "text": "All", "value": "$__all" }
|
||||
},
|
||||
{
|
||||
"name": "container",
|
||||
"label": "Container",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\"}, container)", "refId": "container_var" },
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": { "text": "All", "value": "$__all" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "CPU",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "CPU usage (cores)",
|
||||
"description": "CPU-seconds/s per container = cores busy. A steady high line is a busy service; a spike is a burst (a render, an index, a backup). Only containers are shown, not the host or its daemons.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
|
||||
"legendFormat": "{{host}} / {{container}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "bargauge",
|
||||
"title": "Top containers by CPU (cores)",
|
||||
"description": "Instant ranking over the selected window. Tells you which container to look at when the host is loaded.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": null, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 4 } ] } }, "overrides": [] },
|
||||
"options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "topk(12, sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m])))",
|
||||
"legendFormat": "{{host}} / {{container}}",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "row",
|
||||
"title": "Memory",
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 },
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Memory working set",
|
||||
"description": "container_memory_working_set_bytes per container — the number the OOM killer acts on. Cache-inclusive and reclaimable; a flat high line is resident data, a sawtooth is churn.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 11 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}",
|
||||
"legendFormat": "{{host}} / {{container}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Memory vs limit (%)",
|
||||
"description": "Working set over the cgroup memory limit (container_spec_memory_limit_bytes). Only containers that carry a limit are shown; approaching 1.0 means an OOM is near. Containers with no limit are omitted.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 11 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} / on (job, host, container) group_left container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}) and (container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} > 0)",
|
||||
"legendFormat": "{{host}} / {{container}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "OOM kills",
|
||||
"description": "container_oom_events_total rate. Any non-zero line means the kernel killed a process inside that container — the honest signal that its memory limit is too tight for its real working set.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 20, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (host, container) (rate(container_oom_events_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
|
||||
"legendFormat": "{{host}} / {{container}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -131,3 +131,63 @@ scrape_configs:
|
||||
- targets: ["192.168.0.235:7770"]
|
||||
labels:
|
||||
service: frank-archon
|
||||
|
||||
# Per-container metrics from cadvisor (a rootless quadlet on each host).
|
||||
# cadvisor keys every series by the cgroup path; for a quadlet container that
|
||||
# path is …/app.slice/<name>.service[/libpod-payload-<hex>], so the unit NAME
|
||||
# lives in it. The relabel below pulls that name out into a `container` label
|
||||
# and keeps only the per-unit aggregate — cadvisor's raw host root,
|
||||
# system.slice daemons and machine.slice scopes carry no such name and are
|
||||
# dropped, as is the per-CPU-core split (only cpu="total" survives), so the
|
||||
# scrape stays small. cadvisor cannot bind per-interface, so it is reached at
|
||||
# the host's PUBLISHED LAN port: unlike node-exporter's host-net bind (which
|
||||
# is why node-dankclaw scrapes via host.containers.internal), a published port
|
||||
# IS reachable from this bridge container at the host LAN address.
|
||||
- job_name: "cadvisor-dankclaw"
|
||||
static_configs:
|
||||
- targets: ["192.168.0.143:8080"]
|
||||
labels:
|
||||
host: dankclaw
|
||||
metric_relabel_configs:
|
||||
- source_labels: [id]
|
||||
regex: '.*/app\.slice/([^.]+)\.service(?:/.*)?$'
|
||||
target_label: container
|
||||
replacement: '$1'
|
||||
# Keep only the unit-level cgroup; drop its payload/control children so a
|
||||
# container is not counted twice.
|
||||
- source_labels: [id]
|
||||
regex: '.*/\.service/.*'
|
||||
action: drop
|
||||
# Drop everything that is not a named unit (no `container` was extracted).
|
||||
- source_labels: [container]
|
||||
regex: '.+'
|
||||
action: keep
|
||||
# cpu="total" for cpu series; metrics with no cpu label (memory, network,
|
||||
# filesystem) are kept.
|
||||
- source_labels: [cpu]
|
||||
regex: '^$|total'
|
||||
action: keep
|
||||
- regex: 'id|name|image|container_label.*|machine_id|boot_id'
|
||||
action: labeldrop
|
||||
|
||||
- job_name: "cadvisor-dankserver"
|
||||
static_configs:
|
||||
- targets: ["192.168.0.235:8080"]
|
||||
labels:
|
||||
host: dankserver
|
||||
metric_relabel_configs:
|
||||
- source_labels: [id]
|
||||
regex: '.*/app\.slice/([^.]+)\.service(?:/.*)?$'
|
||||
target_label: container
|
||||
replacement: '$1'
|
||||
- source_labels: [id]
|
||||
regex: '.*/\.service/.*'
|
||||
action: drop
|
||||
- source_labels: [container]
|
||||
regex: '.+'
|
||||
action: keep
|
||||
- source_labels: [cpu]
|
||||
regex: '^$|total'
|
||||
action: keep
|
||||
- regex: 'id|name|image|container_label.*|machine_id|boot_id'
|
||||
action: labeldrop
|
||||
|
||||
Reference in New Issue
Block a user