serverAppConfigs/shared/grafana/provisioning/dashboards/containers.json

{
  "uid": "containers",
  "title": "Containers — Dankclaw & Dankserver",
  "description": "Per-container CPU and memory for the quadlet app stacks, from cadvisor (a rootless quadlet on each host). cadvisor keys each series by the cgroup path, and Prometheus relabels the quadlet unit name (app.slice/<name>.service) into a `container` label. Panels use only the metric families cadvisor actually exports for these rootless containers: CPU usage and memory working-set/limit/OOM. Network and filesystem are NOT exported for them (no netns/classid attribution under the rootless+host-PID layout), so no panels pretend otherwise. See serverAppConfigs-docs/Infrastructure/Observability.md.",
  "tags": ["fleet", "cadvisor", "containers"],
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "30s",
  "time": { "from": "now-6h", "to": "now" },
  "templating": {
    "list": [
      {
        "name": "host",
        "label": "Host",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "prometheus" },
        "query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\"}, host)", "refId": "host_var" },
        "refresh": 2,
        "includeAll": true,
        "multi": true,
        "current": { "text": "All", "value": "$__all" }
      },
      {
        "name": "container",
        "label": "Container",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "prometheus" },
        "query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\"}, container)", "refId": "container_var" },
        "refresh": 2,
        "includeAll": true,
        "multi": true,
        "current": { "text": "All", "value": "$__all" }
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "row",
      "title": "CPU",
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
      "panels": []
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "CPU usage (cores)",
      "description": "CPU-seconds/s per container = cores busy. A steady high line is a busy service; a spike is a burst (a render, an index, a backup). Only containers are shown, not the host or its daemons.",
      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 1 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
      "targets": [
        {
          "expr": "sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
          "legendFormat": "{{host}} / {{container}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 3,
      "type": "bargauge",
      "title": "Top containers by CPU (cores)",
      "description": "Instant ranking over the selected window. Tells you which container to look at when the host is loaded.",
      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": null, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 4 } ] } }, "overrides": [] },
      "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true },
      "targets": [
        {
          "expr": "topk(12, sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m])))",
          "legendFormat": "{{host}} / {{container}}",
          "format": "table",
          "instant": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 4,
      "type": "row",
      "title": "Memory",
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 },
      "panels": []
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "Memory working set",
      "description": "container_memory_working_set_bytes per container — the number the OOM killer acts on. Cache-inclusive and reclaimable; a flat high line is resident data, a sawtooth is churn.",
      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 11 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "bytes", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
      "targets": [
        {
          "expr": "container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}",
          "legendFormat": "{{host}} / {{container}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "Memory vs host total (%)",
      "description": "Working set over the host's total RAM. The fleet sets NO cgroup memory limits (every container_spec_memory_limit_bytes is 0 — a limit-share panel is structurally empty here), so the honest pressure view is share of the host. The join labels the node exporter's MemTotal onto the cadvisor series by host.",
      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 11 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
      "targets": [
        {
          "expr": "container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\", container!=\"\"} / on(host) group_left label_replace(node_memory_MemTotal_bytes{job=~\"node-.*\"}, \"host\", \"$1\", \"job\", \"node-(.*)\")",
          "legendFormat": "{{host}} / {{container}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "OOM kills",
      "description": "container_oom_events_total rate. Any non-zero line means the kernel killed a process inside that container — the honest signal that its memory limit is too tight for its real working set.",
      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 20, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] },
      "targets": [
        {
          "expr": "sum by (host, container) (rate(container_oom_events_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
          "legendFormat": "{{host}} / {{container}}",
          "refId": "A"
        }
      ]
    }
  ]
}