{
  "uid": "containers",
  "title": "Containers — Dankclaw & Dankserver",
  "description": "Per-container CPU and memory for the quadlet app stacks, from cadvisor (a rootless quadlet on each host). cadvisor keys each series by the cgroup path, and Prometheus relabels the quadlet unit name (app.slice/<name>.service) into a `container` label. Panels use only the metric families cadvisor actually exports for these rootless containers: CPU usage and memory working-set/limit/OOM. Network and filesystem are NOT exported for them (no netns/classid attribution under the rootless+host-PID layout), so no panels pretend otherwise. See serverAppConfigs-docs/Infrastructure/Observability.md.",
  "tags": ["fleet", "cadvisor", "containers"],
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "30s",
  "time": { "from": "now-6h", "to": "now" },
  "templating": {
    "list": [
      {
        "name": "host",
        "label": "Host",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "prometheus" },
        "query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\"}, host)", "refId": "host_var" },
        "refresh": 2,
        "includeAll": true,
        "multi": true,
        "current": { "text": "All", "value": "$__all" }
      },
      {
        "name": "container",
        "label": "Container",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "prometheus" },
        "query": { "query": "label_values(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\"}, container)", "refId": "container_var" },
        "refresh": 2,
        "includeAll": true,
        "multi": true,
        "current": { "text": "All", "value": "$__all" }
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "row",
      "title": "CPU",
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
      "panels": []
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "CPU usage (cores)",
      "description": "CPU-seconds/s per container = cores busy. A steady high line is a busy service; a spike is a burst (a render, an index, a backup). Only containers are shown, not the host or its daemons.",
      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 1 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
      "targets": [
        {
          "expr": "sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
          "legendFormat": "{{host}} / {{container}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 3,
      "type": "bargauge",
      "title": "Top containers by CPU (cores)",
      "description": "Instant ranking over the selected window. Tells you which container to look at when the host is loaded.",
      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": null, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 4 } ] } }, "overrides": [] },
      "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true },
      "targets": [
        {
          "expr": "topk(12, sum by (host, container) (rate(container_cpu_usage_seconds_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m])))",
          "legendFormat": "{{host}} / {{container}}",
          "format": "table",
          "instant": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 4,
      "type": "row",
      "title": "Memory",
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 },
      "panels": []
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "Memory working set",
      "description": "container_memory_working_set_bytes per container — the number the OOM killer acts on. Cache-inclusive and reclaimable; a flat high line is resident data, a sawtooth is churn.",
      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 11 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "bytes", "min": 0, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
      "targets": [
        {
          "expr": "container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}",
          "legendFormat": "{{host}} / {{container}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "Memory vs limit (%)",
      "description": "Working set over the cgroup memory limit (container_spec_memory_limit_bytes). Only containers that carry a limit are shown; approaching 1.0 means an OOM is near. Containers with no limit are omitted.",
      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 11 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 8, "lineWidth": 1, "stacking": { "mode": "none" } } }, "overrides": [] },
      "targets": [
        {
          "expr": "(container_memory_working_set_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} / on (job, host, container) group_left container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}) and (container_spec_memory_limit_bytes{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"} > 0)",
          "legendFormat": "{{host}} / {{container}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "OOM kills",
      "description": "container_oom_events_total rate. Any non-zero line means the kernel killed a process inside that container — the honest signal that its memory limit is too tight for its real working set.",
      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 20, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] },
      "targets": [
        {
          "expr": "sum by (host, container) (rate(container_oom_events_total{job=~\"cadvisor-.*\", host=~\"$host\", container=~\"$container\"}[5m]))",
          "legendFormat": "{{host}} / {{container}}",
          "refId": "A"
        }
      ]
    }
  ]
}