{
  "uid": "observability",
  "title": "Observability \u2014 self-monitoring",
  "description": "The health of the stack that watches the fleet: Prometheus scrape/TSDB, Loki ingest, Grafana. If this is red the dashboards lie. Note: Alertmanager is running but NOT scraped by Prometheus (no job) \u2014 its metrics are absent by design gap, not fault.",
  "tags": [
    "observability",
    "meta"
  ],
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "30s",
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "panels": [
    {
      "id": 1,
      "type": "stat",
      "title": "Targets up",
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 0,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "count(up == 1)",
          "instant": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 2,
      "type": "stat",
      "title": "Targets down",
      "description": "Every scrape target that is down. ZERO IS THE HEALTHY STATE: the HTML-probe jobs and the disabled Gitea metrics were retired 2026-10-21 (HTTP liveness now rides the blackbox exporter, job \"services\"). A non-zero count here is a real regression - read /api/v1/targets lastError before triaging.",
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 4,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "count(up == 0)",
          "instant": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 3,
      "type": "stat",
      "title": "TSDB head series",
      "description": "Active series in memory. A jump = a new exporter or a label-cardinality leak.",
      "gridPos": {
        "h": 4,
        "w": 5,
        "x": 8,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "prometheus_tsdb_head_series",
          "instant": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 4,
      "type": "stat",
      "title": "Discovered targets",
      "gridPos": {
        "h": 4,
        "w": 5,
        "x": 13,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "sum(prometheus_sd_discovered_targets)",
          "instant": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 5,
      "type": "stat",
      "title": "Prometheus mem",
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 18,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "process_resident_memory_bytes{job=\"prometheus\"}",
          "instant": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 6,
      "type": "stat",
      "title": "Grafana up",
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 21,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "mappings": [
            {
              "type": "value",
              "options": {
                "1": {
                  "text": "UP",
                  "color": "green"
                },
                "0": {
                  "text": "DOWN",
                  "color": "red"
                }
              }
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "up{job=\"grafana\"}",
          "instant": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 7,
      "type": "table",
      "title": "Target health",
      "description": "Every scrape target and its current up. Sort by Value to surface regressions.",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 4
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "options": {
        "sortBy": [
          {
            "displayName": "Value",
            "sortOrder": "Asc"
          }
        ]
      },
      "targets": [
        {
          "expr": "up",
          "format": "table",
          "instant": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 8,
      "type": "timeseries",
      "title": "Scrape interval drift (p99 actual)",
      "description": "Actual seconds between scrapes (summary quantile). Above the 15s scrape_interval = Prometheus is falling behind its own cadence (slow endpoints or overload).",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 4
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}",
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 9,
      "type": "timeseries",
      "title": "Scrape samples/s",
      "description": "Total samples ingested across all jobs.",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 13
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "sum(rate(scrape_samples_scraped[5m]))",
          "legendFormat": "samples/s",
          "refId": "A"
        }
      ]
    },
    {
      "id": 10,
      "type": "timeseries",
      "title": "Loki request rate",
      "description": "Loki served requests/s (push, query). A drop to zero means ingest or query has stopped.",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 13
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))",
          "legendFormat": "{{route}}",
          "refId": "A"
        }
      ]
    },
    {
      "id": 11,
      "type": "timeseries",
      "title": "Exporter process CPU",
      "description": "CPU by monitored process \u2014 spot a runaway exporter or the Prometheus process itself.",
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 21
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit"
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))",
          "legendFormat": "{{job}}",
          "refId": "A"
        }
      ]
    }
  ]
}