serverAppConfigs/shared/grafana/provisioning/dashboards/observability.json

{
  "uid": "observability",
  "title": "Observability \u2014 self-monitoring",
  "description": "The health of the stack that watches the fleet: Prometheus scrape/TSDB, Loki ingest, Grafana. If this is red the dashboards lie. Note: Alertmanager is running but NOT scraped by Prometheus (no job) \u2014 its metrics are absent by design gap, not fault.",
  "tags": ["observability", "meta"],
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "30s",
  "time": { "from": "now-6h", "to": "now" },
  "panels": [
    {
      "id": 1, "type": "stat", "title": "Targets up",
      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] },
      "targets": [{ "expr": "count(up == 1)", "instant": true, "refId": "A" }]
    },
    {
      "id": 2, "type": "stat", "title": "Targets down",
      "description": "Known structural downs: comfyui (no metrics endpoint), gitea (metrics off), and the homepage 'services' probes. Real regressions ride up from here too \u2014 check the table before dismissing.",
      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] } }, "overrides": [] },
      "targets": [{ "expr": "count(up == 0)", "instant": true, "refId": "A" }]
    },
    {
      "id": 3, "type": "stat", "title": "TSDB head series",
      "description": "Active series in memory. A jump = a new exporter or a label-cardinality leak.",
      "gridPos": { "h": 4, "w": 5, "x": 8, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
      "targets": [{ "expr": "prometheus_tsdb_head_series", "instant": true, "refId": "A" }]
    },
    {
      "id": 4, "type": "stat", "title": "Discovered targets",
      "gridPos": { "h": 4, "w": 5, "x": 13, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
      "targets": [{ "expr": "sum(prometheus_sd_discovered_targets)", "instant": true, "refId": "A" }]
    },
    {
      "id": 5, "type": "stat", "title": "Prometheus mem",
      "gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
      "targets": [{ "expr": "process_resident_memory_bytes{job=\"prometheus\"}", "instant": true, "refId": "A" }]
    },
    {
      "id": 6, "type": "stat", "title": "Grafana up",
      "gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "mappings": [{ "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }], "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] },
      "targets": [{ "expr": "up{job=\"grafana\"}", "instant": true, "refId": "A" }]
    },
    {
      "id": 7, "type": "table", "title": "Target health",
      "description": "Every scrape target and its current up. Sort by Value to surface regressions.",
      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 4 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "options": { "sortBy": [{ "displayName": "Value", "sortOrder": "Asc" }] },
      "targets": [{ "expr": "up", "format": "table", "instant": true, "refId": "A" }]
    },
    {
      "id": 8, "type": "timeseries", "title": "Scrape interval drift (p99 actual)",
      "description": "Actual seconds between scrapes (summary quantile). Above the 15s scrape_interval = Prometheus is falling behind its own cadence (slow endpoints or overload).",
      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 4 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
      "targets": [{ "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}", "legendFormat": "{{instance}}", "refId": "A" }]
    },
    {
      "id": 9, "type": "timeseries", "title": "Scrape samples/s",
      "description": "Total samples ingested across all jobs.",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
      "targets": [{ "expr": "sum(rate(scrape_samples_scraped[5m]))", "legendFormat": "samples/s", "refId": "A" }]
    },
    {
      "id": 10, "type": "timeseries", "title": "Loki request rate",
      "description": "Loki served requests/s (push, query). A drop to zero means ingest or query has stopped.",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
      "targets": [{ "expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))", "legendFormat": "{{route}}", "refId": "A" }]
    },
    {
      "id": 11, "type": "timeseries", "title": "Exporter process CPU",
      "description": "CPU by monitored process \u2014 spot a runaway exporter or the Prometheus process itself.",
      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 21 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] },
      "targets": [{ "expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))", "legendFormat": "{{job}}", "refId": "A" }]
    }
  ]
}