serverAppConfigs/shared/grafana/provisioning/dashboards/llm-fleet.json

{
  "uid": "llm-fleet",
  "title": "LLM fleet — capacity and reuse",
  "description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work. The panels read the endpoint's own counters: KV fill, queue depth, prefix-cache reuse, and request latency. frank-small (llama.cpp on Dankclaw's RTX 3090) exports throughput counters and appears in the same endpoint picker; frank-large runs the halogen engine, which publishes /health, /cache and /v1/* and no metrics endpoint at all, so it cannot appear here. See serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md for the numbers these panels are read against.",
  "tags": [
    "llm",
    "frank-medium",
    "capacity"
  ],
  "timezone": "browser",
  "schemaVersion": 39,
  "version": 1,
  "refresh": "30s",
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "templating": {
    "list": [
      {
        "name": "job",
        "label": "endpoint",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "prometheus"
        },
        "query": {
          "query": "label_values({__name__=~\"vllm:kv_cache_usage_perc|llamacpp:.*\"}, job)",
          "refId": "A"
        },
        "includeAll": true,
        "multi": true,
        "current": {
          "text": "All",
          "value": "$__all"
        }
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "timeseries",
      "title": "KV pool fill (vllm:kv_cache_usage_perc)",
      "description": "How full each card's KV pool is. One card holds ~122,636 tokens; sustained high values with a cold prefix cache is the thrash this fleet measured.",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "min": 0,
          "max": 1,
          "custom": {
            "fillOpacity": 10,
            "lineWidth": 1
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "vllm:kv_cache_usage_perc{job=~\"$job\"} * 100",
          "legendFormat": "{{job}} engine {{engine}}"
        }
      ]
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "Requests waiting (queue depth)",
      "description": "Anything above zero means the engine is serialising: with MAXSEQS=4 the fifth concurrent request waits.",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "min": 0,
          "custom": {
            "fillOpacity": 10,
            "lineWidth": 1
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "vllm:num_requests_waiting{job=~\"$job\"}",
          "legendFormat": "{{job}} waiting"
        },
        {
          "refId": "B",
          "expr": "vllm:num_requests_running{job=~\"$job\"}",
          "legendFormat": "{{job}} running"
        }
      ]
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "Prefix cache reuse rate",
      "description": "rate(hits)/rate(queries). High means pinned conversations are being served from cache (measured: 22.3 s cold vs 1.4 s warm on a ~50k-token prefix). A collapse here with three concurrent large contexts is the capacity wall, not a routing failure.",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "min": 0,
          "max": 1,
          "custom": {
            "fillOpacity": 10,
            "lineWidth": 1
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "rate(vllm:prefix_cache_hits_total{job=~\"$job\"}[5m]) / clamp_min(rate(vllm:prefix_cache_queries_total{job=~\"$job\"}[5m]), 1)",
          "legendFormat": "{{job}} reuse"
        }
      ]
    },
    {
      "id": 4,
      "type": "timeseries",
      "title": "End-to-end request latency p95",
      "description": "Wall-clock per request, prefill included — the number an agent-loop turn feels.",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "fillOpacity": 10,
            "lineWidth": 1
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "histogram_quantile(0.95, sum by (le, job) (rate(vllm:e2e_request_latency_seconds_bucket{job=~\"$job\"}[5m])))",
          "legendFormat": "{{job}} p95"
        }
      ]
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "Decode and prefill throughput",
      "description": "vLLM generation tokens, plus llama.cpp's counters for frank-small: llamacpp:predicted_tokens_seconds is decode and llamacpp:prompt_tokens_seconds is prompt throughput. The llama.cpp series carry the job label of their scrape, so the endpoint picker selects between the runtimes.",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 16
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "tokens/sec",
          "custom": {
            "fillOpacity": 10,
            "lineWidth": 1
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "rate(vllm:generation_tokens_total{job=~\"$job\"}[5m])",
          "legendFormat": "{{job}} generated tok/s"
        },
        {
          "refId": "B",
          "expr": "rate(vllm:prompt_tokens_total{job=~\"$job\"}[5m])",
          "legendFormat": "{{job}} prompt tok/s"
        },
        {
          "refId": "C",
          "expr": "llamacpp:predicted_tokens_seconds{job=~\"$job\"}",
          "legendFormat": "{{job}} llama.cpp decode tok/s"
        },
        {
          "refId": "D",
          "expr": "llamacpp:prompt_tokens_seconds{job=~\"$job\"}",
          "legendFormat": "{{job}} llama.cpp prefill tok/s"
        }
      ]
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "Alerts firing (ALERTS)",
      "description": "Prometheus rules for the LLM fleet (shared/prometheus/rules/llm-fleet.yml): pool saturation, queueing, prefix-cache collapse, endpoint down, p95 latency. A firing alert appears here, on Prometheus' /alerts page, and nowhere else — this stack has no Alertmanager, so nothing is notified.",
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 16
      },
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "min": 0,
          "custom": {
            "fillOpacity": 10,
            "lineWidth": 1
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "ALERTS{alertstate=\"firing\"}",
          "legendFormat": "{{alertname}} {{job}}"
        }
      ]
    }
  ]
}