serverAppConfigs/shared/grafana/provisioning/dashboards/llm-fleet.json

{
  "uid": "llm-fleet",
  "title": "LLM fleet — capacity and reuse",
  "description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-large (llama.cpp) has throughput counters only. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.",
  "tags": ["llm", "frank-medium", "capacity"],
  "timezone": "browser",
  "schemaVersion": 39,
  "version": 1,
  "refresh": "30s",
  "time": { "from": "now-6h", "to": "now" },
  "templating": {
    "list": [
      {
        "name": "job",
        "label": "endpoint",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "prometheus" },
        "query": { "query": "label_values(vllm:kv_cache_usage_perc, job)", "refId": "A" },
        "includeAll": true,
        "multi": true,
        "current": { "text": "All", "value": "$__all" }
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "timeseries",
      "title": "KV pool fill (vllm:kv_cache_usage_perc)",
      "description": "How full each card's KV pool is. One card holds ~122,636 tokens; sustained high values with a cold prefix cache is the thrash this fleet measured.",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "min": 0,
          "max": 1,
          "custom": { "fillOpacity": 10, "lineWidth": 1 }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "vllm:kv_cache_usage_perc{job=~\"$job\"} * 100",
          "legendFormat": "{{job}} engine {{engine}}"
        }
      ]
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "Requests waiting (queue depth)",
      "description": "Anything above zero means the engine is serialising: with MAXSEQS=4 the fifth concurrent request waits.",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": {
        "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1 } },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "vllm:num_requests_waiting{job=~\"$job\"}",
          "legendFormat": "{{job}} waiting"
        },
        {
          "refId": "B",
          "expr": "vllm:num_requests_running{job=~\"$job\"}",
          "legendFormat": "{{job}} running"
        }
      ]
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "Prefix cache reuse rate",
      "description": "rate(hits)/rate(queries). High means pinned conversations are being served from cache (measured: 22.3 s cold vs 1.4 s warm on a ~50k-token prefix). A collapse here with three concurrent large contexts is the capacity wall, not a routing failure.",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": {
        "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 10, "lineWidth": 1 } },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "rate(vllm:prefix_cache_hits_total{job=~\"$job\"}[5m]) / clamp_min(rate(vllm:prefix_cache_queries_total{job=~\"$job\"}[5m]), 1)",
          "legendFormat": "{{job}} reuse"
        }
      ]
    },
    {
      "id": 4,
      "type": "timeseries",
      "title": "End-to-end request latency p95",
      "description": "Wall-clock per request, prefill included — the number an agent-loop turn feels.",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": {
        "defaults": { "unit": "s", "custom": { "fillOpacity": 10, "lineWidth": 1 } },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "histogram_quantile(0.95, sum by (le, job) (rate(vllm:e2e_request_latency_seconds_bucket{job=~\"$job\"}[5m])))",
          "legendFormat": "{{job}} p95"
        }
      ]
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "Decode and prefill throughput",
      "description": "vLLM generation tokens, and llama.cpp's counters for frank-large (llamacpp:prompt_tokens_seconds is prompt throughput).",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": {
        "defaults": { "unit": "tokens/sec", "custom": { "fillOpacity": 10, "lineWidth": 1 } },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "rate(vllm:generation_tokens_total{job=~\"$job\"}[5m])",
          "legendFormat": "{{job}} generated tok/s"
        },
        {
          "refId": "B",
          "expr": "rate(vllm:prompt_tokens_total{job=~\"$job\"}[5m])",
          "legendFormat": "{{job}} prompt tok/s"
        },
        {
          "refId": "C",
          "expr": "llamacpp:predicted_tokens_seconds",
          "legendFormat": "{{job}} llama.cpp decode"
        }
      ]
    }
  ]
}