Frank
af05c890fd
The scraped metrics have no rules over them, so every graph was read after the
fact by a human. This adds shared/prometheus/rules/llm-fleet.yml, loaded via
rule_files and installed by the same copier that ships prometheus.yml (a rule
file nothing loads evaluates nothing, and Prometheus reports no error for it):
LlmKvPoolSaturated kv pool > 90% for 10m — the 122,636-token wall
LlmRequestsQueueing waiting > 0 for 10m — MAXSEQS=4 reached
LlmPrefixCacheCollapse reuse < 20% for 15m with traffic — the thrash signature
LlmEndpointDown scrape target down 2m — both replicas gone
LlmLatencyHigh p95 end-to-end > 60s for 10m — what an operator feels
job:llm_prefix_cache_reuse:rate5m — recording rule shared by the alert and the
dashboard, so the two cannot drift
Every threshold is tied to something measured on this fleet, not to a guess,
and the file says plainly at the top that **nothing is notified**: there is no
Alertmanager here, so a firing alert reaches Prometheus' /alerts page, the
ALERTS metric and the dashboard panel added alongside it — and nowhere else.
Choosing how Frank actually gets paged is his call, not a missing config line.
Seam covers the silent failure mode: rules exist, the config references the
directory, the deployer installs it, and the dashboard graphs ALERTS.
263 lines
6.9 KiB
JSON
263 lines
6.9 KiB
JSON
{
|
|
"uid": "llm-fleet",
|
|
"title": "LLM fleet — capacity and reuse",
|
|
"description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-large (llama.cpp) has throughput counters only. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.",
|
|
"tags": [
|
|
"llm",
|
|
"frank-medium",
|
|
"capacity"
|
|
],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "job",
|
|
"label": "endpoint",
|
|
"type": "query",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"query": {
|
|
"query": "label_values(vllm:kv_cache_usage_perc, job)",
|
|
"refId": "A"
|
|
},
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"current": {
|
|
"text": "All",
|
|
"value": "$__all"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "timeseries",
|
|
"title": "KV pool fill (vllm:kv_cache_usage_perc)",
|
|
"description": "How full each card's KV pool is. One card holds ~122,636 tokens; sustained high values with a cold prefix cache is the thrash this fleet measured.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "vllm:kv_cache_usage_perc{job=~\"$job\"} * 100",
|
|
"legendFormat": "{{job}} engine {{engine}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "timeseries",
|
|
"title": "Requests waiting (queue depth)",
|
|
"description": "Anything above zero means the engine is serialising: with MAXSEQS=4 the fifth concurrent request waits.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "vllm:num_requests_waiting{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} waiting"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "vllm:num_requests_running{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} running"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "timeseries",
|
|
"title": "Prefix cache reuse rate",
|
|
"description": "rate(hits)/rate(queries). High means pinned conversations are being served from cache (measured: 22.3 s cold vs 1.4 s warm on a ~50k-token prefix). A collapse here with three concurrent large contexts is the capacity wall, not a routing failure.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "rate(vllm:prefix_cache_hits_total{job=~\"$job\"}[5m]) / clamp_min(rate(vllm:prefix_cache_queries_total{job=~\"$job\"}[5m]), 1)",
|
|
"legendFormat": "{{job}} reuse"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "timeseries",
|
|
"title": "End-to-end request latency p95",
|
|
"description": "Wall-clock per request, prefill included — the number an agent-loop turn feels.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "histogram_quantile(0.95, sum by (le, job) (rate(vllm:e2e_request_latency_seconds_bucket{job=~\"$job\"}[5m])))",
|
|
"legendFormat": "{{job}} p95"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "timeseries",
|
|
"title": "Decode and prefill throughput",
|
|
"description": "vLLM generation tokens, and llama.cpp's counters for frank-large (llamacpp:prompt_tokens_seconds is prompt throughput).",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "tokens/sec",
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "rate(vllm:generation_tokens_total{job=~\"$job\"}[5m])",
|
|
"legendFormat": "{{job}} generated tok/s"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "rate(vllm:prompt_tokens_total{job=~\"$job\"}[5m])",
|
|
"legendFormat": "{{job}} prompt tok/s"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"expr": "llamacpp:predicted_tokens_seconds",
|
|
"legendFormat": "{{job}} llama.cpp decode"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "timeseries",
|
|
"title": "Alerts firing (ALERTS)",
|
|
"description": "Prometheus rules for the LLM fleet (shared/prometheus/rules/llm-fleet.yml): pool saturation, queueing, prefix-cache collapse, endpoint down, p95 latency. A firing alert appears here, on Prometheus' /alerts page, and nowhere else — this stack has no Alertmanager, so nothing is notified.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 16
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "ALERTS{alertstate=\"firing\"}",
|
|
"legendFormat": "{{alertname}} {{job}}"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|