Frank
79cca19919
frank-large has no /metrics to scrape any more, and scraping it would be red by construction: the halogen engine publishes /health, /cache and /v1/* only. That job is removed with the reason written down (the remedy is the blackbox exporter this fleet already wants for the comfyui/services probes), and frank-small's llama.cpp endpoint takes the slot at 192.168.0.143:7771. Homepage: frank-large's card still described the ik runtime, and the new endpoint had no card at all. Both fixed, with a loopback probe -- the checker runs on Dankclaw, which is also what makes the card green-able. Dashboard: the endpoint picker was sourced from a vLLM-only metric, so a llama.cpp job could never be selected, and the prose named frank-large as the llama.cpp endpoint. The picker now matches both runtimes' series and the llama.cpp targets carry the job filter every other panel already used. The seams assert the new contract including the ABSENCE of a 7770 target, so a well-meaning edit cannot re-add a permanently-red probe.
268 lines
7.5 KiB
JSON
268 lines
7.5 KiB
JSON
{
|
|
"uid": "llm-fleet",
|
|
"title": "LLM fleet — capacity and reuse",
|
|
"description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-small (llama.cpp on Dankclaw's RTX 3090) exports throughput counters and appears in the same endpoint picker; frank-large runs the halogen engine, which publishes /health, /cache and /v1/* and no metrics endpoint at all, so it cannot appear here. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.",
|
|
"tags": [
|
|
"llm",
|
|
"frank-medium",
|
|
"capacity"
|
|
],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "job",
|
|
"label": "endpoint",
|
|
"type": "query",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"query": {
|
|
"query": "label_values({__name__=~\"vllm:kv_cache_usage_perc|llamacpp:.*\"}, job)",
|
|
"refId": "A"
|
|
},
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"current": {
|
|
"text": "All",
|
|
"value": "$__all"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "timeseries",
|
|
"title": "KV pool fill (vllm:kv_cache_usage_perc)",
|
|
"description": "How full each card's KV pool is. One card holds ~122,636 tokens; sustained high values with a cold prefix cache is the thrash this fleet measured.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "vllm:kv_cache_usage_perc{job=~\"$job\"} * 100",
|
|
"legendFormat": "{{job}} engine {{engine}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "timeseries",
|
|
"title": "Requests waiting (queue depth)",
|
|
"description": "Anything above zero means the engine is serialising: with MAXSEQS=4 the fifth concurrent request waits.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "vllm:num_requests_waiting{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} waiting"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "vllm:num_requests_running{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} running"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "timeseries",
|
|
"title": "Prefix cache reuse rate",
|
|
"description": "rate(hits)/rate(queries). High means pinned conversations are being served from cache (measured: 22.3 s cold vs 1.4 s warm on a ~50k-token prefix). A collapse here with three concurrent large contexts is the capacity wall, not a routing failure.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "rate(vllm:prefix_cache_hits_total{job=~\"$job\"}[5m]) / clamp_min(rate(vllm:prefix_cache_queries_total{job=~\"$job\"}[5m]), 1)",
|
|
"legendFormat": "{{job}} reuse"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "timeseries",
|
|
"title": "End-to-end request latency p95",
|
|
"description": "Wall-clock per request, prefill included — the number an agent-loop turn feels.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "histogram_quantile(0.95, sum by (le, job) (rate(vllm:e2e_request_latency_seconds_bucket{job=~\"$job\"}[5m])))",
|
|
"legendFormat": "{{job}} p95"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "timeseries",
|
|
"title": "Decode and prefill throughput",
|
|
"description": "vLLM generation tokens, plus llama.cpp's counters for frank-small: llamacpp:predicted_tokens_seconds is decode and llamacpp:prompt_tokens_seconds is prompt throughput. The llama.cpp series carry the job label of their scrape, so the endpoint picker selects between the runtimes.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "tokens/sec",
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "rate(vllm:generation_tokens_total{job=~\"$job\"}[5m])",
|
|
"legendFormat": "{{job}} generated tok/s"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "rate(vllm:prompt_tokens_total{job=~\"$job\"}[5m])",
|
|
"legendFormat": "{{job}} prompt tok/s"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"expr": "llamacpp:predicted_tokens_seconds{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} llama.cpp decode tok/s"
|
|
},
|
|
{
|
|
"refId": "D",
|
|
"expr": "llamacpp:prompt_tokens_seconds{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} llama.cpp prefill tok/s"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "timeseries",
|
|
"title": "Alerts firing (ALERTS)",
|
|
"description": "Prometheus rules for the LLM fleet (shared/prometheus/rules/llm-fleet.yml): pool saturation, queueing, prefix-cache collapse, endpoint down, p95 latency. A firing alert appears here, on Prometheus' /alerts page, and nowhere else — this stack has no Alertmanager, so nothing is notified.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 16
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "ALERTS{alertstate=\"firing\"}",
|
|
"legendFormat": "{{alertname}} {{job}}"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|