Frank
24742c75b7
frank-archon (radiance, Dankserver) emits vllm:* + radiance:* (job=frank-archon). llm-fleet.json queried only llamacpp:* and its endpoint picker selected via label_values(llamacpp:.*), so archon - despite being scraped and alert-on (vllm-fleet group) - had zero panels. The dashboard description also still claimed archon was llama.cpp, which is stale. - Fix the description: two namespaces; the $job picker is the llama.cpp pair (dragoon/zealot); archon is radiance and gets its own section. - Add a frank-archon row + 8 panels: vllm:* throughput, request outcomes by finish reason, queue/preemption, TTFT p50/p95, and the radiance natives vllm omits - decode/prefill t/s, MTP draft acceptance, engine-step timing, prefix hit. All 14 queries verified live against Prometheus (non-empty). Dashboards-only; the next grafana deploy provisions it.
310 lines
13 KiB
JSON
310 lines
13 KiB
JSON
{
|
|
"uid": "llm-fleet",
|
|
"title": "LLM fleet \u2014 capacity and reuse",
|
|
"description": "The signals that explain the LLM endpoints under load, in TWO namespaces. Panels 1-6 are llama.cpp (llamacpp:*) and the endpoint picker ($job) selects frank-zealot (Dankclaw, RTX 3090) and frank-dragoon (Dankclaw, gfx1151 iGPU). frank-archon (Dankserver, 2x R9700) is radiance, NOT llama.cpp: it emits vllm:* and radiance:* (job=frank-archon) and has its own section at the bottom - those families are alert-on but were never graphed here until now. See serverAppConfigs-docs/Infrastructure/Observability.md.",
|
|
"tags": [
|
|
"llm",
|
|
"capacity"
|
|
],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 2,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "job",
|
|
"label": "endpoint",
|
|
"type": "query",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"query": {
|
|
"query": "label_values({__name__=~\"llamacpp:.*\"}, job)",
|
|
"refId": "A"
|
|
},
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"current": {
|
|
"text": "All",
|
|
"value": "$__all"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "timeseries",
|
|
"title": "Prefix cache reuse rate",
|
|
"description": "rate(hits)/rate(queries) in tokens. The llama.cpp prompt counter EXCLUDES cached tokens (its own HELP line), so this divides by cached + prompt. High means pinned conversations are served from cache: measured 2026-09-20, an identical 13,021-token prompt went 2.30 s cold to 0.68 s warm with 9,888 tokens (76 %) reused.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(llamacpp:prompt_tokens_cached_total{job=~\"$job\"}[5m]) / clamp_min(rate(llamacpp:prompt_tokens_cached_total{job=~\"$job\"}[5m]) + rate(llamacpp:prompt_tokens_total{job=~\"$job\"}[5m]), 0.001)",
|
|
"legendFormat": "{{job}} reuse (llama.cpp)",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "timeseries",
|
|
"title": "Speculative decoding (acceptance)",
|
|
"description": "The one failure no client can see: the drafter silently stops and the endpoint keeps answering at autoregressive speed. Accepted tokens per draft step, and the mean accepted length (1 + accepted/drafts, the recording rule job:llm_spec_accept_length:rate5m) \u2014 1.0 means the draft round is pure overhead.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(llamacpp:spec_decode_num_accepted_tokens_total{job=~\"$job\"}[5m]) / clamp_min(rate(llamacpp:spec_decode_num_drafts_total{job=~\"$job\"}[5m]), 0.001)",
|
|
"legendFormat": "{{job}} accepted per draft step",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "job:llm_spec_accept_length:rate5m{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} mean accepted length",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "rate(llamacpp:spec_decode_num_draft_tokens_total{job=~\"$job\"}[5m])",
|
|
"legendFormat": "{{job}} draft tokens/s",
|
|
"refId": "C"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "timeseries",
|
|
"title": "Requests waiting (queue depth)",
|
|
"description": "llama.cpp: anything above zero means the engine is serialising (with --parallel 1 a second concurrent request always shows up here). All three endpoints are llama.cpp since the 2026-09-26 frank-archon swap.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "llamacpp:requests_deferred{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} deferred",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "llamacpp:requests_processing{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} processing",
|
|
"refId": "B"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "timeseries",
|
|
"title": "Decode and prefill throughput",
|
|
"description": "llamacpp:predicted_tokens_seconds is decode, llamacpp:prompt_tokens_seconds is prompt. The series carry the job label of their scrape, so the endpoint picker selects between the endpoints.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "llamacpp:predicted_tokens_seconds{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} decode tok/s (llama.cpp)",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "llamacpp:prompt_tokens_seconds{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} prefill tok/s (llama.cpp)",
|
|
"refId": "B"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "timeseries",
|
|
"title": "Decode steps and batch size",
|
|
"description": "Per-step counters: steps per second and the busy-slot count per step (the effective batch). With frank-archon's --parallel 1 the slot count stays at 0 or 1 and says nothing about the card; with frank-zealot's 4 slots it tracks the agent-fleet burst shape.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 16
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(llamacpp:n_decode_total{job=~\"$job\"}[5m])",
|
|
"legendFormat": "{{job}} decode steps/s",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "llamacpp:n_busy_slots_per_decode{job=~\"$job\"}",
|
|
"legendFormat": "{{job}} sequences per decode step",
|
|
"refId": "B"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "timeseries",
|
|
"title": "Firing fleet alerts",
|
|
"description": "The LLM-fleet rules from shared/prometheus/rules/llm-fleet.yml. A firing alert is delivered to Signal: Prometheus -> Alertmanager (dankclaw quadlet) -> the host-side signal-relay user unit -> signal-cli, since 2026-09-17.",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "ALERTS{alertname=~\"Llm.*\", alertstate=\"firing\"}",
|
|
"legendFormat": "{{alertname}} {{job}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 100,
|
|
"type": "row",
|
|
"title": "frank-archon \u2014 radiance (vllm:* + radiance:*)",
|
|
"collapsed": false,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 },
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 101, "type": "timeseries", "title": "Archon throughput (tokens/s)",
|
|
"description": "vLLM-compat counters: prompt and generation token rates for frank-archon.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1 } }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "sum(rate(vllm:prompt_tokens_total{job=\"frank-archon\"}[5m]))", "legendFormat": "prompt/s", "refId": "A" },
|
|
{ "expr": "sum(rate(vllm:generation_tokens_total{job=\"frank-archon\"}[5m]))", "legendFormat": "generation/s", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 102, "type": "timeseries", "title": "Archon request outcomes",
|
|
"description": "Completed requests/s by finish reason (stop / length / tool_calls / abort).",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [
|
|
{ "expr": "sum by (finished_reason)(rate(vllm:request_success_total{job=\"frank-archon\"}[5m]))", "legendFormat": "{{finished_reason}}", "refId": "A" }
|
|
]
|
|
},
|
|
{
|
|
"id": 103, "type": "timeseries", "title": "Archon queue and preemptions",
|
|
"description": "num_requests_waiting is the queue gauge (VllmRequestsQueueing fires on >0); preemption rate (VllmPreempting).",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [
|
|
{ "expr": "vllm:num_requests_waiting{job=\"frank-archon\"}", "legendFormat": "waiting", "refId": "A" },
|
|
{ "expr": "sum(rate(vllm:num_preemptions_total{job=\"frank-archon\"}[5m]))", "legendFormat": "preemptions/s", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 104, "type": "timeseries", "title": "Archon time-to-first-token",
|
|
"description": "TTFT p50/p95 from the vLLM histogram. VllmTtftSlow alerts when the p95 recording rule exceeds 120 s.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "s", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1 } }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "histogram_quantile(0.5, sum by (le)(rate(vllm:time_to_first_token_seconds_bucket{job=\"frank-archon\"}[5m])))", "legendFormat": "p50", "refId": "A" },
|
|
{ "expr": "histogram_quantile(0.95, sum by (le)(rate(vllm:time_to_first_token_seconds_bucket{job=\"frank-archon\"}[5m])))", "legendFormat": "p95", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 105, "type": "timeseries", "title": "Archon radiance decode/prefill t/s",
|
|
"description": "radiance-native instantaneous throughput gauges - the engine's own view, not the vLLM counter derivative.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 50 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [
|
|
{ "expr": "radiance:decode_tokens_per_second{job=\"frank-archon\"}", "legendFormat": "decode t/s", "refId": "A" },
|
|
{ "expr": "radiance:prefill_tokens_per_second{job=\"frank-archon\"}", "legendFormat": "prefill t/s", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 106, "type": "timeseries", "title": "Archon speculative decode (MTP)",
|
|
"description": "radiance draft acceptance - the native signal vLLM metrics omit. The whole reason radiance exposes its own endpoint.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 50 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 10, "lineWidth": 1 } }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "radiance:draft_acceptance_ratio{job=\"frank-archon\"}", "legendFormat": "acceptance ratio", "refId": "A" },
|
|
{ "expr": "sum(rate(radiance:draft_accepted_total{job=\"frank-archon\"}[5m])) / clamp_min(sum(rate(radiance:draft_tokens_total{job=\"frank-archon\"}[5m])), 0.001)", "legendFormat": "accepted/token", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 107, "type": "timeseries", "title": "Archon engine step time",
|
|
"description": "radiance engine-step accounting: mean decode step duration (seconds per step) plus prefill steps/s.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 58 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "s", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1 } }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "rate(radiance:engine_step_seconds_decode_total{job=\"frank-archon\"}[5m]) / clamp_min(rate(radiance:engine_steps_decode_total{job=\"frank-archon\"}[5m]), 0.001)", "legendFormat": "avg decode step", "refId": "A" },
|
|
{ "expr": "sum(rate(radiance:engine_steps_prefill_total{job=\"frank-archon\"}[5m]))", "legendFormat": "prefill steps/s", "refId": "B" }
|
|
]
|
|
},
|
|
{
|
|
"id": 108, "type": "timeseries", "title": "Archon prefix cache hit rate",
|
|
"description": "vLLM and radiance prefix-cache hit rates. A collapse fires VllmPrefixCacheCollapse.",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 58 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 10, "lineWidth": 1 } }, "overrides": [] },
|
|
"targets": [
|
|
{ "expr": "vllm:gpu_prefix_cache_hit_rate{job=\"frank-archon\"}", "legendFormat": "vllm prefix hit", "refId": "A" },
|
|
{ "expr": "radiance:linear_prefix_cache_hit_rate{job=\"frank-archon\"}", "legendFormat": "radiance prefix hit", "refId": "B" }
|
|
]
|
|
}
|
|
]
|
|
}
|