observability: LLM alerting rules, evaluated but not yet delivered
The scraped metrics have no rules over them, so every graph was read after the
fact by a human. This adds shared/prometheus/rules/llm-fleet.yml, loaded via
rule_files and installed by the same copier that ships prometheus.yml (a rule
file nothing loads evaluates nothing, and Prometheus reports no error for it):
LlmKvPoolSaturated kv pool > 90% for 10m — the 122,636-token wall
LlmRequestsQueueing waiting > 0 for 10m — MAXSEQS=4 reached
LlmPrefixCacheCollapse reuse < 20% for 15m with traffic — the thrash signature
LlmEndpointDown scrape target down 2m — both replicas gone
LlmLatencyHigh p95 end-to-end > 60s for 10m — what an operator feels
job:llm_prefix_cache_reuse:rate5m — recording rule shared by the alert and the
dashboard, so the two cannot drift
Every threshold is tied to something measured on this fleet, not to a guess,
and the file says plainly at the top that **nothing is notified**: there is no
Alertmanager here, so a firing alert reaches Prometheus' /alerts page, the
ALERTS metric and the dashboard panel added alongside it — and nowhere else.
Choosing how Frank actually gets paged is his call, not a missing config line.
Seam covers the silent failure mode: rules exist, the config references the
directory, the deployer installs it, and the dashboard graphs ALERTS.
This commit is contained in:
Frank committed
1 parent
c272407e6e
commit
af05c890fd
5 files changed
+275
-21
No files matched your search
@@ -2,23 +2,39 @@
|
||||
"uid": "llm-fleet",
|
||||
"title": "LLM fleet — capacity and reuse",
|
||||
"description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-large (llama.cpp) has throughput counters only. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.",
|
||||
"tags": ["llm", "frank-medium", "capacity"],
|
||||
"tags": [
|
||||
"llm",
|
||||
"frank-medium",
|
||||
"capacity"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "job",
|
||||
"label": "endpoint",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": { "query": "label_values(vllm:kv_cache_usage_perc, job)", "refId": "A" },
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"query": {
|
||||
"query": "label_values(vllm:kv_cache_usage_perc, job)",
|
||||
"refId": "A"
|
||||
},
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": { "text": "All", "value": "$__all" }
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -28,14 +44,25 @@
|
||||
"type": "timeseries",
|
||||
"title": "KV pool fill (vllm:kv_cache_usage_perc)",
|
||||
"description": "How full each card's KV pool is. One card holds ~122,636 tokens; sustained high values with a cold prefix cache is the thrash this fleet measured.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"custom": { "fillOpacity": 10, "lineWidth": 1 }
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@@ -52,10 +79,25 @@
|
||||
"type": "timeseries",
|
||||
"title": "Requests waiting (queue depth)",
|
||||
"description": "Anything above zero means the engine is serialising: with MAXSEQS=4 the fifth concurrent request waits.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1 } },
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
@@ -76,10 +118,26 @@
|
||||
"type": "timeseries",
|
||||
"title": "Prefix cache reuse rate",
|
||||
"description": "rate(hits)/rate(queries). High means pinned conversations are being served from cache (measured: 22.3 s cold vs 1.4 s warm on a ~50k-token prefix). A collapse here with three concurrent large contexts is the capacity wall, not a routing failure.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 10, "lineWidth": 1 } },
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
@@ -95,10 +153,24 @@
|
||||
"type": "timeseries",
|
||||
"title": "End-to-end request latency p95",
|
||||
"description": "Wall-clock per request, prefill included — the number an agent-loop turn feels.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "s", "custom": { "fillOpacity": 10, "lineWidth": 1 } },
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
@@ -114,10 +186,24 @@
|
||||
"type": "timeseries",
|
||||
"title": "Decode and prefill throughput",
|
||||
"description": "vLLM generation tokens, and llama.cpp's counters for frank-large (llamacpp:prompt_tokens_seconds is prompt throughput).",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "tokens/sec", "custom": { "fillOpacity": 10, "lineWidth": 1 } },
|
||||
"defaults": {
|
||||
"unit": "tokens/sec",
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
@@ -137,6 +223,40 @@
|
||||
"legendFormat": "{{job}} llama.cpp decode"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Alerts firing (ALERTS)",
|
||||
"description": "Prometheus rules for the LLM fleet (shared/prometheus/rules/llm-fleet.yml): pool saturation, queueing, prefix-cache collapse, endpoint down, p95 latency. A firing alert appears here, on Prometheus' /alerts page, and nowhere else — this stack has no Alertmanager, so nothing is notified.",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 16
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0,
|
||||
"custom": {
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "ALERTS{alertstate=\"firing\"}",
|
||||
"legendFormat": "{{alertname}} {{job}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user