observability: LLM alerting rules, evaluated but not yet delivered

The scraped metrics have no rules over them, so every graph was read after the
fact by a human. This adds shared/prometheus/rules/llm-fleet.yml, loaded via
rule_files and installed by the same copier that ships prometheus.yml (a rule
file nothing loads evaluates nothing, and Prometheus reports no error for it):

  LlmKvPoolSaturated      kv pool > 90% for 10m — the 122,636-token wall
  LlmRequestsQueueing     waiting > 0 for 10m — MAXSEQS=4 reached
  LlmPrefixCacheCollapse  reuse < 20% for 15m with traffic — the thrash signature
  LlmEndpointDown         scrape target down 2m — both replicas gone
  LlmLatencyHigh          p95 end-to-end > 60s for 10m — what an operator feels
  job:llm_prefix_cache_reuse:rate5m — recording rule shared by the alert and the
                          dashboard, so the two cannot drift

Every threshold is tied to something measured on this fleet, not to a guess,
and the file says plainly at the top that **nothing is notified**: there is no
Alertmanager here, so a firing alert reaches Prometheus' /alerts page, the
ALERTS metric and the dashboard panel added alongside it — and nowhere else.
Choosing how Frank actually gets paged is his call, not a missing config line.

Seam covers the silent failure mode: rules exist, the config references the
directory, the deployer installs it, and the dashboard graphs ALERTS.

scripts/deploy-remote.sh

@@ -201,8 +201,13 @@ copy_searxng_config() {
 }
 
 copy_prometheus_config() {
-    run_privileged mkdir -p /opt/prometheus/config
+    run_privileged mkdir -p /opt/prometheus/config /opt/prometheus/config/rules
     run_privileged cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml
+    # Rule files are read from the same mount (--config.file sits beside them),
+    # so they deploy with the config that references them: a rule file without
+    # its rule_files entry evaluates nothing, silently.
+    run_privileged cp shared/prometheus/rules/llm-fleet.yml \
+        /opt/prometheus/config/rules/llm-fleet.yml
 }
 
 copy_loki_config() {

shared/grafana/provisioning/dashboards/llm-fleet.json

@@ -2,23 +2,39 @@
   "uid": "llm-fleet",
   "title": "LLM fleet — capacity and reuse",
   "description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-large (llama.cpp) has throughput counters only. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.",
-  "tags": ["llm", "frank-medium", "capacity"],
+  "tags": [
+    "llm",
+    "frank-medium",
+    "capacity"
+  ],
   "timezone": "browser",
   "schemaVersion": 39,
   "version": 1,
   "refresh": "30s",
-  "time": { "from": "now-6h", "to": "now" },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
   "templating": {
     "list": [
       {
         "name": "job",
         "label": "endpoint",
         "type": "query",
-        "datasource": { "type": "prometheus", "uid": "prometheus" },
-        "query": { "query": "label_values(vllm:kv_cache_usage_perc, job)", "refId": "A" },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "query": {
+          "query": "label_values(vllm:kv_cache_usage_perc, job)",
+          "refId": "A"
+        },
         "includeAll": true,
         "multi": true,
-        "current": { "text": "All", "value": "$__all" }
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        }
       }
     ]
   },
@@ -28,14 +44,25 @@
       "type": "timeseries",
       "title": "KV pool fill (vllm:kv_cache_usage_perc)",
       "description": "How full each card's KV pool is. One card holds ~122,636 tokens; sustained high values with a cold prefix cache is the thrash this fleet measured.",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "fieldConfig": {
         "defaults": {
           "unit": "percentunit",
           "min": 0,
           "max": 1,
-          "custom": { "fillOpacity": 10, "lineWidth": 1 }
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1
+          }
         },
         "overrides": []
       },
@@ -52,10 +79,25 @@
       "type": "timeseries",
       "title": "Requests waiting (queue depth)",
       "description": "Anything above zero means the engine is serialising: with MAXSEQS=4 the fifth concurrent request waits.",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "fieldConfig": {
-        "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1 } },
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1
+          }
+        },
         "overrides": []
       },
       "targets": [
@@ -76,10 +118,26 @@
       "type": "timeseries",
       "title": "Prefix cache reuse rate",
       "description": "rate(hits)/rate(queries). High means pinned conversations are being served from cache (measured: 22.3 s cold vs 1.4 s warm on a ~50k-token prefix). A collapse here with three concurrent large contexts is the capacity wall, not a routing failure.",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "fieldConfig": {
-        "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 10, "lineWidth": 1 } },
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1
+          }
+        },
         "overrides": []
       },
       "targets": [
@@ -95,10 +153,24 @@
       "type": "timeseries",
       "title": "End-to-end request latency p95",
       "description": "Wall-clock per request, prefill included — the number an agent-loop turn feels.",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "fieldConfig": {
-        "defaults": { "unit": "s", "custom": { "fillOpacity": 10, "lineWidth": 1 } },
+        "defaults": {
+          "unit": "s",
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1
+          }
+        },
         "overrides": []
       },
       "targets": [
@@ -114,10 +186,24 @@
       "type": "timeseries",
       "title": "Decode and prefill throughput",
       "description": "vLLM generation tokens, and llama.cpp's counters for frank-large (llamacpp:prompt_tokens_seconds is prompt throughput).",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "fieldConfig": {
-        "defaults": { "unit": "tokens/sec", "custom": { "fillOpacity": 10, "lineWidth": 1 } },
+        "defaults": {
+          "unit": "tokens/sec",
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1
+          }
+        },
         "overrides": []
       },
       "targets": [
@@ -137,6 +223,40 @@
           "legendFormat": "{{job}} llama.cpp decode"
         }
       ]
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "Alerts firing (ALERTS)",
+      "description": "Prometheus rules for the LLM fleet (shared/prometheus/rules/llm-fleet.yml): pool saturation, queueing, prefix-cache collapse, endpoint down, p95 latency. A firing alert appears here, on Prometheus' /alerts page, and nowhere else — this stack has no Alertmanager, so nothing is notified.",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "ALERTS{alertstate=\"firing\"}",
+          "legendFormat": "{{alertname}} {{job}}"
+        }
+      ]
     }
   ]
 }

shared/prometheus/prometheus.yml

@@ -6,6 +6,12 @@ global:
   scrape_timeout: 10s
   evaluation_interval: 15s
 
+# Alerting and recording rules for the LLM fleet. They evaluate and expose state
+# through /alerts, the ALERTS metric and the Grafana dashboard; nothing is
+# notified, because this stack has no Alertmanager (see the rules file header).
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
 scrape_configs:
   # Prometheus itself
   - job_name: "prometheus"

shared/prometheus/rules/llm-fleet.yml

@@ -0,0 +1,95 @@
+# Alerting and recording rules for the LLM fleet.
+#
+# Thresholds are not guesses: each one corresponds to a failure mode measured on
+# this fleet on 2026-09-16 (see serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md).
+#
+# IMPORTANT: nothing is notified. There is no Alertmanager in this stack, so a
+# firing alert appears on Prometheus' /alerts page, in the ALERTS metric (which
+# the LLM fleet dashboard graphs) and nowhere else. Turning these into something
+# that reaches a person means adding an Alertmanager or using Grafana's own
+# alerting with a contact point — a decision about how Frank wants to be paged,
+# not a missing line of configuration.
+groups:
+  - name: llm-fleet
+    interval: 30s
+    rules:
+      # Recording rule so the dashboard and the alert below share one
+      # expression. The exported series are counters; the rate is the reuse.
+      - record: job:llm_prefix_cache_reuse:rate5m
+        expr: |
+          rate(vllm:prefix_cache_hits_total[5m])
+            / clamp_min(rate(vllm:prefix_cache_queries_total[5m]), 0.001)
+
+      # The capacity wall. Each R9700 carries 122,636 KV tokens; past ~90 %
+      # the next large request queues or evicts a cached prefix.
+      - alert: LlmKvPoolSaturated
+        expr: vllm:kv_cache_usage_perc > 0.90
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.job }}: KV pool above 90 % for 10 minutes"
+          description: >-
+            A card's KV pool is nearly full. Expect queueing or cold re-prefills
+            for large requests; the wall and its arithmetic are in the
+            frank-medium runbook.
+
+      # Concurrency limit reached: MAXSEQS=4 per replica.
+      - alert: LlmRequestsQueueing
+        expr: vllm:num_requests_waiting > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.job }}: requests waiting for 10 minutes"
+          description: >-
+            All sequence slots are busy and work is queueing. Either the fleet is
+            genuinely busy or one endpoint is carrying it alone.
+
+      # The thrash signature measured on 2026-09-16: requests keep arriving but
+      # almost none are served from cache, which is what three concurrent
+      # 50-80k-token conversations do to a two-card fleet.
+      - alert: LlmPrefixCacheCollapse
+        expr: >-
+          job:llm_prefix_cache_reuse:rate5m < 0.20
+            and rate(vllm:prefix_cache_queries_total[5m]) > 0.05
+        for: 15m
+        labels:
+          severity: info
+        annotations:
+          summary: "{{ $labels.job }}: prefix reuse below 20 % for 15 minutes"
+          description: >-
+            Nearly every request is prefilling from scratch. If several large
+            contexts are in flight together, this is the capacity wall rather
+            than a routing fault — placement and admission were both measured
+            and neither fixes it.
+
+      # The endpoint, not the replica: two replicas serve :7770 through the
+      # proxy, so this fires only when both are down (a single restart is
+      # invisible — the proxy fails over).
+      - alert: LlmEndpointDown
+        expr: up{job=~"frank-.*"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "{{ $labels.job }}: scrape target down for 2 minutes"
+          description: >-
+            Prometheus cannot reach the LLM endpoint. Check the host's units:
+            systemctl --user is-active frank-medium-{a,b,pro} on Dankclaw's
+            peers (frank-large) or Dankserver (frank-medium).
+
+      # What an operator actually feels: wall-clock per request, prefill
+      # included. A cold 50k-token prefill alone is ~22 s on one card.
+      - alert: LlmLatencyHigh
+        expr: >-
+          histogram_quantile(0.95,
+            sum by (le, job) (rate(vllm:e2e_request_latency_seconds_bucket[10m]))) > 60
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.job }}: p95 request latency above 60 s for 10 minutes"
+          description: >-
+            Requests are taking a minute or more end to end. On this hardware
+            that is usually large contexts competing for one card's KV pool.

tests/observability_quadlet.bats

@@ -324,3 +324,31 @@ PAIRS
     [ "$status" -eq 0 ]
     [ "$output" = "grafana" ]
 }
+
+@test "LLM alerting rules exist and are referenced by the config that ships them" {
+    local rules="$REPO_ROOT/shared/prometheus/rules/llm-fleet.yml"
+    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
+    [ -f "$rules" ]
+    python3 -c "
+import yaml
+d = yaml.safe_load(open('$rules'))
+names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rules']]
+assert 'record' or True
+for want in ('LlmKvPoolSaturated', 'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
+             'LlmEndpointDown', 'LlmLatencyHigh'):
+    assert want in names, want
+print('rules ok:', len(names))
+"
+    # A rule file that nothing loads evaluates nothing, and Prometheus reports
+    # no error for a missing rule_files entry — the failure is invisible.
+    grep -q '/etc/prometheus/rules/\*.yml' "$cfg"
+    # ...and the deployer must install the directory the config points at.
+    grep -q 'cp shared/prometheus/rules/llm-fleet.yml' "$REPO_ROOT/scripts/deploy-remote.sh"
+    # The dashboard surfaces firing alerts, since nothing else will.
+    python3 -c "
+import json
+d = json.load(open('$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json'))
+exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
+assert any('ALERTS' in e for e in exprs), exprs
+"
+}