diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index f74b20a..fb9d7ff 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -201,8 +201,13 @@ copy_searxng_config() { } copy_prometheus_config() { - run_privileged mkdir -p /opt/prometheus/config + run_privileged mkdir -p /opt/prometheus/config /opt/prometheus/config/rules run_privileged cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml + # Rule files are read from the same mount (--config.file sits beside them), + # so they deploy with the config that references them: a rule file without + # its rule_files entry evaluates nothing, silently. + run_privileged cp shared/prometheus/rules/llm-fleet.yml \ + /opt/prometheus/config/rules/llm-fleet.yml } copy_loki_config() { diff --git a/shared/grafana/provisioning/dashboards/llm-fleet.json b/shared/grafana/provisioning/dashboards/llm-fleet.json index da18275..865974a 100644 --- a/shared/grafana/provisioning/dashboards/llm-fleet.json +++ b/shared/grafana/provisioning/dashboards/llm-fleet.json @@ -2,23 +2,39 @@ "uid": "llm-fleet", "title": "LLM fleet — capacity and reuse", "description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-large (llama.cpp) has throughput counters only. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.", - "tags": ["llm", "frank-medium", "capacity"], + "tags": [ + "llm", + "frank-medium", + "capacity" + ], "timezone": "browser", "schemaVersion": 39, "version": 1, "refresh": "30s", - "time": { "from": "now-6h", "to": "now" }, + "time": { + "from": "now-6h", + "to": "now" + }, "templating": { "list": [ { "name": "job", "label": "endpoint", "type": "query", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "query": { "query": "label_values(vllm:kv_cache_usage_perc, job)", "refId": "A" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(vllm:kv_cache_usage_perc, job)", + "refId": "A" + }, "includeAll": true, "multi": true, - "current": { "text": "All", "value": "$__all" } + "current": { + "text": "All", + "value": "$__all" + } } ] }, @@ -28,14 +44,25 @@ "type": "timeseries", "title": "KV pool fill (vllm:kv_cache_usage_perc)", "description": "How full each card's KV pool is. One card holds ~122,636 tokens; sustained high values with a cold prefix cache is the thrash this fleet measured.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, - "custom": { "fillOpacity": 10, "lineWidth": 1 } + "custom": { + "fillOpacity": 10, + "lineWidth": 1 + } }, "overrides": [] }, @@ -52,10 +79,25 @@ "type": "timeseries", "title": "Requests waiting (queue depth)", "description": "Anything above zero means the engine is serialising: with MAXSEQS=4 the fifth concurrent request waits.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "fieldConfig": { - "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1 } }, + "defaults": { + "unit": "short", + "min": 0, + "custom": { + "fillOpacity": 10, + "lineWidth": 1 + } + }, "overrides": [] }, "targets": [ @@ -76,10 +118,26 @@ "type": "timeseries", "title": "Prefix cache reuse rate", "description": "rate(hits)/rate(queries). High means pinned conversations are being served from cache (measured: 22.3 s cold vs 1.4 s warm on a ~50k-token prefix). A collapse here with three concurrent large contexts is the capacity wall, not a routing failure.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "fieldConfig": { - "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 10, "lineWidth": 1 } }, + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "fillOpacity": 10, + "lineWidth": 1 + } + }, "overrides": [] }, "targets": [ @@ -95,10 +153,24 @@ "type": "timeseries", "title": "End-to-end request latency p95", "description": "Wall-clock per request, prefill included — the number an agent-loop turn feels.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "fieldConfig": { - "defaults": { "unit": "s", "custom": { "fillOpacity": 10, "lineWidth": 1 } }, + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 10, + "lineWidth": 1 + } + }, "overrides": [] }, "targets": [ @@ -114,10 +186,24 @@ "type": "timeseries", "title": "Decode and prefill throughput", "description": "vLLM generation tokens, and llama.cpp's counters for frank-large (llamacpp:prompt_tokens_seconds is prompt throughput).", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "fieldConfig": { - "defaults": { "unit": "tokens/sec", "custom": { "fillOpacity": 10, "lineWidth": 1 } }, + "defaults": { + "unit": "tokens/sec", + "custom": { + "fillOpacity": 10, + "lineWidth": 1 + } + }, "overrides": [] }, "targets": [ @@ -137,6 +223,40 @@ "legendFormat": "{{job}} llama.cpp decode" } ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Alerts firing (ALERTS)", + "description": "Prometheus rules for the LLM fleet (shared/prometheus/rules/llm-fleet.yml): pool saturation, queueing, prefix-cache collapse, endpoint down, p95 latency. A firing alert appears here, on Prometheus' /alerts page, and nowhere else — this stack has no Alertmanager, so nothing is notified.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "custom": { + "fillOpacity": 10, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertstate=\"firing\"}", + "legendFormat": "{{alertname}} {{job}}" + } + ] } ] } diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index c9d0a01..f93584f 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -6,6 +6,12 @@ global: scrape_timeout: 10s evaluation_interval: 15s +# Alerting and recording rules for the LLM fleet. They evaluate and expose state +# through /alerts, the ALERTS metric and the Grafana dashboard; nothing is +# notified, because this stack has no Alertmanager (see the rules file header). +rule_files: + - /etc/prometheus/rules/*.yml + scrape_configs: # Prometheus itself - job_name: "prometheus" diff --git a/shared/prometheus/rules/llm-fleet.yml b/shared/prometheus/rules/llm-fleet.yml new file mode 100644 index 0000000..51fd1bc --- /dev/null +++ b/shared/prometheus/rules/llm-fleet.yml @@ -0,0 +1,95 @@ +# Alerting and recording rules for the LLM fleet. +# +# Thresholds are not guesses: each one corresponds to a failure mode measured on +# this fleet on 2026-09-16 (see serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md). +# +# IMPORTANT: nothing is notified. There is no Alertmanager in this stack, so a +# firing alert appears on Prometheus' /alerts page, in the ALERTS metric (which +# the LLM fleet dashboard graphs) and nowhere else. Turning these into something +# that reaches a person means adding an Alertmanager or using Grafana's own +# alerting with a contact point — a decision about how Frank wants to be paged, +# not a missing line of configuration. +groups: + - name: llm-fleet + interval: 30s + rules: + # Recording rule so the dashboard and the alert below share one + # expression. The exported series are counters; the rate is the reuse. + - record: job:llm_prefix_cache_reuse:rate5m + expr: | + rate(vllm:prefix_cache_hits_total[5m]) + / clamp_min(rate(vllm:prefix_cache_queries_total[5m]), 0.001) + + # The capacity wall. Each R9700 carries 122,636 KV tokens; past ~90 % + # the next large request queues or evicts a cached prefix. + - alert: LlmKvPoolSaturated + expr: vllm:kv_cache_usage_perc > 0.90 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.job }}: KV pool above 90 % for 10 minutes" + description: >- + A card's KV pool is nearly full. Expect queueing or cold re-prefills + for large requests; the wall and its arithmetic are in the + frank-medium runbook. + + # Concurrency limit reached: MAXSEQS=4 per replica. + - alert: LlmRequestsQueueing + expr: vllm:num_requests_waiting > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.job }}: requests waiting for 10 minutes" + description: >- + All sequence slots are busy and work is queueing. Either the fleet is + genuinely busy or one endpoint is carrying it alone. + + # The thrash signature measured on 2026-09-16: requests keep arriving but + # almost none are served from cache, which is what three concurrent + # 50-80k-token conversations do to a two-card fleet. + - alert: LlmPrefixCacheCollapse + expr: >- + job:llm_prefix_cache_reuse:rate5m < 0.20 + and rate(vllm:prefix_cache_queries_total[5m]) > 0.05 + for: 15m + labels: + severity: info + annotations: + summary: "{{ $labels.job }}: prefix reuse below 20 % for 15 minutes" + description: >- + Nearly every request is prefilling from scratch. If several large + contexts are in flight together, this is the capacity wall rather + than a routing fault — placement and admission were both measured + and neither fixes it. + + # The endpoint, not the replica: two replicas serve :7770 through the + # proxy, so this fires only when both are down (a single restart is + # invisible — the proxy fails over). + - alert: LlmEndpointDown + expr: up{job=~"frank-.*"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "{{ $labels.job }}: scrape target down for 2 minutes" + description: >- + Prometheus cannot reach the LLM endpoint. Check the host's units: + systemctl --user is-active frank-medium-{a,b,pro} on Dankclaw's + peers (frank-large) or Dankserver (frank-medium). + + # What an operator actually feels: wall-clock per request, prefill + # included. A cold 50k-token prefill alone is ~22 s on one card. + - alert: LlmLatencyHigh + expr: >- + histogram_quantile(0.95, + sum by (le, job) (rate(vllm:e2e_request_latency_seconds_bucket[10m]))) > 60 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.job }}: p95 request latency above 60 s for 10 minutes" + description: >- + Requests are taking a minute or more end to end. On this hardware + that is usually large contexts competing for one card's KV pool. diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index b74cdbd..16bd079 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -324,3 +324,31 @@ PAIRS [ "$status" -eq 0 ] [ "$output" = "grafana" ] } + +@test "LLM alerting rules exist and are referenced by the config that ships them" { + local rules="$REPO_ROOT/shared/prometheus/rules/llm-fleet.yml" + local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" + [ -f "$rules" ] + python3 -c " +import yaml +d = yaml.safe_load(open('$rules')) +names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rules']] +assert 'record' or True +for want in ('LlmKvPoolSaturated', 'LlmRequestsQueueing', 'LlmPrefixCacheCollapse', + 'LlmEndpointDown', 'LlmLatencyHigh'): + assert want in names, want +print('rules ok:', len(names)) +" + # A rule file that nothing loads evaluates nothing, and Prometheus reports + # no error for a missing rule_files entry — the failure is invisible. + grep -q '/etc/prometheus/rules/\*.yml' "$cfg" + # ...and the deployer must install the directory the config points at. + grep -q 'cp shared/prometheus/rules/llm-fleet.yml' "$REPO_ROOT/scripts/deploy-remote.sh" + # The dashboard surfaces firing alerts, since nothing else will. + python3 -c " +import json +d = json.load(open('$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json')) +exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])] +assert any('ALERTS' in e for e in exprs), exprs +" +}