# Alerting and recording rules for the LLM fleet.
#
# Thresholds are not guesses: each one corresponds to a failure mode measured on
# this fleet on 2026-09-16 (see serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md).
#
# IMPORTANT: nothing is notified. There is no Alertmanager in this stack, so a
# firing alert appears on Prometheus' /alerts page, in the ALERTS metric (which
# the LLM fleet dashboard graphs) and nowhere else. Turning these into something
# that reaches a person means adding an Alertmanager or using Grafana's own
# alerting with a contact point — a decision about how Frank wants to be paged,
# not a missing line of configuration.
groups:
  - name: llm-fleet
    interval: 30s
    rules:
      # Recording rule so the dashboard and the alert below share one
      # expression. The exported series are counters; the rate is the reuse.
      - record: job:llm_prefix_cache_reuse:rate5m
        expr: |
          rate(vllm:prefix_cache_hits_total[5m])
            / clamp_min(rate(vllm:prefix_cache_queries_total[5m]), 0.001)

      # The capacity wall. Each R9700 carries 122,636 KV tokens; past ~90 %
      # the next large request queues or evicts a cached prefix.
      - alert: LlmKvPoolSaturated
        expr: vllm:kv_cache_usage_perc > 0.90
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: KV pool above 90 % for 10 minutes"
          description: >-
            A card's KV pool is nearly full. Expect queueing or cold re-prefills
            for large requests; the wall and its arithmetic are in the
            frank-medium runbook.

      # Concurrency limit reached: MAXSEQS=4 per replica.
      - alert: LlmRequestsQueueing
        expr: vllm:num_requests_waiting > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: requests waiting for 10 minutes"
          description: >-
            All sequence slots are busy and work is queueing. Either the fleet is
            genuinely busy or one endpoint is carrying it alone.

      # The thrash signature measured on 2026-09-16: requests keep arriving but
      # almost none are served from cache, which is what three concurrent
      # 50-80k-token conversations do to a two-card fleet.
      - alert: LlmPrefixCacheCollapse
        expr: >-
          job:llm_prefix_cache_reuse:rate5m < 0.20
            and rate(vllm:prefix_cache_queries_total[5m]) > 0.05
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "{{ $labels.job }}: prefix reuse below 20 % for 15 minutes"
          description: >-
            Nearly every request is prefilling from scratch. If several large
            contexts are in flight together, this is the capacity wall rather
            than a routing fault — placement and admission were both measured
            and neither fixes it.

      # The endpoint, not the replica: two replicas serve :7770 through the
      # proxy, so this fires only when both are down (a single restart is
      # invisible — the proxy fails over).
      - alert: LlmEndpointDown
        expr: up{job=~"frank-.*"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.job }}: scrape target down for 2 minutes"
          description: >-
            Prometheus cannot reach the LLM endpoint. Check the units:
            `systemctl --user is-active frank-medium-{a,b,proxy}` on Dankserver,
            `systemctl is-active frank-small` (the llama.cpp root unit) or
            `systemctl --user is-active frank-large` (the halogen quadlet) on
            Dankclaw. frank-large is not scraped at all — the halogen engine
            exports no metrics — so this alert can only ever name the endpoints
            that do export one.

      # What an operator actually feels: wall-clock per request, prefill
      # included. A cold 50k-token prefill alone is ~22 s on one card.
      - alert: LlmLatencyHigh
        expr: >-
          histogram_quantile(0.95,
            sum by (le, job) (rate(vllm:e2e_request_latency_seconds_bucket[10m]))) > 60
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: p95 request latency above 60 s for 10 minutes"
          description: >-
            Requests are taking a minute or more end to end. On this hardware
            that is usually large contexts competing for one card's KV pool.