# Alerting and recording rules for the LLM fleet.
#
# Every expression here is keyed to a series that EXISTS as of 2026-09-16. When
# frank-medium moved from vllm-radiance to llama.cpp the exported namespace
# changed with it (vllm: -> llamacpp:), and two signals the old file alerted on
# no longer exist in any form:
#
#   * vllm:kv_cache_usage_perc — llama.cpp exports no KV-fill gauge. Its
#     observable consequence is queueing, which is what LlmRequestsQueueing
#     watches instead. A renamed query would have been a rule that can never
#     fire.
#   * vllm:e2e_request_latency_seconds_* — no latency histogram either. The
#     rule it fed (p95 wall clock) is retired rather than restated, and
#     LlmSpecDecodeOff takes its place: the failure an operator actually
#     cannot see from the outside on this deployment.
#
# IMPORTANT: nothing is notified. There is no Alertmanager in this stack, so a
# firing alert appears on Prometheus' /alerts page, in the ALERTS metric (which
# the LLM fleet dashboard graphs) and nowhere else. Turning these into something
# that reaches a person means adding an Alertmanager or using Grafana's own
# alerting with a contact point — a decision about how Frank wants to be paged,
# not a missing line of configuration.
groups:
  - name: llm-fleet
    interval: 30s
    rules:
      # Recording rule so the dashboard and the alert below share one
      # expression. The exported series are counters; the rate is the reuse.
      - record: job:llm_prefix_cache_reuse:rate5m
        expr: |
          rate(llamacpp:prompt_tokens_cached_total[5m])
            / clamp_min(rate(llamacpp:prompt_tokens_total[5m]), 0.001)

      # The capacity wall, in the only form llama.cpp exports it: work waiting
      # for the single slot. Past here the next large request queues.
      - alert: LlmRequestsQueueing
        expr: llamacpp:requests_deferred > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: requests waiting for 10 minutes"
          description: >-
            The endpoint's slot(s) are busy and work is queueing. Either the
            fleet is genuinely busy or one endpoint is carrying it alone. With
            --parallel 1 a second concurrent request always shows up here.

      # The thrash signature measured on 2026-09-16: requests keep arriving but
      # almost none are served from cache, which is what several concurrent
      # 50-80k-token conversations do to this fleet.
      - alert: LlmPrefixCacheCollapse
        expr: >-
          job:llm_prefix_cache_reuse:rate5m < 0.20
            and rate(llamacpp:prompt_tokens_total[5m]) > 0.05
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "{{ $labels.job }}: prefix reuse below 20 % for 15 minutes"
          description: >-
            Nearly every request is prefilling from scratch. If several large
            contexts are in flight together, this is the capacity wall rather
            than a routing fault.

      # The speculative path is the one failure no client can see: the server
      # answers normally, just at half the speed, when the drafter is gone.
      # Drafts and accepts are counters, so a rate that stays at zero while the
      # endpoint is serving tokens means nothing is being drafted.
      - alert: LlmSpecDecodeOff
        expr: >-
          rate(llamacpp:spec_decode_num_drafts_total[5m]) == 0
            and rate(llamacpp:tokens_predicted_total[5m]) > 0.1
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: serving without speculative decoding"
          description: >-
            Tokens are being produced but no draft has been proposed for 15
            minutes. On frank-medium that means the DFlash2 drafter is not
            running (check the unit's log for common_speculative_impl_draft_dflash),
            and the endpoint is decoding autoregressively.

      # The endpoint, not a replica: one process serves each port now, so a
      # restart of the unit is a scrape gap, not a failover.
      - alert: LlmEndpointDown
        expr: up{job=~"frank-.*"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.job }}: scrape target down for 2 minutes"
          description: >-
            Prometheus cannot reach the LLM endpoint. Check the units:
            `systemctl is-active frank-medium` on Dankserver (a root llama.cpp
            unit) and `systemctl is-active frank-small` on Dankclaw. frank-large
            is not scraped at all — the halogen engine exports no metrics — so
            this alert can only ever name the endpoints that do export one.