serverAppConfigs/shared/prometheus/rules/llm-fleet.yml

# Alerting and recording rules for the LLM fleet.
#
# Every expression here is keyed to a series that EXISTS as of 2026-09-16. When
# frank-medium moved from vllm-radiance to llama.cpp the exported namespace
# changed with it (vllm: -> llamacpp:), and two signals the old file alerted on
# no longer exist in any form:
#
#   * vllm:kv_cache_usage_perc — llama.cpp exports no KV-fill gauge. Its
#     observable consequence is queueing, which is what LlmRequestsQueueing
#     watches instead. A renamed query would have been a rule that can never
#     fire.
#   * vllm:e2e_request_latency_seconds_* — no latency histogram either. The
#     rule it fed (p95 wall clock) is retired rather than restated, and
#     LlmSpecDecodeOff takes its place: the failure an operator actually
#     cannot see from the outside on this deployment.
#
# IMPORTANT: delivery. Since 2026-09-17 a firing alert reaches Frank on
# Signal: Prometheus pushes to Alertmanager (dankclaw/observability/quadlet/
# alertmanager.container), whose single receiver webhooks the host-side
# signal-relay user unit, which hands the text to signal-cli under the
# existing account on this host. Before that date a firing alert appeared on
# Prometheus' /alerts page, in the ALERTS metric (which the LLM fleet
# dashboard graphs) and nowhere else.
groups:
  - name: llm-fleet
    interval: 30s
    rules:
      # Recording rule so the dashboard and the alert below share one
      # expression. The exported series are counters; the rate is the reuse.
      - record: job:llm_prefix_cache_reuse:rate5m
        expr: |
          rate(llamacpp:prompt_tokens_cached_total[5m])
            / clamp_min(rate(llamacpp:prompt_tokens_total[5m]), 0.001)

      # The capacity wall, in the only form llama.cpp exports it: work waiting
      # for the single slot. Past here the next large request queues.
      - alert: LlmRequestsQueueing
        expr: llamacpp:requests_deferred > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: requests waiting for 10 minutes"
          description: >-
            The endpoint's slot(s) are busy and work is queueing. Either the
            fleet is genuinely busy or one endpoint is carrying it alone. With
            --parallel 1 a second concurrent request always shows up here.

      # The thrash signature measured on 2026-09-16: requests keep arriving but
      # almost none are served from cache, which is what several concurrent
      # 50-80k-token conversations do to this fleet.
      - alert: LlmPrefixCacheCollapse
        expr: >-
          job:llm_prefix_cache_reuse:rate5m < 0.20
            and rate(llamacpp:prompt_tokens_total[5m]) > 0.05
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "{{ $labels.job }}: prefix reuse below 20 % for 15 minutes"
          description: >-
            Nearly every request is prefilling from scratch. If several large
            contexts are in flight together, this is the capacity wall rather
            than a routing fault.

      # The speculative path is the one failure no client can see: the server
      # answers normally, just at half the speed, when the drafter is gone.
      # Drafts and accepts are counters, so a rate that stays at zero while the
      # endpoint is serving tokens means nothing is being drafted.
      - alert: LlmSpecDecodeOff
        expr: >-
          rate(llamacpp:spec_decode_num_drafts_total[5m]) == 0
            and rate(llamacpp:tokens_predicted_total[5m]) > 0.1
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: serving without speculative decoding"
          description: >-
            Tokens are being produced but no draft has been proposed for 15
            minutes. On the llama.cpp endpoints that means the drafter is not
            running (check the unit's log for the
            "adding speculative implementation" line — draft-mtp on both
            endpoints as of 2026-09-17), and the endpoint is decoding
            autoregressively.

      # The endpoint, not a replica: one process serves each port now, so a
      # restart of the unit is a scrape gap, not a failover.
      - alert: LlmEndpointDown
        expr: up{job=~"frank-.*"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.job }}: scrape target down for 2 minutes"
          description: >-
            Prometheus cannot reach the LLM endpoint. Check the units:
            `systemctl is-active frank-medium` on Dankserver (a root llama.cpp
            unit) and `systemctl is-active frank-small` on Dankclaw. frank-large
            is not scraped at all — the halogen engine exports no metrics — so
            this alert can only ever name the endpoints that do export one.