serverAppConfigs/shared/prometheus/rules/llm-fleet.yml

# Alerting and recording rules for the LLM fleet.
#
# One metric namespace: llamacpp:* — frank-small, frank-large and
# frank-primary, all llama.cpp with /metrics (the family verified from
# /metrics on 2026-09-16 and 2026-09-20). frank-primary is scraped across
# the LAN at 192.168.0.235:7770 (the netavark published-port quirk that
# forces the gateway for the Dankclaw endpoints is same-host only).
#
# Every expression is keyed to a series that EXISTS. Nothing matches on
# absence: when an endpoint's runtime leaves the fleet, its selectors are
# deleted rather than left vacuous (the vllm: group went with frank-medium's
# radiance runtime on 2026-09-26).
#
# What llamacpp:* can and cannot see:
#
#   * KV pressure: llama.cpp exports no fill gauge, so the capacity wall is
#     observed as queueing (llamacpp:requests_deferred).
#   * Prefix reuse: counted in tokens, and prompt_tokens_total EXCLUDES
#     cached tokens (its own HELP line says so), so a reuse fraction divides
#     cached by cached + prompt.
#   * Speculative decoding: drafts and accepted tokens are both counters,
#     so LlmSpecDecodeOff watches the pair.
#   * No latency histograms exist, so there is no p95 alert.
#
# IMPORTANT: delivery. Since 2026-09-17 a firing alert reaches Frank on
# Signal: Prometheus pushes to Alertmanager (dankclaw/observability/quadlet/
# alertmanager.container), whose single receiver webhooks the host-side
# signal-relay user unit, which hands the text to signal-cli under the
# existing account on this host.
groups:
  - name: llm-fleet
    interval: 30s
    rules:
      # Recording rules so the dashboard and the collapse alert below share
      # one expression. llamacpp's prompt counter EXCLUDES cached tokens, so
      # the reuse fraction is cached / (cached + prompt).
      - record: job:llm_prefix_cache_reuse:rate5m
        expr: |
          rate(llamacpp:prompt_tokens_cached_total[5m])
            / clamp_min(rate(llamacpp:prompt_tokens_cached_total[5m])
                + rate(llamacpp:prompt_tokens_total[5m]), 0.001)

      # Mean accepted draft length (1.0 = the drafter proposes and nothing is
      # taken, so the draft round is pure overhead).
      - record: job:llm_spec_accept_length:rate5m
        expr: |
          1 + rate(llamacpp:spec_decode_num_accepted_tokens_total[5m])
              / clamp_min(rate(llamacpp:spec_decode_num_drafts_total[5m]), 0.001)

      # The capacity wall, in the only form llama.cpp exports: work waiting
      # for a slot. Past here the next large request queues.
      - alert: LlmRequestsQueueing
        expr: llamacpp:requests_deferred > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: requests waiting for 10 minutes"
          description: >-
            The endpoint's slot(s) are busy and work is queueing. Either the
            fleet is genuinely busy or one endpoint is carrying it alone. With
            --parallel 1 a second concurrent request always shows up here.

      # The thrash signature measured on 2026-09-16: requests keep arriving but
      # almost none are served from cache, which is what several concurrent
      # 50-80k-token conversations do to this fleet. One threshold with a
      # traffic floor so an idle endpoint cannot fire it on a 0/0.
      - alert: LlmPrefixCacheCollapse
        expr: >-
          job:llm_prefix_cache_reuse:rate5m < 0.20
            and rate(llamacpp:prompt_tokens_total[5m])
                + rate(llamacpp:prompt_tokens_cached_total[5m]) > 0.05
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "{{ $labels.job }}: prefix reuse below 20 % for 15 minutes"
          description: >-
            Nearly every request is prefilling from scratch. If several large
            contexts are in flight together, this is the capacity wall rather
            than a routing fault.

      # The speculative path is the one failure no client can see: the server
      # answers normally, just at half the speed, when the drafter is gone.
      # Drafts and accepts are counters, so a rate that stays at zero while
      # the endpoint is serving tokens means nothing is drafted.
      - alert: LlmSpecDecodeOff
        expr: rate(llamacpp:spec_decode_num_drafts_total[5m]) == 0
              and rate(llamacpp:tokens_predicted_total[5m]) > 0.1
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: serving without speculative decoding"
          description: >-
            Tokens are being produced but no draft has been proposed for 15
            minutes. The llama.cpp load log should carry an "adding
            speculative implementation" line; the endpoint is decoding
            autoregressively.

      # The endpoint, not a replica: one process serves each port now, so a
      # restart of the unit is a scrape gap, not a failover.
      - alert: LlmEndpointDown
        expr: up{job=~"frank-.*"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.job }}: scrape target down for 2 minutes"
          description: >-
            Prometheus cannot reach the LLM endpoint. frank-small and
            frank-large are root units on Dankclaw: `systemctl is-active
            frank-small` / `frank-large`. frank-primary is a root unit on
            Dankserver: `systemctl is-active frank-primary`. All three
            scraped endpoints export the llamacpp:* family.