serverAppConfigs/shared/prometheus/rules/llm-fleet.yml

# Alerting and recording rules for the LLM fleet.
#
# Two metric namespaces, because the fleet runs two runtimes:
#
#   * llamacpp:* — frank-small (Dankclaw, llama.cpp with --metrics).
#   * vllm:* — frank-medium (Dankserver, vllm-radiance TP=2 since 2026-09-20).
#
# Every expression is keyed to a series that EXISTS, verified against
# 192.168.0.235:7770/metrics and 192.168.0.143:7771/metrics on 2026-09-20.
# Nothing here matches on absence: when frank-medium moved back to vLLM the
# llamacpp: selectors narrowed to frank-small rather than going vacuous, and
# the vllm: group below was added beside them, not in place of them.
#
# What each namespace can and cannot see:
#
#   * KV pressure. vLLM exports a fill gauge (vllm:kv_cache_usage_perc) and a
#     preemption counter; llama.cpp exports neither, so its capacity wall is
#     observed as queueing (llamacpp:requests_deferred).
#   * Prefix reuse. Both export a hits/queries pair counted in tokens, but the
#     units are NOT interchangeable: llama.cpp's prompt_tokens_total EXCLUDES
#     cached tokens (its own HELP line says so), so that side must divide by
#     cached + prompt to get a fraction, while vLLM divides hits by queries.
#     The two recording rules below keep that straight, and the collapse alert
#     watches both sides with one threshold.
#   * Speculative decoding. Both export drafts and accepted tokens, so
#     LlmSpecDecodeOff covers both runtimes.
#   * Latency histograms exist only on vLLM (e2e, TTFT, inter-token). No
#     p95 alert is defined for the llama.cpp side because there is nothing to
#     compute one from.
#
# IMPORTANT: delivery. Since 2026-09-17 a firing alert reaches Frank on
# Signal: Prometheus pushes to Alertmanager (dankclaw/observability/quadlet/
# alertmanager.container), whose single receiver webhooks the host-side
# signal-relay user unit, which hands the text to signal-cli under the
# existing account on this host.
groups:
  - name: llm-fleet
    interval: 30s
    rules:
      # Recording rules so the dashboard and the collapse alert below share one
      # expression. Both namespaces export hits/queries pairs in TOKENS; the
      # rate of the quotient is the reuse rate. Kept as two records because the
      # series names differ and a vacuous joint query is worse than two honest
      # ones.
      - record: job:llm_prefix_cache_reuse:rate5m
        expr: |
          rate(llamacpp:prompt_tokens_cached_total[5m])
            / clamp_min(rate(llamacpp:prompt_tokens_cached_total[5m])
                + rate(llamacpp:prompt_tokens_total[5m]), 0.001)

      - record: job:llm_vllm_prefix_cache_reuse:rate5m
        expr: |
          rate(vllm:prefix_cache_hits_total[5m])
            / clamp_min(rate(vllm:prefix_cache_queries_total[5m]), 0.001)

      # Mean accepted draft length (1.0 = the drafter proposes and nothing is
      # taken, so the draft round is pure overhead). Measured 3.0-4.3 on
      # frank-medium at boot with the DFlash2 drafter, 2026-09-20.
      - record: job:llm_spec_accept_length:rate5m
        expr: |
          1 + rate(vllm:spec_decode_num_accepted_tokens_total[5m])
              / clamp_min(rate(vllm:spec_decode_num_drafts_total[5m]), 0.001)

      # The capacity wall on the llama.cpp side, in the only form it exports:
      # work waiting for a slot. Past here the next large request queues.
      - alert: LlmRequestsQueueing
        expr: llamacpp:requests_deferred > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: requests waiting for 10 minutes"
          description: >-
            The endpoint's slot(s) are busy and work is queueing. Either the
            fleet is genuinely busy or one endpoint is carrying it alone. With
            --parallel 1 a second concurrent request always shows up here.

      # The capacity wall on the vLLM side, in its own two terms: requests the
      # scheduler has not admitted, and requests it admitted and then PREEMPTED
      # (a preemption throws away the KV and re-prefills, so its rate is the
      # cost of running the pool full). frank-medium auto-sizes its pool to
      # 943,581 tokens = 3.59 concurrent full 262,144-token conversations, so
      # saturation is four such conversations, not two.
      - alert: LlmRequestsWaiting
        expr: vllm:num_requests_waiting > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: vLLM has not admitted work for 10 minutes"
          description: >-
            The scheduler is holding requests back. Check the reason label on
            vllm:num_requests_waiting_by_reason (capacity = no KV room, defer =
            waiting for a better batch shape) and the preemption counter
            vllm:num_preemptions_total.

      - alert: LlmKvCacheSaturated
        expr: vllm:kv_cache_usage_perc > 0.95
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: KV pool above 95 % for 10 minutes"
          description: >-
            The pool is full, which is where preemption and re-prefill start:
            decode slows before anything errors. The pool is auto-sized by the
            quadlet (no KV_CACHE_MEMORY_MB pin); the boot ledger prints the
            token count — `podman logs frank-medium | grep "GPU KV cache size"`
            — and it was 943,581 tokens on 2026-09-20.

      # The thrash signature measured on 2026-09-16: requests keep arriving but
      # almost none are served from cache, which is what several concurrent
      # 50-80k-token conversations do to this fleet. One threshold, two
      # namespaces, and a traffic floor per side so an idle endpoint cannot
      # fire it on a 0/0.
      - alert: LlmPrefixCacheCollapse
        expr: >-
          (job:llm_prefix_cache_reuse:rate5m < 0.20
            and rate(llamacpp:prompt_tokens_total[5m])
                + rate(llamacpp:prompt_tokens_cached_total[5m]) > 0.05)
          or (job:llm_vllm_prefix_cache_reuse:rate5m < 0.20
            and rate(vllm:prefix_cache_queries_total[5m]) > 1)
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "{{ $labels.job }}: prefix reuse below 20 % for 15 minutes"
          description: >-
            Nearly every request is prefilling from scratch. If several large
            contexts are in flight together, this is the capacity wall rather
            than a routing fault.

      # The speculative path is the one failure no client can see: the server
      # answers normally, just at half the speed, when the drafter is gone.
      # Drafts and accepts are counters in both runtimes, so a rate that stays
      # at zero while the endpoint is serving tokens means nothing is drafted.
      - alert: LlmSpecDecodeOff
        expr: >-
          (rate(llamacpp:spec_decode_num_drafts_total[5m]) == 0
            and rate(llamacpp:tokens_predicted_total[5m]) > 0.1)
          or (rate(vllm:spec_decode_num_drafts_total[5m]) == 0
            and rate(vllm:generation_tokens_total[5m]) > 0.1)
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }}: serving without speculative decoding"
          description: >-
            Tokens are being produced but no draft has been proposed for 15
            minutes. frank-small (llama.cpp) should log "adding speculative
            implementation" at load; frank-medium (vllm-radiance) should be
            counting vllm:spec_decode_num_drafts_total and reporting 3+ mean
            accept length (job:llm_spec_accept_length:rate5m). Either way the
            endpoint is decoding autoregressively.

      # The endpoint, not a replica: one process serves each port now, so a
      # restart of the unit is a scrape gap, not a failover.
      - alert: LlmEndpointDown
        expr: up{job=~"frank-.*"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.job }}: scrape target down for 2 minutes"
          description: >-
            Prometheus cannot reach the LLM endpoint. frank-medium is a
            rootless quadlet: `systemctl --user is-active frank-medium` on
            Dankserver, then `podman ps -a --filter name=frank-medium`.
            frank-small is a root unit: `systemctl is-active frank-small` on
            Dankclaw. All three endpoints are scraped: frank-large's CIRU
            llama.cpp build does export the llamacpp:* family (verified
            2026-09-20), so this alert covers the whole fleet.