diff --git a/shared/prometheus/rules/llm-fleet.yml b/shared/prometheus/rules/llm-fleet.yml index e843b68..4249ae1 100644 --- a/shared/prometheus/rules/llm-fleet.yml +++ b/shared/prometheus/rules/llm-fleet.yml @@ -1,10 +1,15 @@ # Alerting and recording rules for the LLM fleet. # -# One metric namespace: llamacpp:* — frank-zealot and frank-dragoon, both -# llama.cpp with /metrics (the family verified from /metrics on 2026-09-16 -# and 2026-09-20). frank-archon LEFT the family 2026-10-05: FreeToken (the -# PR587 ROCm engine) has no /metrics endpoint, so its scrape job was -# deleted rather than left never-green. +# Two metric namespaces, because two runtime families are scraped: +# +# * llamacpp:* — frank-zealot (llama.cpp) and frank-dragoon (halogen, which +# re-exports the llama.cpp metric names). Verified from /metrics on +# 2026-09-16 / 2026-09-20. +# * vllm:* — frank-archon, the radiance engine on Dankserver since +# 2026-10-05. It exports vLLM-compatible names, a DIFFERENT namespace, so +# it is a separate scrape job in prometheus.yml and the rules below are a +# separate group. (The old claim that its scrape job was deleted is false: +# it re-joined the day it left the FreeToken engine and is `up` today.) # (The netavark published-port quirk that forces the Dankclaw endpoints # onto the gateway is same-host only.) # @@ -22,7 +27,8 @@ # cached by cached + prompt. # * Speculative decoding: drafts and accepted tokens are both counters, # so LlmSpecDecodeOff watches the pair. -# * No latency histograms exist, so there is no p95 alert. +# * No latency histograms in llama.cpp, so no p95 alert there; radiance's +# vllm:* namespace DOES export one, so the vllm-fleet group alerts on it. # # IMPORTANT: delivery. Since 2026-09-17 a firing alert reaches Frank on # Signal: Prometheus pushes to Alertmanager (dankclaw/observability/quadlet/ @@ -112,6 +118,84 @@ groups: description: >- Prometheus cannot reach the LLM endpoint. frank-zealot and frank-dragoon are root units on Dankclaw: `systemctl is-active - frank-zealot` / `frank-dragoon`. Both scraped endpoints export - the llamacpp:* family. (frank-archon left the scrape set - 2026-10-05 — FreeToken exports no /metrics.) + frank-zealot` / `frank-dragoon` (both export llamacpp:* — + dragoon via halogen's llama.cpp-compatible names). frank-archon + is a radiance endpoint on Dankserver (`systemctl show + frank-archon` there) exporting the vllm:* family; the + `frank-.*` selector covers all three regardless of namespace. + + # frank-archon (radiance) exports the vLLM-compatible namespace, NOT + # llamacpp:*. Every rule here is keyed to a series the exporter actually + # emits (verified from /metrics 2026-10-20: num_requests_waiting, + # gpu_prefix_cache_hit_rate, num_preemptions_total, prompt_tokens_total, + # time_to_first_token_seconds). The llamacpp rules above evaluate nothing for + # this job and these evaluate nothing for llama.cpp/halogen — two clean + # namespaces, each keyed only to what its runtime exports. + - name: vllm-fleet + interval: 30s + rules: + # p95 time-to-first-token, shared with the dashboard. When the endpoint + # is idle every bucket rate is zero, histogram_quantile yields NaN, and + # neither this nor the alert below can fire on an idle engine. + - record: job:vllm_ttft_p95:rate5m + expr: | + histogram_quantile(0.95, sum by (job, le) + (rate(vllm:time_to_first_token_seconds_bucket[5m]))) + + # Radiance exports a real queue-depth gauge (llama.cpp only shows the + # wall as requests_deferred). --max-num-seqs is small, so a sustained + # wait is either real load or this endpoint carrying the fleet alone. + - alert: VllmRequestsQueueing + expr: vllm:num_requests_waiting > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.job }}: requests waiting for 10 minutes" + description: >- + The radiance scheduler has requests queued. Either the fleet is + genuinely busy or this endpoint is carrying it alone. + + # Radiance reports prefix reuse directly as a hit-rate gauge — no + # counter division needed. The token-rate floor keeps an idle engine + # (hit rate 0 with no traffic) from firing. + - alert: VllmPrefixCacheCollapse + expr: >- + vllm:gpu_prefix_cache_hit_rate < 0.20 + and rate(vllm:prompt_tokens_total[5m]) > 0.05 + for: 15m + labels: + severity: info + annotations: + summary: "{{ $labels.job }}: prefix hit rate below 20 % for 15 minutes" + description: >- + Nearly every request prefilling from scratch. With several large + contexts in flight this is the capacity wall, not a routing fault. + + # Preemption is the KV-thrash signature radiance reports that llama.cpp + # cannot: the engine evicts a running request to admit another. Any + # sustained rate means the KV cache cannot hold the concurrency in flight. + - alert: VllmPreempting + expr: rate(vllm:num_preemptions_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "{{ $labels.job }}: engine preempting requests" + description: >- + The engine is evicting running requests to admit new ones — the KV + cache is undersized for the concurrency in flight. + + # p95 first-token latency, kept HIGH on purpose: a large-prompt prefill + # legitimately spends many seconds before token 1, so this fires only on + # a wedged scheduler or a queue backed up behind one request. + - alert: VllmTtftSlow + expr: job:vllm_ttft_p95:rate5m > 120 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.job }}: p95 time-to-first-token over 120 s" + description: >- + First-token latency is pathological, not an honest big prefill. + Likely a wedged scheduler or one request stalling the queue. diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index ff2d312..12119cd 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -313,33 +313,33 @@ PAIRS done } -@test "prometheus scrapes every LLM endpoint that exports metrics" { - # The LLM endpoints are the services whose behaviour this fleet argues about - # most (KV pressure, prefix-cache reuse, queue depth). Both Dankclaw - # endpoints export the llamacpp:* family. frank-archon LEFT the set - # 2026-10-05: the FreeToken PR587 ROCm engine has NO /metrics endpoint — - # scraping it was a structural never-green (404), so the job was deleted - # rather than left vacuous (the Lucebox DS4.1 precedent, 2026-09-26). - # The two Dankclaw endpoints are HOST processes reached through - # host.containers.internal: this Prometheus is a bridge-networked - # container, and on this netavark host a container can only reach the - # host's LAN IP at a PUBLISHED port — a host-process bind like - # llama-server's 0.0.0.0:7770 is refused at 192.168.0.143:7770 while it - # answers at 169.254.1.2:7770 (why frank-zealot stayed red by construction - # for days until the 2026-09-20 /targets audit). +@test "prometheus scrapes every LLM endpoint, in its real metric namespace" { + # Every LLM endpoint that exports /metrics must have a scrape job. As of + # 2026-10-05 there are TWO runtime namespaces scraped: + # * llamacpp:* — frank-zealot (llama.cpp) and frank-dragoon (halogen, + # re-exporting the llama.cpp names), both root host processes on + # Dankclaw reached via host.containers.internal — this Prometheus is a + # bridge container and a host-process bind is refused at the LAN IP, + # which is why frank-zealot read red by construction for days. + # * vllm:* — frank-archon (radiance) on Dankserver, a different machine, + # so plain LAN routing reaches 192.168.0.235:7770. It RE-JOINED the + # scrape set with radiance; the old "job deleted, exports no /metrics" + # note belonged to the retired FreeToken engine and is false today, so + # this seam now asserts archon IS scraped (a bare `! grep` here was + # set -e-exempt and never enforced the opposite). local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-zealot not scraped"; return 1; } grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-dragoon not scraped"; return 1; } - ! grep -q '"192.168.0.235:7770"' "$cfg" - grep -q '"host.containers.internal:7770,192.168.0.235:7770"' "$cfg" \ - && { echo "frank-archon folded into a Dankclaw gateway target"; return 1; } - for job in frank-zealot frank-dragoon; do + grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-archon not scraped"; return 1; } + for job in frank-zealot frank-dragoon frank-archon; do grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; } done - ! grep -q 'job_name: "frank-archon"' "$cfg" - # None of them is reached through a guarded vhost: Prometheus cannot solve - # an Anubis challenge, so a franksdank.cloud target is red by construction. - ! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg" + # No LLM endpoint is reached through a guarded vhost: Prometheus cannot + # solve an Anubis challenge, so a franksdank.cloud target is red by + # construction. + grep -qE '^[[:space:]]*-[[:space:]]*"?https?://[^"]*franksdank' "$cfg" \ + && { echo "a scrape target points at a guarded vhost"; return 1; } + true } @test "grafana provisioning is declared in git and installed by the deployer" { @@ -378,28 +378,44 @@ names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rule for want in ('job:llm_prefix_cache_reuse:rate5m', 'job:llm_spec_accept_length:rate5m', 'LlmRequestsQueueing', 'LlmPrefixCacheCollapse', - 'LlmSpecDecodeOff', 'LlmEndpointDown'): + 'LlmSpecDecodeOff', 'LlmEndpointDown', + 'job:vllm_ttft_p95:rate5m', 'VllmRequestsQueueing', + 'VllmPrefixCacheCollapse', 'VllmPreempting', 'VllmTtftSlow'): assert want in names, want # Every rule must be keyed to a series that actually exists. A rule naming a # series nobody exports evaluates nothing forever and errors never — the -# failure this seam exists for. Since the 2026-09-26 cutover the only scraped -# LLM runtime is llama.cpp (frank-zealot, frank-dragoon); the vllm: family left -# with frank-medium's radiance runtime and must not survive in any expr. +# failure this seam exists for. Two runtimes are scraped in two namespaces: +# llama.cpp/halogen (llamacpp:*) for frank-zealot and frank-dragoon, and +# radiance (vllm:*) for frank-archon. Both families were read straight off the +# endpoints' /metrics; anything else is a guess. exprs = [r.get('expr', '') for g in d['groups'] for r in g['rules']] -# llama.cpp families measured from /metrics on 2026-09-16 and 2026-09-20 -# (frank-zealot, frank-dragoon). Anything else is a guess. -live = {'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total', - 'llamacpp:requests_deferred', 'llamacpp:requests_processing', - 'llamacpp:spec_decode_num_drafts_total', - 'llamacpp:spec_decode_num_accepted_tokens_total', - 'llamacpp:spec_decode_num_draft_tokens_total', - 'llamacpp:tokens_predicted_total', - 'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds', - 'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode'} +live = { + # llama.cpp / halogen, measured from /metrics 2026-09-16 and 2026-09-20. + 'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total', + 'llamacpp:requests_deferred', 'llamacpp:requests_processing', + 'llamacpp:spec_decode_num_drafts_total', + 'llamacpp:spec_decode_num_accepted_tokens_total', + 'llamacpp:spec_decode_num_draft_tokens_total', + 'llamacpp:tokens_predicted_total', + 'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds', + 'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode', + # radiance (frank-archon), measured from /metrics 2026-10-20. + 'vllm:num_requests_waiting', 'vllm:gpu_prefix_cache_hit_rate', + 'vllm:num_preemptions_total', 'vllm:prompt_tokens_total', + 'vllm:time_to_first_token_seconds_bucket', +} used = {m for e in exprs for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', e)} assert used <= live, used - live -assert not any(':vllm' in e for e in exprs), 'vllm: series in rules but no vllm endpoint is scraped' +# The two namespaces must not bleed across groups: a llamacpp: selector +# evaluates nothing for the radiance job and vice versa, so each group may +# reference only its own family. +per = {g['name']: {m.split(':')[0] for r in g['rules'] + for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', r.get('expr', ''))} + for g in d['groups']} +for name, fams in per.items(): + allowed = {'llamacpp'} if name == 'llm-fleet' else {'vllm'} + assert fams <= allowed, (name, fams) print('rules ok:', len(names)) " # A rule file that nothing loads evaluates nothing, and Prometheus reports