feat(observability): add vllm-fleet rule group for frank-archon
radiance exports vllm:* metrics (verified on the live endpoint), not llamacpp:*: nothing in llm-fleet could ever see frank-archon. Add a vllm-fleet group - 1 recording + 4 alert (queueing, prefix-cache collapse, preemption, p95 time-to-first-token) keyed to vllm:*. Split the observability seam so llm-fleet rules bind llamacpp:* and vllm-fleet rules bind vllm:*, and neither may evaluate the other's namespace. Drops the retired no-metrics-for-archon doctrine the seam still asserted and the vacuous namespace grep.
This commit is contained in:
Frank committed
1 parent
5cc2b3c25a
commit
eb4d9910cb
2 files changed
+146
-46
No files matched your search
@@ -313,33 +313,33 @@ PAIRS
|
||||
done
|
||||
}
|
||||
|
||||
@test "prometheus scrapes every LLM endpoint that exports metrics" {
|
||||
# The LLM endpoints are the services whose behaviour this fleet argues about
|
||||
# most (KV pressure, prefix-cache reuse, queue depth). Both Dankclaw
|
||||
# endpoints export the llamacpp:* family. frank-archon LEFT the set
|
||||
# 2026-10-05: the FreeToken PR587 ROCm engine has NO /metrics endpoint —
|
||||
# scraping it was a structural never-green (404), so the job was deleted
|
||||
# rather than left vacuous (the Lucebox DS4.1 precedent, 2026-09-26).
|
||||
# The two Dankclaw endpoints are HOST processes reached through
|
||||
# host.containers.internal: this Prometheus is a bridge-networked
|
||||
# container, and on this netavark host a container can only reach the
|
||||
# host's LAN IP at a PUBLISHED port — a host-process bind like
|
||||
# llama-server's 0.0.0.0:7770 is refused at 192.168.0.143:7770 while it
|
||||
# answers at 169.254.1.2:7770 (why frank-zealot stayed red by construction
|
||||
# for days until the 2026-09-20 /targets audit).
|
||||
@test "prometheus scrapes every LLM endpoint, in its real metric namespace" {
|
||||
# Every LLM endpoint that exports /metrics must have a scrape job. As of
|
||||
# 2026-10-05 there are TWO runtime namespaces scraped:
|
||||
# * llamacpp:* — frank-zealot (llama.cpp) and frank-dragoon (halogen,
|
||||
# re-exporting the llama.cpp names), both root host processes on
|
||||
# Dankclaw reached via host.containers.internal — this Prometheus is a
|
||||
# bridge container and a host-process bind is refused at the LAN IP,
|
||||
# which is why frank-zealot read red by construction for days.
|
||||
# * vllm:* — frank-archon (radiance) on Dankserver, a different machine,
|
||||
# so plain LAN routing reaches 192.168.0.235:7770. It RE-JOINED the
|
||||
# scrape set with radiance; the old "job deleted, exports no /metrics"
|
||||
# note belonged to the retired FreeToken engine and is false today, so
|
||||
# this seam now asserts archon IS scraped (a bare `! grep` here was
|
||||
# set -e-exempt and never enforced the opposite).
|
||||
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
||||
grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-zealot not scraped"; return 1; }
|
||||
grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-dragoon not scraped"; return 1; }
|
||||
! grep -q '"192.168.0.235:7770"' "$cfg"
|
||||
grep -q '"host.containers.internal:7770,192.168.0.235:7770"' "$cfg" \
|
||||
&& { echo "frank-archon folded into a Dankclaw gateway target"; return 1; }
|
||||
for job in frank-zealot frank-dragoon; do
|
||||
grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-archon not scraped"; return 1; }
|
||||
for job in frank-zealot frank-dragoon frank-archon; do
|
||||
grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; }
|
||||
done
|
||||
! grep -q 'job_name: "frank-archon"' "$cfg"
|
||||
# None of them is reached through a guarded vhost: Prometheus cannot solve
|
||||
# an Anubis challenge, so a franksdank.cloud target is red by construction.
|
||||
! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg"
|
||||
# No LLM endpoint is reached through a guarded vhost: Prometheus cannot
|
||||
# solve an Anubis challenge, so a franksdank.cloud target is red by
|
||||
# construction.
|
||||
grep -qE '^[[:space:]]*-[[:space:]]*"?https?://[^"]*franksdank' "$cfg" \
|
||||
&& { echo "a scrape target points at a guarded vhost"; return 1; }
|
||||
true
|
||||
}
|
||||
|
||||
@test "grafana provisioning is declared in git and installed by the deployer" {
|
||||
@@ -378,28 +378,44 @@ names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rule
|
||||
for want in ('job:llm_prefix_cache_reuse:rate5m',
|
||||
'job:llm_spec_accept_length:rate5m',
|
||||
'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
|
||||
'LlmSpecDecodeOff', 'LlmEndpointDown'):
|
||||
'LlmSpecDecodeOff', 'LlmEndpointDown',
|
||||
'job:vllm_ttft_p95:rate5m', 'VllmRequestsQueueing',
|
||||
'VllmPrefixCacheCollapse', 'VllmPreempting', 'VllmTtftSlow'):
|
||||
assert want in names, want
|
||||
# Every rule must be keyed to a series that actually exists. A rule naming a
|
||||
# series nobody exports evaluates nothing forever and errors never — the
|
||||
# failure this seam exists for. Since the 2026-09-26 cutover the only scraped
|
||||
# LLM runtime is llama.cpp (frank-zealot, frank-dragoon); the vllm: family left
|
||||
# with frank-medium's radiance runtime and must not survive in any expr.
|
||||
# failure this seam exists for. Two runtimes are scraped in two namespaces:
|
||||
# llama.cpp/halogen (llamacpp:*) for frank-zealot and frank-dragoon, and
|
||||
# radiance (vllm:*) for frank-archon. Both families were read straight off the
|
||||
# endpoints' /metrics; anything else is a guess.
|
||||
exprs = [r.get('expr', '') for g in d['groups'] for r in g['rules']]
|
||||
# llama.cpp families measured from /metrics on 2026-09-16 and 2026-09-20
|
||||
# (frank-zealot, frank-dragoon). Anything else is a guess.
|
||||
live = {'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
|
||||
'llamacpp:requests_deferred', 'llamacpp:requests_processing',
|
||||
'llamacpp:spec_decode_num_drafts_total',
|
||||
'llamacpp:spec_decode_num_accepted_tokens_total',
|
||||
'llamacpp:spec_decode_num_draft_tokens_total',
|
||||
'llamacpp:tokens_predicted_total',
|
||||
'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
|
||||
'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode'}
|
||||
live = {
|
||||
# llama.cpp / halogen, measured from /metrics 2026-09-16 and 2026-09-20.
|
||||
'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
|
||||
'llamacpp:requests_deferred', 'llamacpp:requests_processing',
|
||||
'llamacpp:spec_decode_num_drafts_total',
|
||||
'llamacpp:spec_decode_num_accepted_tokens_total',
|
||||
'llamacpp:spec_decode_num_draft_tokens_total',
|
||||
'llamacpp:tokens_predicted_total',
|
||||
'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
|
||||
'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode',
|
||||
# radiance (frank-archon), measured from /metrics 2026-10-20.
|
||||
'vllm:num_requests_waiting', 'vllm:gpu_prefix_cache_hit_rate',
|
||||
'vllm:num_preemptions_total', 'vllm:prompt_tokens_total',
|
||||
'vllm:time_to_first_token_seconds_bucket',
|
||||
}
|
||||
used = {m for e in exprs
|
||||
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', e)}
|
||||
assert used <= live, used - live
|
||||
assert not any(':vllm' in e for e in exprs), 'vllm: series in rules but no vllm endpoint is scraped'
|
||||
# The two namespaces must not bleed across groups: a llamacpp: selector
|
||||
# evaluates nothing for the radiance job and vice versa, so each group may
|
||||
# reference only its own family.
|
||||
per = {g['name']: {m.split(':')[0] for r in g['rules']
|
||||
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', r.get('expr', ''))}
|
||||
for g in d['groups']}
|
||||
for name, fams in per.items():
|
||||
allowed = {'llamacpp'} if name == 'llm-fleet' else {'vllm'}
|
||||
assert fams <= allowed, (name, fams)
|
||||
print('rules ok:', len(names))
|
||||
"
|
||||
# A rule file that nothing loads evaluates nothing, and Prometheus reports
|
||||
|
||||
Reference in new issue
Block a user