fix(frank-zealot): serve /metrics so Prometheus can see the endpoint
The scrape has never worked: lastError is "server returned HTTP status 501 Not Implemented" on http://host.containers.internal:7771/metrics, and the oldest retained sample of up{job="frank-zealot"} (2026-10-11) is already 0 -- the target has been red across four payload generations while /health answered 200 the whole time, keeping the critical LlmEndpointDown alert permanently active. llama.cpp gates /metrics behind --metrics; halogen serves it natively, which is why the asymmetry looked like a 3090-only problem. METRICS=1 in the shipped env, --metrics reaches the argv through EXTRA_ARGS, and the seam asserts both directions (default ships the flag, METRICS=0 strips it). Diagnosed via /api/v1/targets lastError rather than the alert list, which is also the lesson: Alertmanager here has no persistence, so a reboot re-stamps startsAt and a weeks-old fault looks fresh.
This commit is contained in:
1 parent
34a30d8505
commit
a4fe5f6839
6 files changed
+75
-1
No files matched your search
@@ -63,6 +63,7 @@ CACHE_TYPE_V=$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")
|
||||
SPEC_TYPE=$(sed -nE 's/^SPEC_TYPE=(.+)$/\1/p' "$ENV_FILE")
|
||||
SPEC_DRAFT_N_MAX=$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
VERBOSITY=$(sed -nE 's/^VERBOSITY=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
METRICS=$(sed -nE 's/^METRICS=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
NGL=$(sed -nE 's/^NGL=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
EOF
|
||||
run env FRANK_ZEALOT_ENV_FILE="$envf" nvidia-modprobe=true bash "$SERVE"
|
||||
@@ -176,6 +177,22 @@ run_stage() {
|
||||
[ "$(argv_value --log-verbosity)" = "3" ]
|
||||
}
|
||||
|
||||
@test "the Prometheus metrics endpoint is served unless opted out" {
|
||||
# Output contract, not a config read: what matters to the operator is that
|
||||
# the engine is exec'd with --metrics, because llama.cpp answers 501 on
|
||||
# /metrics without it and the fleet scrape then reports the endpoint DOWN
|
||||
# (the oldest retained sample of this target's scrape, 2026-10-11, is a 0).
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
grep -qxF -- '--metrics' <<<"$output"
|
||||
|
||||
# And the opt-out must be real: METRICS=0 ships no /metrics at all.
|
||||
sed -i 's/^METRICS=.*/METRICS=0/' "$STAGE/test.env"
|
||||
run_stage
|
||||
[ "$status" -eq 0 ]
|
||||
! grep -qxF -- '--metrics' <<<"$output"
|
||||
}
|
||||
|
||||
@test "the compute-buffer pair follows the env file and stays self-consistent" {
|
||||
# The contract is: whatever rung the repo carries is the rung the engine
|
||||
# is exec'd with — the number a benchmark attributes a prefill rate to
|
||||
|
||||
Reference in new issue
Block a user