From a4fe5f6839e54d2a228592747fdb9c0b0fb6e42d Mon Sep 17 00:00:00 2001 From: Frank Date: Sun, 18 Oct 2026 01:52:19 -0700 Subject: [PATCH] fix(frank-zealot): serve /metrics so Prometheus can see the endpoint The scrape has never worked: lastError is "server returned HTTP status 501 Not Implemented" on http://host.containers.internal:7771/metrics, and the oldest retained sample of up{job="frank-zealot"} (2026-10-11) is already 0 -- the target has been red across four payload generations while /health answered 200 the whole time, keeping the critical LlmEndpointDown alert permanently active. llama.cpp gates /metrics behind --metrics; halogen serves it natively, which is why the asymmetry looked like a 3090-only problem. METRICS=1 in the shipped env, --metrics reaches the argv through EXTRA_ARGS, and the seam asserts both directions (default ships the flag, METRICS=0 strips it). Diagnosed via /api/v1/targets lastError rather than the alert list, which is also the lesson: Alertmanager here has no persistence, so a reboot re-stamps startsAt and a weeks-old fault looks fresh. --- AGENTS.md | 13 ++++++++++++- scripts/frank-zealot.env | 9 +++++++++ scripts/serve-frank-zealot.sh | 15 +++++++++++++++ .../Observability Provisioning.md | 15 +++++++++++++++ .../frank-zealot-llama-prism-bonsai2-pq20.md | 7 +++++++ tests/frank_zealot_unit.bats | 17 +++++++++++++++++ 6 files changed, 75 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 734816c..27dca71 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -153,7 +153,9 @@ alertmanager.container`, loopback :9093) → host-side Signal relay IP, the one bind a container can reach as `host.containers.internal`; iptables admits loopback, the host and the podman bridge subnet only) → signal-cli account 318311. Prometheus targets read down for structural reasons (HTML-scraped "probes", -Gitea metrics off); the remedy is a blackbox exporter. See +Gitea metrics off); the remedy is a blackbox exporter. `frank-zealot` was down for +a different reason until 2026-10-18: llama.cpp will not serve `/metrics` at all +without `--metrics`. See `serverAppConfigs-docs/Infrastructure/Observability.md`. ## Testing @@ -1091,6 +1093,15 @@ linked runbooks. B/token (the `GGML_CUDA_FA_ALL_QUANTS` door), so a build decision must be proven by the KV buffer line in the boot ledger, not by the build flags you remember passing. +- **A llama.cpp endpoint is invisible to Prometheus unless `--metrics` is passed.** + Without it the engine answers **501 Not Implemented** on `/metrics`, so the scrape + fails while `/health` answers 200 and `LlmEndpointDown` fires beside a healthy + service — that is how `frank-zealot` sat red across four payload generations + (the oldest retained sample of `up{job="frank-zealot"}`, 2026-10-11, is already + 0). halogen serves `/metrics` natively, so the asymmetry reads as "only the 3090 + is broken". Diagnose with `lastError` from `/api/v1/targets`, not the alert list, + and note Alertmanager has no persistence: a reboot re-stamps `startsAt` on an + alert that has been firing for weeks. **Ops** diff --git a/scripts/frank-zealot.env b/scripts/frank-zealot.env index 11e816b..e66488d 100644 --- a/scripts/frank-zealot.env +++ b/scripts/frank-zealot.env @@ -70,3 +70,12 @@ NGL=99 # and graph capture land in the journal, which is what the VRAM-fit forensics # read. Levels above 4 flood the journal. VERBOSITY=4 + +# Serve /metrics. llama.cpp answers 501 without --metrics, and Dankclaw's +# Prometheus scrapes /metrics on every fleet endpoint — a 501 reads as DOWN, so +# this service has held the critical LlmEndpointDown alert for as long as the +# metric has history (verified 2026-10-18: the oldest retained sample of +# up{job="frank-zealot"}, 2026-10-11, is already 0; scrapeUrl +# http://host.containers.internal:7771/metrics, lastError "server returned HTTP +# status 501 Not Implemented"). 0 opts out and re-breaks the scrape. +METRICS=1 diff --git a/scripts/serve-frank-zealot.sh b/scripts/serve-frank-zealot.sh index 18024b4..03f3ee4 100755 --- a/scripts/serve-frank-zealot.sh +++ b/scripts/serve-frank-zealot.sh @@ -78,6 +78,15 @@ fi # 4 adds debug detail (KV layout, buffer sizes, graph capture) which is what # the VRAM-fit forensics need. Levels above 4 flood the journal. : "${VERBOSITY:=3}" +# /metrics needs the flag: llama.cpp returns 501 unless the server starts with +# --metrics, and Dankclaw's Prometheus scrapes every fleet endpoint's /metrics — +# a 501 reads as DOWN, which is how the critical LlmEndpointDown alert ended up +# firing for this service ever since the metric has history at all — the oldest +# retained sample of up{job="frank-zealot"} (2026-10-11) is already 0, so this +# target has never been scraped successfully, across every payload era. The +# counters cost nothing on the request path; +# METRICS=0 is the documented opt-out. +: "${METRICS:=1}" # Vision projector. No colon on purpose: an explicitly empty value means # "text-only, on purpose" and must NOT be replaced by the default path. : "${MMPROJ_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-mmproj-Q8_0.gguf}" @@ -94,6 +103,12 @@ if [[ -n "$MMPROJ_FILE" ]]; then EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE") fi +# Prometheus scraping is opt-IN in llama.cpp: without --metrics the server +# answers 501 on /metrics, and the fleet scrape then reports the endpoint DOWN. +if [[ "$METRICS" = "1" ]]; then + EXTRA_ARGS+=(--metrics) +fi + # Speculative flags move as a set: they reach the argv ONLY when a drafter # is named. SPEC_TYPE=none (the shipped contract) strips both — a no-spec # baseline must not carry half a spec pair. diff --git a/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md b/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md index 2291595..e2e8768 100644 --- a/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md +++ b/serverAppConfigs-docs/Infrastructure/Observability Provisioning.md @@ -87,6 +87,21 @@ via `host.containers.internal`) → signal-cli. Some targets read down for structural reasons (HTML-scraped "probes", Gitea metrics off); the remedy is the blackbox exporter. See [[Observability]]. +**A llama.cpp endpoint is invisible unless it opts in.** `up{job="frank-zealot"}` +has been 0 for the entire retained history of the metric — the oldest sample, +2026-10-11, is already 0 — while the service itself answered `/health` 200 the +whole time. Cause: llama.cpp answers **501 Not Implemented** on `/metrics` unless +the server is started with `--metrics`, so the scrape was refused at the protocol +layer and `LlmEndpointDown` (critical) sat active across four payload generations. +Fixed 2026-10-18 by the `METRICS=1` knob in `scripts/frank-zealot.env`. Two laws +fall out of it: a target can read down beside a healthy service, so read +`lastError` from `/api/v1/targets` before believing an outage; and Alertmanager +runs here without persistence, so every restart re-stamps `startsAt` on a +long-standing alert — a "fresh" alert after a reboot is not evidence of a fresh +fault. Remaining known downs are the structural ones above (HTML-scraped probes, +Gitea metrics, ComfyUI), and halogen serves `/metrics` natively, which is why +`frank-dragoon` has always been green. + ## Related - [[Observability]] — stack topology and the alert delivery chain diff --git a/serverAppConfigs-docs/LLM/frank-zealot-llama-prism-bonsai2-pq20.md b/serverAppConfigs-docs/LLM/frank-zealot-llama-prism-bonsai2-pq20.md index d2ee43c..9691ef1 100644 --- a/serverAppConfigs-docs/LLM/frank-zealot-llama-prism-bonsai2-pq20.md +++ b/serverAppConfigs-docs/LLM/frank-zealot-llama-prism-bonsai2-pq20.md @@ -143,6 +143,13 @@ ladder above carries to the shipped shape. `BIND_ADDR`, `NGL`. An invented `KV_CACHE_TYPE` is silently swallowed by the script's `: "${CACHE_TYPE_K:=f16}"` default and the arm boots **f16**, which for this payload is an OOM that looks like a model problem. +- **`/metrics` needs `--metrics` (fixed 2026-10-18):** the fork answers **501 Not + Implemented** without the flag, and Dankclaw's Prometheus scrapes `/metrics`, so + `up{job="frank-zealot"}` has been 0 for the whole retained metric history — the + oldest sample (2026-10-11) is already 0 — while the endpoint itself answered + `/health` 200 the entire time. `METRICS=1` in `scripts/frank-zealot.env` now + ships the flag and the seam asserts it reaches the argv. halogen serves + `/metrics` natively, which is why that target was always green. - **Base alignment:** Bonsai-2 is the BASE (censored) Qwen3.8-27B lineage, so a refusal is possible on a compaction or subagent request. The CRACK re-export (`dealignai/Bonsai-2-27B-Ternary-CRACK-GGUF`, already on disk at diff --git a/tests/frank_zealot_unit.bats b/tests/frank_zealot_unit.bats index f1a1a17..87f2f3c 100755 --- a/tests/frank_zealot_unit.bats +++ b/tests/frank_zealot_unit.bats @@ -63,6 +63,7 @@ CACHE_TYPE_V=$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE") SPEC_TYPE=$(sed -nE 's/^SPEC_TYPE=(.+)$/\1/p' "$ENV_FILE") SPEC_DRAFT_N_MAX=$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE") VERBOSITY=$(sed -nE 's/^VERBOSITY=([0-9]+)$/\1/p' "$ENV_FILE") +METRICS=$(sed -nE 's/^METRICS=([0-9]+)$/\1/p' "$ENV_FILE") NGL=$(sed -nE 's/^NGL=([0-9]+)$/\1/p' "$ENV_FILE") EOF run env FRANK_ZEALOT_ENV_FILE="$envf" nvidia-modprobe=true bash "$SERVE" @@ -176,6 +177,22 @@ run_stage() { [ "$(argv_value --log-verbosity)" = "3" ] } +@test "the Prometheus metrics endpoint is served unless opted out" { + # Output contract, not a config read: what matters to the operator is that + # the engine is exec'd with --metrics, because llama.cpp answers 501 on + # /metrics without it and the fleet scrape then reports the endpoint DOWN + # (the oldest retained sample of this target's scrape, 2026-10-11, is a 0). + serve_argv + [ "$status" -eq 0 ] + grep -qxF -- '--metrics' <<<"$output" + + # And the opt-out must be real: METRICS=0 ships no /metrics at all. + sed -i 's/^METRICS=.*/METRICS=0/' "$STAGE/test.env" + run_stage + [ "$status" -eq 0 ] + ! grep -qxF -- '--metrics' <<<"$output" +} + @test "the compute-buffer pair follows the env file and stays self-consistent" { # The contract is: whatever rung the repo carries is the rung the engine # is exec'd with — the number a benchmark attributes a prefill rate to