diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index ed644ef..797066e 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -98,10 +98,12 @@ scrape_configs: # service down. Same host, same rule, so both local endpoints use the # gateway. # - # (2026-09-26) frank-primary is llama.cpp again (the unsloth b11030-mix - # prebuilt serving Qwen3.8-Flash-Next Q8_0) on Dankserver, a DIFFERENT - # machine: the netavark published-port quirk above is same-host only, so - # plain LAN routing reaches 192.168.0.235:7770 and the job returns. + # (2026-10-05) frank-primary left this file: FreeToken (the PR587 ROCm + # engine now serving it) has NO /metrics endpoint — the job scraped 404s + # forever, which is a structural never-green, not an outage. The + # netavark published-port quirk above is same-host only; plain LAN + # routing REACHES 192.168.0.235:7770 (health answers) — only the metrics + # path is absent. If FreeToken grows a /metrics, re-add the job here. - job_name: "frank-small" metrics_path: "/metrics" static_configs: @@ -116,9 +118,3 @@ scrape_configs: labels: service: frank-large - - job_name: "frank-primary" - metrics_path: "/metrics" - static_configs: - - targets: ["192.168.0.235:7770"] - labels: - service: frank-primary diff --git a/shared/prometheus/rules/llm-fleet.yml b/shared/prometheus/rules/llm-fleet.yml index d042842..7d167df 100644 --- a/shared/prometheus/rules/llm-fleet.yml +++ b/shared/prometheus/rules/llm-fleet.yml @@ -1,10 +1,12 @@ # Alerting and recording rules for the LLM fleet. # -# One metric namespace: llamacpp:* — frank-small, frank-large and -# frank-primary, all llama.cpp with /metrics (the family verified from -# /metrics on 2026-09-16 and 2026-09-20). frank-primary is scraped across -# the LAN at 192.168.0.235:7770 (the netavark published-port quirk that -# forces the gateway for the Dankclaw endpoints is same-host only). +# One metric namespace: llamacpp:* — frank-small and frank-large, both +# llama.cpp with /metrics (the family verified from /metrics on 2026-09-16 +# and 2026-09-20). frank-primary LEFT the family 2026-10-05: FreeToken (the +# PR587 ROCm engine) has no /metrics endpoint, so its scrape job was +# deleted rather than left never-green. +# (The netavark published-port quirk that forces the Dankclaw endpoints +# onto the gateway is same-host only.) # # Every expression is keyed to a series that EXISTS. Nothing matches on # absence: when an endpoint's runtime leaves the fleet, its selectors are @@ -110,6 +112,6 @@ groups: description: >- Prometheus cannot reach the LLM endpoint. frank-small and frank-large are root units on Dankclaw: `systemctl is-active - frank-small` / `frank-large`. frank-primary is a root unit on - Dankserver: `systemctl is-active frank-primary`. All three - scraped endpoints export the llamacpp:* family. + frank-small` / `frank-large`. Both scraped endpoints export + the llamacpp:* family. (frank-primary left the scrape set + 2026-10-05 — FreeToken exports no /metrics.) diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index bbff296..ad66936 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -315,31 +315,28 @@ PAIRS @test "prometheus scrapes every LLM endpoint that exports metrics" { # The LLM endpoints are the services whose behaviour this fleet argues about - # most (KV pressure, prefix-cache reuse, queue depth). All three export - # the llamacpp:* family since 2026-09-26, when frank-primary (Dankserver) - # went back to llama.cpp — the unsloth b11030-mix prebuilt serving - # Qwen3.8-Flash-Next Q8_0 — after two days of the Lucebox DS4.1 engine, - # which exported nothing (404) and had no job. + # most (KV pressure, prefix-cache reuse, queue depth). Both Dankclaw + # endpoints export the llamacpp:* family. frank-primary LEFT the set + # 2026-10-05: the FreeToken PR587 ROCm engine has NO /metrics endpoint — + # scraping it was a structural never-green (404), so the job was deleted + # rather than left vacuous (the Lucebox DS4.1 precedent, 2026-09-26). # The two Dankclaw endpoints are HOST processes reached through # host.containers.internal: this Prometheus is a bridge-networked # container, and on this netavark host a container can only reach the # host's LAN IP at a PUBLISHED port — a host-process bind like # llama-server's 0.0.0.0:7770 is refused at 192.168.0.143:7770 while it # answers at 169.254.1.2:7770 (why frank-small stayed red by construction - # for days until the 2026-09-20 /targets audit). frank-primary is a - # DIFFERENT machine, so that quirk does not apply: plain LAN routing - # reaches 192.168.0.235:7770, and the job must say so (a gateway target - # would point at Dankclaw's own :7770 — frank-large — and double-scrape - # it under the wrong label). + # for days until the 2026-09-20 /targets audit). local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-small not scraped"; return 1; } grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-large not scraped"; return 1; } - grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-primary not scraped"; return 1; } + ! grep -q '"192.168.0.235:7770"' "$cfg" grep -q '"host.containers.internal:7770,192.168.0.235:7770"' "$cfg" \ && { echo "frank-primary folded into a Dankclaw gateway target"; return 1; } - for job in frank-small frank-large frank-primary; do + for job in frank-small frank-large; do grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; } done + ! grep -q 'job_name: "frank-primary"' "$cfg" # None of them is reached through a guarded vhost: Prometheus cannot solve # an Anubis challenge, so a franksdank.cloud target is red by construction. ! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg"