From 79cca19919f90cb7dd84242f712970bbb5a988eb Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2026 13:59:49 -0700 Subject: [PATCH] observability: the LLM surfaces follow the two-endpoint split frank-large has no /metrics to scrape any more, and scraping it would be red by construction: the halogen engine publishes /health, /cache and /v1/* only. That job is removed with the reason written down (the remedy is the blackbox exporter this fleet already wants for the comfyui/services probes), and frank-small's llama.cpp endpoint takes the slot at 192.168.0.143:7771. Homepage: frank-large's card still described the ik runtime, and the new endpoint had no card at all. Both fixed, with a loopback probe -- the checker runs on Dankclaw, which is also what makes the card green-able. Dashboard: the endpoint picker was sourced from a vLLM-only metric, so a llama.cpp job could never be selected, and the prose named frank-large as the llama.cpp endpoint. The picker now matches both runtimes' series and the llama.cpp targets carry the job filter every other panel already used. The seams assert the new contract including the ABSENCE of a 7770 target, so a well-meaning edit cannot re-add a permanently-red probe. --- .../provisioning/dashboards/llm-fleet.json | 15 ++++++++++----- shared/homepage/services.json | 13 ++++++++++++- shared/prometheus/prometheus.yml | 16 ++++++++++++---- shared/prometheus/rules/llm-fleet.yml | 10 +++++++--- tests/homepage_config.bats | 15 ++++++++------- tests/observability_quadlet.bats | 10 ++++++++-- 6 files changed, 57 insertions(+), 22 deletions(-) diff --git a/shared/grafana/provisioning/dashboards/llm-fleet.json b/shared/grafana/provisioning/dashboards/llm-fleet.json index 865974a..325abcf 100644 --- a/shared/grafana/provisioning/dashboards/llm-fleet.json +++ b/shared/grafana/provisioning/dashboards/llm-fleet.json @@ -1,7 +1,7 @@ { "uid": "llm-fleet", "title": "LLM fleet — capacity and reuse", - "description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-large (llama.cpp) has throughput counters only. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.", + "description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-small (llama.cpp on Dankclaw's RTX 3090) exports throughput counters and appears in the same endpoint picker; frank-large runs the halogen engine, which publishes /health, /cache and /v1/* and no metrics endpoint at all, so it cannot appear here. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.", "tags": [ "llm", "frank-medium", @@ -26,7 +26,7 @@ "uid": "prometheus" }, "query": { - "query": "label_values(vllm:kv_cache_usage_perc, job)", + "query": "label_values({__name__=~\"vllm:kv_cache_usage_perc|llamacpp:.*\"}, job)", "refId": "A" }, "includeAll": true, @@ -185,7 +185,7 @@ "id": 5, "type": "timeseries", "title": "Decode and prefill throughput", - "description": "vLLM generation tokens, and llama.cpp's counters for frank-large (llamacpp:prompt_tokens_seconds is prompt throughput).", + "description": "vLLM generation tokens, plus llama.cpp's counters for frank-small: llamacpp:predicted_tokens_seconds is decode and llamacpp:prompt_tokens_seconds is prompt throughput. The llama.cpp series carry the job label of their scrape, so the endpoint picker selects between the runtimes.", "gridPos": { "h": 8, "w": 12, @@ -219,8 +219,13 @@ }, { "refId": "C", - "expr": "llamacpp:predicted_tokens_seconds", - "legendFormat": "{{job}} llama.cpp decode" + "expr": "llamacpp:predicted_tokens_seconds{job=~\"$job\"}", + "legendFormat": "{{job}} llama.cpp decode tok/s" + }, + { + "refId": "D", + "expr": "llamacpp:prompt_tokens_seconds{job=~\"$job\"}", + "legendFormat": "{{job}} llama.cpp prefill tok/s" } ] }, diff --git a/shared/homepage/services.json b/shared/homepage/services.json index a686de0..c6aa557 100644 --- a/shared/homepage/services.json +++ b/shared/homepage/services.json @@ -222,7 +222,7 @@ { "name": "frank-large", "category": "admin", - "description": "The large LLM (Qwen3.8-Flash-Next Uncensored, ik_llama.cpp) on Dankclaw.", + "description": "The large LLM (Qwen3.8-Flash-Next w4b, halogen-flash-server on the Strix Halo iGPU). Two 200k-token conversations resident.", "url": "", "docs": "", "probe": "127.0.0.1:7770", @@ -241,6 +241,17 @@ "wip": false, "showLink": false }, + { + "name": "frank-small", + "category": "admin", + "description": "The small LLM (Qwen3.8-27B GSQ-RCO IQ3_S with MTP and vision, llama.cpp) on Dankclaw's RTX 3090. The fleet's image path.", + "url": "", + "docs": "", + "probe": "127.0.0.1:7771", + "method": "http", + "wip": false, + "showLink": false + }, { "name": "mariadb", "category": "admin", diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index f93584f..76b4fad 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -68,7 +68,15 @@ scrape_configs: # and the prefix-cache pair vllm:prefix_cache_hits_total over # vllm:prefix_cache_queries_total (their rates give the hit rate, which says # whether pinned conversations are actually being reused). - # frank-large is llama.cpp and exposes throughput counters under llamacpp:. + # frank-large is deliberately NOT scraped: the halogen engine that serves it + # publishes /health, /cache and /v1/* and no Prometheus text endpoint, so a + # target here would be red by construction and would fire LlmEndpointDown + # forever. Its liveness is on the homepage (a loopback probe) and in the + # engine's own boot ledger. The remedy, when the fleet wants it, is the + # blackbox exporter already wanted for the comfyui/services probes. + # + # frank-small is llama.cpp (its serve script passes --metrics) and does + # export text-format counters under llamacpp: on the port its unit publishes. - job_name: "frank-medium" metrics_path: "/metrics" static_configs: @@ -76,9 +84,9 @@ scrape_configs: labels: service: frank-medium - - job_name: "frank-large" + - job_name: "frank-small" metrics_path: "/metrics" static_configs: - - targets: ["192.168.0.143:7770"] + - targets: ["192.168.0.143:7771"] labels: - service: frank-large + service: frank-small diff --git a/shared/prometheus/rules/llm-fleet.yml b/shared/prometheus/rules/llm-fleet.yml index 51fd1bc..e507ad5 100644 --- a/shared/prometheus/rules/llm-fleet.yml +++ b/shared/prometheus/rules/llm-fleet.yml @@ -75,9 +75,13 @@ groups: annotations: summary: "{{ $labels.job }}: scrape target down for 2 minutes" description: >- - Prometheus cannot reach the LLM endpoint. Check the host's units: - systemctl --user is-active frank-medium-{a,b,pro} on Dankclaw's - peers (frank-large) or Dankserver (frank-medium). + Prometheus cannot reach the LLM endpoint. Check the units: + `systemctl --user is-active frank-medium-{a,b,proxy}` on Dankserver, + `systemctl is-active frank-small` (the llama.cpp root unit) or + `systemctl --user is-active frank-large` (the halogen quadlet) on + Dankclaw. frank-large is not scraped at all — the halogen engine + exports no metrics — so this alert can only ever name the endpoints + that do export one. # What an operator actually feels: wall-clock per request, prefill # included. A cold 50k-token prefill alone is ~22 s on one card. diff --git a/tests/homepage_config.bats b/tests/homepage_config.bats index 803e4dc..a5312b1 100644 --- a/tests/homepage_config.bats +++ b/tests/homepage_config.bats @@ -41,18 +41,19 @@ PY } @test "the LLM status cards track the fleet, not retired service names" { - # This drifted: the card probing Dankserver's :7770 was still named - # frank-small long after frank-small was retired and frank-medium took that - # endpoint, so the homepage labelled the fleet's busiest service with a dead - # name. The probe target was right; only the identity was wrong. + # This drifted twice: the card probing Dankserver's :7770 was still named + # frank-small long after frank-medium took that endpoint, and Dankclaw's + # second endpoint now exists again as a real service on its own port. The + # probe target is what matters; the name has to follow it. python3 - "$REPO_ROOT/shared/homepage/services.json" <<'PY' import json, sys services = json.load(open(sys.argv[1])) by_probe = {s["probe"]: s["name"] for s in services} medium = by_probe.get("192.168.0.235:7770") assert medium == "frank-medium", f"Dankserver :7770 card is named {medium!r}" -assert "frank-small" not in {s["name"] for s in services}, "retired frank-small still listed" -# The big endpoint keeps its card too, so a fleet overview shows both. -assert by_probe.get("127.0.0.1:7770") == "frank-large" +# Dankclaw serves two endpoints on two ports: frank-large (halogen, 7770) and +# frank-small (llama.cpp on the RTX 3090, 7771). +assert by_probe.get("127.0.0.1:7770") == "frank-large", by_probe.get("127.0.0.1:7770") +assert by_probe.get("127.0.0.1:7771") == "frank-small", by_probe.get("127.0.0.1:7771") PY } diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index 6654b55..7189013 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -295,17 +295,23 @@ PAIRS done } -@test "prometheus scrapes both LLM endpoints on the fleet's :7770" { +@test "prometheus scrapes every LLM endpoint that exports metrics" { # The LLM endpoints are the services whose behaviour this fleet argues about # most (KV pressure, prefix-cache reuse, queue depth), and vLLM exports all # of it. If a scrape target drifts from the endpoint the units publish, # nobody notices until someone needs the graph during an incident. local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-medium not scraped"; return 1; } - grep -q '"192.168.0.143:7770"' "$cfg" || { echo "frank-large not scraped"; return 1; } + grep -q '"192.168.0.143:7771"' "$cfg" || { echo "frank-small not scraped"; return 1; } # The medium endpoint is exported by the proxy on the host network; the # quadlet's own contract test pins that port, so name the job too. grep -q 'job_name: "frank-medium"' "$cfg" + grep -q 'job_name: "frank-small"' "$cfg" + # frank-large is NOT scraped on purpose: the halogen engine serves + # /health, /cache and /v1/* and no metrics endpoint, so a target here would + # be red by construction and would fire LlmEndpointDown forever. Assert the + # absence, so a well-meaning edit cannot re-add a permanently-red probe. + ! grep -q '192.168.0.143:7770' "$cfg" ! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg" }