diff --git a/shared/grafana/provisioning/dashboards/llm-fleet.json b/shared/grafana/provisioning/dashboards/llm-fleet.json index 865974a..325abcf 100644 --- a/shared/grafana/provisioning/dashboards/llm-fleet.json +++ b/shared/grafana/provisioning/dashboards/llm-fleet.json @@ -1,7 +1,7 @@ { "uid": "llm-fleet", "title": "LLM fleet — capacity and reuse", - "description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-large (llama.cpp) has throughput counters only. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.", + "description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-small (llama.cpp on Dankclaw's RTX 3090) exports throughput counters and appears in the same endpoint picker; frank-large runs the halogen engine, which publishes /health, /cache and /v1/* and no metrics endpoint at all, so it cannot appear here. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.", "tags": [ "llm", "frank-medium", @@ -26,7 +26,7 @@ "uid": "prometheus" }, "query": { - "query": "label_values(vllm:kv_cache_usage_perc, job)", + "query": "label_values({__name__=~\"vllm:kv_cache_usage_perc|llamacpp:.*\"}, job)", "refId": "A" }, "includeAll": true, @@ -185,7 +185,7 @@ "id": 5, "type": "timeseries", "title": "Decode and prefill throughput", - "description": "vLLM generation tokens, and llama.cpp's counters for frank-large (llamacpp:prompt_tokens_seconds is prompt throughput).", + "description": "vLLM generation tokens, plus llama.cpp's counters for frank-small: llamacpp:predicted_tokens_seconds is decode and llamacpp:prompt_tokens_seconds is prompt throughput. The llama.cpp series carry the job label of their scrape, so the endpoint picker selects between the runtimes.", "gridPos": { "h": 8, "w": 12, @@ -219,8 +219,13 @@ }, { "refId": "C", - "expr": "llamacpp:predicted_tokens_seconds", - "legendFormat": "{{job}} llama.cpp decode" + "expr": "llamacpp:predicted_tokens_seconds{job=~\"$job\"}", + "legendFormat": "{{job}} llama.cpp decode tok/s" + }, + { + "refId": "D", + "expr": "llamacpp:prompt_tokens_seconds{job=~\"$job\"}", + "legendFormat": "{{job}} llama.cpp prefill tok/s" } ] }, diff --git a/shared/homepage/services.json b/shared/homepage/services.json index a686de0..c6aa557 100644 --- a/shared/homepage/services.json +++ b/shared/homepage/services.json @@ -222,7 +222,7 @@ { "name": "frank-large", "category": "admin", - "description": "The large LLM (Qwen3.8-Flash-Next Uncensored, ik_llama.cpp) on Dankclaw.", + "description": "The large LLM (Qwen3.8-Flash-Next w4b, halogen-flash-server on the Strix Halo iGPU). Two 200k-token conversations resident.", "url": "", "docs": "", "probe": "127.0.0.1:7770", @@ -241,6 +241,17 @@ "wip": false, "showLink": false }, + { + "name": "frank-small", + "category": "admin", + "description": "The small LLM (Qwen3.8-27B GSQ-RCO IQ3_S with MTP and vision, llama.cpp) on Dankclaw's RTX 3090. The fleet's image path.", + "url": "", + "docs": "", + "probe": "127.0.0.1:7771", + "method": "http", + "wip": false, + "showLink": false + }, { "name": "mariadb", "category": "admin", diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index f93584f..76b4fad 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -68,7 +68,15 @@ scrape_configs: # and the prefix-cache pair vllm:prefix_cache_hits_total over # vllm:prefix_cache_queries_total (their rates give the hit rate, which says # whether pinned conversations are actually being reused). - # frank-large is llama.cpp and exposes throughput counters under llamacpp:. + # frank-large is deliberately NOT scraped: the halogen engine that serves it + # publishes /health, /cache and /v1/* and no Prometheus text endpoint, so a + # target here would be red by construction and would fire LlmEndpointDown + # forever. Its liveness is on the homepage (a loopback probe) and in the + # engine's own boot ledger. The remedy, when the fleet wants it, is the + # blackbox exporter already wanted for the comfyui/services probes. + # + # frank-small is llama.cpp (its serve script passes --metrics) and does + # export text-format counters under llamacpp: on the port its unit publishes. - job_name: "frank-medium" metrics_path: "/metrics" static_configs: @@ -76,9 +84,9 @@ scrape_configs: labels: service: frank-medium - - job_name: "frank-large" + - job_name: "frank-small" metrics_path: "/metrics" static_configs: - - targets: ["192.168.0.143:7770"] + - targets: ["192.168.0.143:7771"] labels: - service: frank-large + service: frank-small diff --git a/shared/prometheus/rules/llm-fleet.yml b/shared/prometheus/rules/llm-fleet.yml index 51fd1bc..e507ad5 100644 --- a/shared/prometheus/rules/llm-fleet.yml +++ b/shared/prometheus/rules/llm-fleet.yml @@ -75,9 +75,13 @@ groups: annotations: summary: "{{ $labels.job }}: scrape target down for 2 minutes" description: >- - Prometheus cannot reach the LLM endpoint. Check the host's units: - systemctl --user is-active frank-medium-{a,b,pro} on Dankclaw's - peers (frank-large) or Dankserver (frank-medium). + Prometheus cannot reach the LLM endpoint. Check the units: + `systemctl --user is-active frank-medium-{a,b,proxy}` on Dankserver, + `systemctl is-active frank-small` (the llama.cpp root unit) or + `systemctl --user is-active frank-large` (the halogen quadlet) on + Dankclaw. frank-large is not scraped at all — the halogen engine + exports no metrics — so this alert can only ever name the endpoints + that do export one. # What an operator actually feels: wall-clock per request, prefill # included. A cold 50k-token prefill alone is ~22 s on one card. diff --git a/tests/homepage_config.bats b/tests/homepage_config.bats index 803e4dc..a5312b1 100644 --- a/tests/homepage_config.bats +++ b/tests/homepage_config.bats @@ -41,18 +41,19 @@ PY } @test "the LLM status cards track the fleet, not retired service names" { - # This drifted: the card probing Dankserver's :7770 was still named - # frank-small long after frank-small was retired and frank-medium took that - # endpoint, so the homepage labelled the fleet's busiest service with a dead - # name. The probe target was right; only the identity was wrong. + # This drifted twice: the card probing Dankserver's :7770 was still named + # frank-small long after frank-medium took that endpoint, and Dankclaw's + # second endpoint now exists again as a real service on its own port. The + # probe target is what matters; the name has to follow it. python3 - "$REPO_ROOT/shared/homepage/services.json" <<'PY' import json, sys services = json.load(open(sys.argv[1])) by_probe = {s["probe"]: s["name"] for s in services} medium = by_probe.get("192.168.0.235:7770") assert medium == "frank-medium", f"Dankserver :7770 card is named {medium!r}" -assert "frank-small" not in {s["name"] for s in services}, "retired frank-small still listed" -# The big endpoint keeps its card too, so a fleet overview shows both. -assert by_probe.get("127.0.0.1:7770") == "frank-large" +# Dankclaw serves two endpoints on two ports: frank-large (halogen, 7770) and +# frank-small (llama.cpp on the RTX 3090, 7771). +assert by_probe.get("127.0.0.1:7770") == "frank-large", by_probe.get("127.0.0.1:7770") +assert by_probe.get("127.0.0.1:7771") == "frank-small", by_probe.get("127.0.0.1:7771") PY } diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index 6654b55..7189013 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -295,17 +295,23 @@ PAIRS done } -@test "prometheus scrapes both LLM endpoints on the fleet's :7770" { +@test "prometheus scrapes every LLM endpoint that exports metrics" { # The LLM endpoints are the services whose behaviour this fleet argues about # most (KV pressure, prefix-cache reuse, queue depth), and vLLM exports all # of it. If a scrape target drifts from the endpoint the units publish, # nobody notices until someone needs the graph during an incident. local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-medium not scraped"; return 1; } - grep -q '"192.168.0.143:7770"' "$cfg" || { echo "frank-large not scraped"; return 1; } + grep -q '"192.168.0.143:7771"' "$cfg" || { echo "frank-small not scraped"; return 1; } # The medium endpoint is exported by the proxy on the host network; the # quadlet's own contract test pins that port, so name the job too. grep -q 'job_name: "frank-medium"' "$cfg" + grep -q 'job_name: "frank-small"' "$cfg" + # frank-large is NOT scraped on purpose: the halogen engine serves + # /health, /cache and /v1/* and no metrics endpoint, so a target here would + # be red by construction and would fire LlmEndpointDown forever. Assert the + # absence, so a well-meaning edit cannot re-add a permanently-red probe. + ! grep -q '192.168.0.143:7770' "$cfg" ! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg" }