From 42b47063ec2e527f537e362f030113e5c76da1ce Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2026 06:32:33 -0700 Subject: [PATCH] observability: scrape the LLM fleet into Prometheus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Neither LLM endpoint was scraped, and frank-medium exports 418 vllm: series — including the ones this fleet argues about under load: kv_cache_usage_perc (how full the per-card pool is), prefix_cache_hit_rate (whether pinned conversations are being reused), num_requests_waiting (queue depth) and the request-latency histograms. Everything diagnosed tonight by hand — 122,636 KV tokens per card, thrash when three large contexts overlap, cold-vs-warm prefix reuse — is a graph now, on the LAN addresses rather than the guarded vhosts. frank-large comes along in the same change: llama.cpp exposes throughput counters under llamacpp:, and watching one LLM endpoint without the other was arbitrary. The seam added to observability_quadlet.bats pins the cross-file contract that breaks silently: if a scrape target drifts from the :7770 the units publish, the graphs vanish with no error anywhere. --- shared/prometheus/prometheus.yml | 21 +++++++++++++++++++++ tests/observability_quadlet.bats | 14 ++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index 200269d..6272cbd 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -53,3 +53,24 @@ scrape_configs: - targets: - "192.168.0.143:8081" # SearXNG (via the Gluetun tunnel) - "192.168.0.143:3005" # OpenWebUI + + # LLM fleet. Scraped on the LAN addresses, not the guarded vhosts: these are + # machine clients and Prometheus cannot solve an Anubis challenge. + # frank-medium is vLLM, and its series are the ones that explain behaviour + # under load — vllm:kv_cache_usage_perc (how full the pool is), + # vllm:prefix_cache_hit_rate (whether pinned conversations are being reused), + # vllm:num_requests_waiting (queue depth), and the request-latency histograms. + # frank-large is llama.cpp and exposes throughput counters under llamacpp:. + - job_name: "frank-medium" + metrics_path: "/metrics" + static_configs: + - targets: ["192.168.0.235:7770"] + labels: + service: frank-medium + + - job_name: "frank-large" + metrics_path: "/metrics" + static_configs: + - targets: ["192.168.0.143:7770"] + labels: + service: frank-large diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index 38d209b..ee175b5 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -285,3 +285,17 @@ PAIRS [ "$output" = "$svc" ] || { echo "$svc planned as '$output'"; return 1; } done } + +@test "prometheus scrapes both LLM endpoints on the fleet's :7770" { + # The LLM endpoints are the services whose behaviour this fleet argues about + # most (KV pressure, prefix-cache reuse, queue depth), and vLLM exports all + # of it. If a scrape target drifts from the endpoint the units publish, + # nobody notices until someone needs the graph during an incident. + local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" + grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-medium not scraped"; return 1; } + grep -q '"192.168.0.143:7770"' "$cfg" || { echo "frank-large not scraped"; return 1; } + # The medium endpoint is exported by the proxy on the host network; the + # quadlet's own contract test pins that port, so name the job too. + grep -q 'job_name: "frank-medium"' "$cfg" + ! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg" +}