diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml index 200269d..6272cbd 100644 --- a/shared/prometheus/prometheus.yml +++ b/shared/prometheus/prometheus.yml @@ -53,3 +53,24 @@ scrape_configs: - targets: - "192.168.0.143:8081" # SearXNG (via the Gluetun tunnel) - "192.168.0.143:3005" # OpenWebUI + + # LLM fleet. Scraped on the LAN addresses, not the guarded vhosts: these are + # machine clients and Prometheus cannot solve an Anubis challenge. + # frank-medium is vLLM, and its series are the ones that explain behaviour + # under load — vllm:kv_cache_usage_perc (how full the pool is), + # vllm:prefix_cache_hit_rate (whether pinned conversations are being reused), + # vllm:num_requests_waiting (queue depth), and the request-latency histograms. + # frank-large is llama.cpp and exposes throughput counters under llamacpp:. + - job_name: "frank-medium" + metrics_path: "/metrics" + static_configs: + - targets: ["192.168.0.235:7770"] + labels: + service: frank-medium + + - job_name: "frank-large" + metrics_path: "/metrics" + static_configs: + - targets: ["192.168.0.143:7770"] + labels: + service: frank-large diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index 38d209b..ee175b5 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -285,3 +285,17 @@ PAIRS [ "$output" = "$svc" ] || { echo "$svc planned as '$output'"; return 1; } done } + +@test "prometheus scrapes both LLM endpoints on the fleet's :7770" { + # The LLM endpoints are the services whose behaviour this fleet argues about + # most (KV pressure, prefix-cache reuse, queue depth), and vLLM exports all + # of it. If a scrape target drifts from the endpoint the units publish, + # nobody notices until someone needs the graph during an incident. + local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml" + grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-medium not scraped"; return 1; } + grep -q '"192.168.0.143:7770"' "$cfg" || { echo "frank-large not scraped"; return 1; } + # The medium endpoint is exported by the proxy on the host network; the + # quadlet's own contract test pins that port, so name the job too. + grep -q 'job_name: "frank-medium"' "$cfg" + ! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg" +}