diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index 3a03454..40e249a 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -250,6 +250,10 @@ def services_for_path(path: str, target: str) -> set[str]: _add(services, "loki") elif group == "alloy": _add(services, "alloy") + elif group == "grafana": + # Provisioned datasources and dashboards: git is the source of + # truth, so a change here must restart the service that reads them. + _add(services, "grafana") elif group == "homepage": # Homepage is a Dankclaw-owned deployment. Shared homepage files # are intentionally not copied during a Dankserver-only run. diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index b9f93c6..f74b20a 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -215,6 +215,22 @@ copy_alloy_config() { run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy } +copy_grafana_provisioning() { + # Datasources and dashboards are declared in git, never clicked into + # grafana.db: this Grafana is an empty install, and a UI-configured + # datasource would be invisible to review, lost with the data volume, and + # unknown to anyone reading the repository. The unit already mounts + # /opt/grafana/provisioning read-only, so this only has to populate it. + run_privileged mkdir -p /opt/grafana/provisioning/datasources \ + /opt/grafana/provisioning/dashboards + run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \ + /opt/grafana/provisioning/datasources/prometheus.yml + run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \ + /opt/grafana/provisioning/dashboards/provider.yml + run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \ + /opt/grafana/provisioning/dashboards/llm-fleet.json +} + render_caddyfile() { local hash hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)" @@ -417,6 +433,7 @@ deploy_observability() { build_compose_image alloy ;; grafana) + copy_grafana_provisioning seed_grafana_env || return 1 ;; node-exporter) diff --git a/shared/grafana/provisioning/dashboards/llm-fleet.json b/shared/grafana/provisioning/dashboards/llm-fleet.json new file mode 100644 index 0000000..da18275 --- /dev/null +++ b/shared/grafana/provisioning/dashboards/llm-fleet.json @@ -0,0 +1,142 @@ +{ + "uid": "llm-fleet", + "title": "LLM fleet — capacity and reuse", + "description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-large (llama.cpp) has throughput counters only. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.", + "tags": ["llm", "frank-medium", "capacity"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { "from": "now-6h", "to": "now" }, + "templating": { + "list": [ + { + "name": "job", + "label": "endpoint", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": { "query": "label_values(vllm:kv_cache_usage_perc, job)", "refId": "A" }, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "KV pool fill (vllm:kv_cache_usage_perc)", + "description": "How full each card's KV pool is. One card holds ~122,636 tokens; sustained high values with a cold prefix cache is the thrash this fleet measured.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { "fillOpacity": 10, "lineWidth": 1 } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "vllm:kv_cache_usage_perc{job=~\"$job\"} * 100", + "legendFormat": "{{job}} engine {{engine}}" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Requests waiting (queue depth)", + "description": "Anything above zero means the engine is serialising: with MAXSEQS=4 the fifth concurrent request waits.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1 } }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "vllm:num_requests_waiting{job=~\"$job\"}", + "legendFormat": "{{job}} waiting" + }, + { + "refId": "B", + "expr": "vllm:num_requests_running{job=~\"$job\"}", + "legendFormat": "{{job}} running" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Prefix cache reuse rate", + "description": "rate(hits)/rate(queries). High means pinned conversations are being served from cache (measured: 22.3 s cold vs 1.4 s warm on a ~50k-token prefix). A collapse here with three concurrent large contexts is the capacity wall, not a routing failure.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 10, "lineWidth": 1 } }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "rate(vllm:prefix_cache_hits_total{job=~\"$job\"}[5m]) / clamp_min(rate(vllm:prefix_cache_queries_total{job=~\"$job\"}[5m]), 1)", + "legendFormat": "{{job}} reuse" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "End-to-end request latency p95", + "description": "Wall-clock per request, prefill included — the number an agent-loop turn feels.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "unit": "s", "custom": { "fillOpacity": 10, "lineWidth": 1 } }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, job) (rate(vllm:e2e_request_latency_seconds_bucket{job=~\"$job\"}[5m])))", + "legendFormat": "{{job}} p95" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Decode and prefill throughput", + "description": "vLLM generation tokens, and llama.cpp's counters for frank-large (llamacpp:prompt_tokens_seconds is prompt throughput).", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "unit": "tokens/sec", "custom": { "fillOpacity": 10, "lineWidth": 1 } }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "rate(vllm:generation_tokens_total{job=~\"$job\"}[5m])", + "legendFormat": "{{job}} generated tok/s" + }, + { + "refId": "B", + "expr": "rate(vllm:prompt_tokens_total{job=~\"$job\"}[5m])", + "legendFormat": "{{job}} prompt tok/s" + }, + { + "refId": "C", + "expr": "llamacpp:predicted_tokens_seconds", + "legendFormat": "{{job}} llama.cpp decode" + } + ] + } + ] +} diff --git a/shared/grafana/provisioning/dashboards/provider.yml b/shared/grafana/provisioning/dashboards/provider.yml new file mode 100644 index 0000000..c6c685d --- /dev/null +++ b/shared/grafana/provisioning/dashboards/provider.yml @@ -0,0 +1,18 @@ +# Dashboard provider: every JSON in this directory becomes a dashboard. +# +# allowUiUpdates is on because these panels are a starting point — an operator +# may want a different rate window or a new panel, and forcing them to edit +# JSON and redeploy would be the wrong trade for a three-person homelab. The +# file name is the source of truth; Grafana's copy is generated state. +apiVersion: 1 + +providers: + - name: fleet + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/shared/grafana/provisioning/datasources/prometheus.yml b/shared/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..f776c2c --- /dev/null +++ b/shared/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,26 @@ +# Grafana datasource, declared in git rather than clicked into grafana.db. +# +# This Grafana is an empty install on purpose: every datasource and dashboard +# it has comes from this directory, which the deployer installs into +# /opt/grafana/provisioning (mounted read-only into the container). A +# UI-configured datasource would be invisible to review, lost with the data +# volume, and unknown to anyone reading the repository. +# +# The URL uses the container name on dankclaw_default — the same address Caddy +# proxies to — not the published loopback port, so the path is the same from +# inside the network as from the browser's server side. +apiVersion: 1 + +datasources: + - name: Prometheus + # Fixed uid so dashboards can reference it without a lookup. + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + # Matches prometheus.yml's scrape_interval; without it Grafana's rate + # windows can be shorter than the sample spacing. + timeInterval: 15s diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index ee175b5..b74cdbd 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -299,3 +299,28 @@ PAIRS grep -q 'job_name: "frank-medium"' "$cfg" ! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg" } + +@test "grafana provisioning is declared in git and installed by the deployer" { + # Datasources belong in the repository, not in grafana.db: a UI-configured + # one is invisible to review and lost with the data volume. + local ds="$REPO_ROOT/shared/grafana/provisioning/datasources/prometheus.yml" + local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json" + local prov="$REPO_ROOT/shared/grafana/provisioning/dashboards/provider.yml" + [ -f "$ds" ] && [ -f "$dash" ] && [ -f "$prov" ] + python3 -c "import json,sys; json.load(open('$dash'))" + # Container-name address, like every other on-network scrape target: the + # published port is loopback-only and would be wrong from Grafana. + grep -q 'url: http://prometheus:9090' "$ds" + grep -q 'uid: prometheus' "$ds" + # The unit already mounts the directory; the deployer must fill it. + grep -q '/opt/grafana/provisioning:/etc/grafana/provisioning:ro' \ + "$REPO_ROOT/dankclaw/observability/quadlet/grafana.container" + grep -q 'copy_grafana_provisioning' "$REPO_ROOT/scripts/deploy-remote.sh" +} + +@test "the planner selects grafana for shared/grafana changes" { + run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \ + --paths shared/grafana/provisioning/datasources/prometheus.yml + [ "$status" -eq 0 ] + [ "$output" = "grafana" ] +}