observability: provision Grafana from git — datasource plus the LLM dashboard
Grafana has been running with zero datasources and zero dashboards, so the metrics scraped last commit were only reachable through the Prometheus API. This wires the plumbing and one dashboard, both declared in shared/grafana/provisioning/ and installed to /opt/grafana/provisioning by the deployer (the unit already mounted that directory read-only, so nothing about the container changes): - datasources/prometheus.yml — Prometheus at http://prometheus:9090, the container name, not the loopback publish, with a fixed uid so dashboards reference it without a lookup. - dashboards/llm-fleet.json — KV pool fill, queue depth, prefix-cache reuse rate, e2e latency p95, and decode/prefill throughput, with an endpoint variable covering both frank-medium and frank-large. Every series name was checked against the live /metrics endpoint: this fleet already shipped one comment naming a metric that does not exist, and a dashboard full of those would be worse. Dashboards stay editable in the UI (allowUiUpdates) because the panels are a starting point, but the file is the source of truth and the deployer overwrites Grafana's copy on every deploy. Planner and seams follow the existing pattern: shared/grafana/** selects the grafana service, and the tests pin the container-name address plus the provisioning contract so a rename cannot silently empty the dashboard.
This commit is contained in:
Frank
@@ -250,6 +250,10 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
_add(services, "loki")
|
||||
elif group == "alloy":
|
||||
_add(services, "alloy")
|
||||
elif group == "grafana":
|
||||
# Provisioned datasources and dashboards: git is the source of
|
||||
# truth, so a change here must restart the service that reads them.
|
||||
_add(services, "grafana")
|
||||
elif group == "homepage":
|
||||
# Homepage is a Dankclaw-owned deployment. Shared homepage files
|
||||
# are intentionally not copied during a Dankserver-only run.
|
||||
|
||||
@@ -215,6 +215,22 @@ copy_alloy_config() {
|
||||
run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
|
||||
}
|
||||
|
||||
copy_grafana_provisioning() {
|
||||
# Datasources and dashboards are declared in git, never clicked into
|
||||
# grafana.db: this Grafana is an empty install, and a UI-configured
|
||||
# datasource would be invisible to review, lost with the data volume, and
|
||||
# unknown to anyone reading the repository. The unit already mounts
|
||||
# /opt/grafana/provisioning read-only, so this only has to populate it.
|
||||
run_privileged mkdir -p /opt/grafana/provisioning/datasources \
|
||||
/opt/grafana/provisioning/dashboards
|
||||
run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \
|
||||
/opt/grafana/provisioning/datasources/prometheus.yml
|
||||
run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \
|
||||
/opt/grafana/provisioning/dashboards/provider.yml
|
||||
run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \
|
||||
/opt/grafana/provisioning/dashboards/llm-fleet.json
|
||||
}
|
||||
|
||||
render_caddyfile() {
|
||||
local hash
|
||||
hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
|
||||
@@ -417,6 +433,7 @@ deploy_observability() {
|
||||
build_compose_image alloy
|
||||
;;
|
||||
grafana)
|
||||
copy_grafana_provisioning
|
||||
seed_grafana_env || return 1
|
||||
;;
|
||||
node-exporter)
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
{
|
||||
"uid": "llm-fleet",
|
||||
"title": "LLM fleet — capacity and reuse",
|
||||
"description": "The signals that explain frank-medium under load, from the 2026-09-16 capacity work: per-card KV fill, queue depth, prefix-cache reuse, and request latency. frank-large (llama.cpp) has throughput counters only. See serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md for the numbers these panels are read against.",
|
||||
"tags": ["llm", "frank-medium", "capacity"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "job",
|
||||
"label": "endpoint",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": { "query": "label_values(vllm:kv_cache_usage_perc, job)", "refId": "A" },
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": { "text": "All", "value": "$__all" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "KV pool fill (vllm:kv_cache_usage_perc)",
|
||||
"description": "How full each card's KV pool is. One card holds ~122,636 tokens; sustained high values with a cold prefix cache is the thrash this fleet measured.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"custom": { "fillOpacity": 10, "lineWidth": 1 }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "vllm:kv_cache_usage_perc{job=~\"$job\"} * 100",
|
||||
"legendFormat": "{{job}} engine {{engine}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Requests waiting (queue depth)",
|
||||
"description": "Anything above zero means the engine is serialising: with MAXSEQS=4 the fifth concurrent request waits.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1 } },
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "vllm:num_requests_waiting{job=~\"$job\"}",
|
||||
"legendFormat": "{{job}} waiting"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "vllm:num_requests_running{job=~\"$job\"}",
|
||||
"legendFormat": "{{job}} running"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "Prefix cache reuse rate",
|
||||
"description": "rate(hits)/rate(queries). High means pinned conversations are being served from cache (measured: 22.3 s cold vs 1.4 s warm on a ~50k-token prefix). A collapse here with three concurrent large contexts is the capacity wall, not a routing failure.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "fillOpacity": 10, "lineWidth": 1 } },
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "rate(vllm:prefix_cache_hits_total{job=~\"$job\"}[5m]) / clamp_min(rate(vllm:prefix_cache_queries_total{job=~\"$job\"}[5m]), 1)",
|
||||
"legendFormat": "{{job}} reuse"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "End-to-end request latency p95",
|
||||
"description": "Wall-clock per request, prefill included — the number an agent-loop turn feels.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "s", "custom": { "fillOpacity": 10, "lineWidth": 1 } },
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le, job) (rate(vllm:e2e_request_latency_seconds_bucket{job=~\"$job\"}[5m])))",
|
||||
"legendFormat": "{{job}} p95"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Decode and prefill throughput",
|
||||
"description": "vLLM generation tokens, and llama.cpp's counters for frank-large (llamacpp:prompt_tokens_seconds is prompt throughput).",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "tokens/sec", "custom": { "fillOpacity": 10, "lineWidth": 1 } },
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "rate(vllm:generation_tokens_total{job=~\"$job\"}[5m])",
|
||||
"legendFormat": "{{job}} generated tok/s"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "rate(vllm:prompt_tokens_total{job=~\"$job\"}[5m])",
|
||||
"legendFormat": "{{job}} prompt tok/s"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "llamacpp:predicted_tokens_seconds",
|
||||
"legendFormat": "{{job}} llama.cpp decode"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
# Dashboard provider: every JSON in this directory becomes a dashboard.
|
||||
#
|
||||
# allowUiUpdates is on because these panels are a starting point — an operator
|
||||
# may want a different rate window or a new panel, and forcing them to edit
|
||||
# JSON and redeploy would be the wrong trade for a three-person homelab. The
|
||||
# file name is the source of truth; Grafana's copy is generated state.
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: fleet
|
||||
orgId: 1
|
||||
folder: ""
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /etc/grafana/provisioning/dashboards
|
||||
@@ -0,0 +1,26 @@
|
||||
# Grafana datasource, declared in git rather than clicked into grafana.db.
|
||||
#
|
||||
# This Grafana is an empty install on purpose: every datasource and dashboard
|
||||
# it has comes from this directory, which the deployer installs into
|
||||
# /opt/grafana/provisioning (mounted read-only into the container). A
|
||||
# UI-configured datasource would be invisible to review, lost with the data
|
||||
# volume, and unknown to anyone reading the repository.
|
||||
#
|
||||
# The URL uses the container name on dankclaw_default — the same address Caddy
|
||||
# proxies to — not the published loopback port, so the path is the same from
|
||||
# inside the network as from the browser's server side.
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
# Fixed uid so dashboards can reference it without a lookup.
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
# Matches prometheus.yml's scrape_interval; without it Grafana's rate
|
||||
# windows can be shorter than the sample spacing.
|
||||
timeInterval: 15s
|
||||
@@ -299,3 +299,28 @@ PAIRS
|
||||
grep -q 'job_name: "frank-medium"' "$cfg"
|
||||
! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg"
|
||||
}
|
||||
|
||||
@test "grafana provisioning is declared in git and installed by the deployer" {
|
||||
# Datasources belong in the repository, not in grafana.db: a UI-configured
|
||||
# one is invisible to review and lost with the data volume.
|
||||
local ds="$REPO_ROOT/shared/grafana/provisioning/datasources/prometheus.yml"
|
||||
local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json"
|
||||
local prov="$REPO_ROOT/shared/grafana/provisioning/dashboards/provider.yml"
|
||||
[ -f "$ds" ] && [ -f "$dash" ] && [ -f "$prov" ]
|
||||
python3 -c "import json,sys; json.load(open('$dash'))"
|
||||
# Container-name address, like every other on-network scrape target: the
|
||||
# published port is loopback-only and would be wrong from Grafana.
|
||||
grep -q 'url: http://prometheus:9090' "$ds"
|
||||
grep -q 'uid: prometheus' "$ds"
|
||||
# The unit already mounts the directory; the deployer must fill it.
|
||||
grep -q '/opt/grafana/provisioning:/etc/grafana/provisioning:ro' \
|
||||
"$REPO_ROOT/dankclaw/observability/quadlet/grafana.container"
|
||||
grep -q 'copy_grafana_provisioning' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
}
|
||||
|
||||
@test "the planner selects grafana for shared/grafana changes" {
|
||||
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
||||
--paths shared/grafana/provisioning/datasources/prometheus.yml
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$output" = "grafana" ]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user