observability: provision Grafana from git — datasource plus the LLM dashboard

Grafana has been running with zero datasources and zero dashboards, so the
metrics scraped last commit were only reachable through the Prometheus API.
This wires the plumbing and one dashboard, both declared in
shared/grafana/provisioning/ and installed to /opt/grafana/provisioning by the
deployer (the unit already mounted that directory read-only, so nothing about
the container changes):

- datasources/prometheus.yml — Prometheus at http://prometheus:9090, the
  container name, not the loopback publish, with a fixed uid so dashboards
  reference it without a lookup.
- dashboards/llm-fleet.json — KV pool fill, queue depth, prefix-cache reuse
  rate, e2e latency p95, and decode/prefill throughput, with an endpoint
  variable covering both frank-medium and frank-large. Every series name was
  checked against the live /metrics endpoint: this fleet already shipped one
  comment naming a metric that does not exist, and a dashboard full of those
  would be worse.

Dashboards stay editable in the UI (allowUiUpdates) because the panels are a
starting point, but the file is the source of truth and the deployer
overwrites Grafana's copy on every deploy.

Planner and seams follow the existing pattern: shared/grafana/** selects the
grafana service, and the tests pin the container-name address plus the
provisioning contract so a rename cannot silently empty the dashboard.
This commit is contained in:
Frank
2026-09-16 06:39:15 -07:00
parent 3037240f4f
commit 6a191430ca
6 changed files with 232 additions and 0 deletions
+17
View File
@@ -215,6 +215,22 @@ copy_alloy_config() {
run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
}
copy_grafana_provisioning() {
# Datasources and dashboards are declared in git, never clicked into
# grafana.db: this Grafana is an empty install, and a UI-configured
# datasource would be invisible to review, lost with the data volume, and
# unknown to anyone reading the repository. The unit already mounts
# /opt/grafana/provisioning read-only, so this only has to populate it.
run_privileged mkdir -p /opt/grafana/provisioning/datasources \
/opt/grafana/provisioning/dashboards
run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \
/opt/grafana/provisioning/datasources/prometheus.yml
run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \
/opt/grafana/provisioning/dashboards/provider.yml
run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \
/opt/grafana/provisioning/dashboards/llm-fleet.json
}
render_caddyfile() {
local hash
hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
@@ -417,6 +433,7 @@ deploy_observability() {
build_compose_image alloy
;;
grafana)
copy_grafana_provisioning
seed_grafana_env || return 1
;;
node-exporter)