observability: provision Grafana from git — datasource plus the LLM dashboard
Grafana has been running with zero datasources and zero dashboards, so the metrics scraped last commit were only reachable through the Prometheus API. This wires the plumbing and one dashboard, both declared in shared/grafana/provisioning/ and installed to /opt/grafana/provisioning by the deployer (the unit already mounted that directory read-only, so nothing about the container changes): - datasources/prometheus.yml — Prometheus at http://prometheus:9090, the container name, not the loopback publish, with a fixed uid so dashboards reference it without a lookup. - dashboards/llm-fleet.json — KV pool fill, queue depth, prefix-cache reuse rate, e2e latency p95, and decode/prefill throughput, with an endpoint variable covering both frank-medium and frank-large. Every series name was checked against the live /metrics endpoint: this fleet already shipped one comment naming a metric that does not exist, and a dashboard full of those would be worse. Dashboards stay editable in the UI (allowUiUpdates) because the panels are a starting point, but the file is the source of truth and the deployer overwrites Grafana's copy on every deploy. Planner and seams follow the existing pattern: shared/grafana/** selects the grafana service, and the tests pin the container-name address plus the provisioning contract so a rename cannot silently empty the dashboard.
This commit is contained in:
@@ -215,6 +215,22 @@ copy_alloy_config() {
|
||||
run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
|
||||
}
|
||||
|
||||
copy_grafana_provisioning() {
|
||||
# Datasources and dashboards are declared in git, never clicked into
|
||||
# grafana.db: this Grafana is an empty install, and a UI-configured
|
||||
# datasource would be invisible to review, lost with the data volume, and
|
||||
# unknown to anyone reading the repository. The unit already mounts
|
||||
# /opt/grafana/provisioning read-only, so this only has to populate it.
|
||||
run_privileged mkdir -p /opt/grafana/provisioning/datasources \
|
||||
/opt/grafana/provisioning/dashboards
|
||||
run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \
|
||||
/opt/grafana/provisioning/datasources/prometheus.yml
|
||||
run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \
|
||||
/opt/grafana/provisioning/dashboards/provider.yml
|
||||
run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \
|
||||
/opt/grafana/provisioning/dashboards/llm-fleet.json
|
||||
}
|
||||
|
||||
render_caddyfile() {
|
||||
local hash
|
||||
hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
|
||||
@@ -417,6 +433,7 @@ deploy_observability() {
|
||||
build_compose_image alloy
|
||||
;;
|
||||
grafana)
|
||||
copy_grafana_provisioning
|
||||
seed_grafana_env || return 1
|
||||
;;
|
||||
node-exporter)
|
||||
|
||||
Reference in New Issue
Block a user