feat(observability): per-GPU metrics for the mixed AMD/NVIDIA fleet
Add a host-process GPU exporter, scrape it on both servers, and graph it on the host-system dashboard. The fleet has four GPUs across three vendors of interest (2x R9700 on Dankserver, the 8060S iGPU + RTX 3090 on Dankclaw) and nothing exposed them until now. scripts/gpu-exporter.py reads each AMD card's amdgpu sysfs (gpu_busy_percent / mem_info_vram_* / hwmon temp1_input / power1_average) plus one nvidia-smi --query-gpu CSV, and emits unified gpu_* Prometheus families (utilization_percent, memory_used/total_bytes, temperature_celsius, power_watts) labelled gpu/vendor/model. Absent hardware yields ABSENT series, never a fabricated 0: the R9700s expose no sysfs power, so they simply carry no gpu_power_watts line. Both sources are injectable, so the seam runs on a GPU-less dev box; live-verified on Dankserver's real sysfs. It runs as a plain user unit (not a quadlet): it must read the host's own DRM sysfs and shell to nvidia-smi, which a container cannot do without GPU passthrough. Same posture as signal-relay. The base unit binds loopback; the deploy drop-in appends the host LAN address, never a wildcard (matching node-exporter/cadvisor). deploy_gpu_exporter installs/enables/health-gates it on both hosts and is wired into the planner (both targets), the test-changed router, and two Prometheus scrape jobs using the node/cadvisor reach rules (gateway on Dankclaw, LAN on Dankserver).
This commit is contained in:
1 parent
b35269bfa5
commit
b02647b992
9 files changed
+592
-9
No files matched your search
@@ -86,7 +86,7 @@ service_is_allowed() {
|
||||
dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
|
||||
dankclaw:comfyui|dankclaw:hermes|\
|
||||
dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
|
||||
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:alertmanager|\
|
||||
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:cadvisor|dankclaw:gpu-exporter|dankclaw:alertmanager|\
|
||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
|
||||
@@ -97,7 +97,7 @@ service_is_allowed() {
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
||||
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
|
||||
dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|\
|
||||
dankserver:frank-archon|dankserver:node-exporter|dankserver:cadvisor|dankserver:gpu-exporter|\
|
||||
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
||||
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
||||
frankbook-pro:readarr)
|
||||
@@ -275,6 +275,36 @@ install_signal_relay() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# Host-process GPU metrics exporter (amdgpu sysfs + nvidia-smi). Not a quadlet:
|
||||
# it runs python3 from the checkout on the host so it can read the DRM/cgroup
|
||||
# sysfs and shell out to nvidia-smi. Same two-address bind posture as
|
||||
# node-exporter/cadvisor (loopback + this host's LAN), so it is never a wildcard.
|
||||
deploy_gpu_exporter() {
|
||||
local unit_dir="$HOME/.config/systemd/user" lan dropin i
|
||||
case "$TARGET" in
|
||||
dankclaw) lan=192.168.0.143 ;;
|
||||
dankserver) lan=192.168.0.235 ;;
|
||||
*) echo "gpu-exporter: unknown target '$TARGET'" >&2; return 2 ;;
|
||||
esac
|
||||
mkdir -p "$unit_dir"
|
||||
cp scripts/gpu-exporter.service "$unit_dir/gpu-exporter.service"
|
||||
dropin="$unit_dir/gpu-exporter.service.d"
|
||||
mkdir -p "$dropin"
|
||||
printf '[Service]\nEnvironment=GPU_EXPORTER_BINDS=127.0.0.1,%s\n' "$lan" \
|
||||
> "$dropin/bind.conf"
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable gpu-exporter.service
|
||||
systemctl --user restart gpu-exporter.service
|
||||
# The interpreter needs a moment to bind after restart (the same race that
|
||||
# once failed a healthy signal-relay deploy).
|
||||
for i in $(seq 1 15); do
|
||||
curl -fsS -m 5 http://127.0.0.1:9401/metrics >/dev/null && return 0
|
||||
sleep 1
|
||||
done
|
||||
echo "gpu-exporter is not answering on 127.0.0.1:9401/metrics" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
copy_loki_config() {
|
||||
run_privileged mkdir -p /opt/loki/config
|
||||
run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
|
||||
@@ -2004,6 +2034,7 @@ deploy_service() {
|
||||
tautulli) deploy_tautulli ;;
|
||||
alloy) deploy_alloy ;;
|
||||
prometheus|grafana|loki|node-exporter|cadvisor|alertmanager) deploy_observability "$service" ;;
|
||||
gpu-exporter) deploy_gpu_exporter ;;
|
||||
comfyui) deploy_comfyui ;;
|
||||
*) deploy_compose_service "$service" ;;
|
||||
esac
|
||||
|
||||
Reference in new issue
Block a user