Frank
650b6e0281
Two shipped-2026-10-20 bugs that left gpu-dankclaw, gpu-dankserver and cadvisor-dankclaw DOWN beside a green deploy: - gpu-exporter.py built one ThreadingHTTPServer per bind but ran serve_forever() on servers[0] only: the LAN-class socket sat listen()ing but never accept()ed, loopback deploy gate passed, every scrape that reached the other bind timed out on the SYN backlog. Now one thread per extra bind. Regression test spins --serve on two loopback binds and requires BOTH to answer /metrics. - cadvisor-dankclaw scraped its own host's LAN IP. Measured from inside the Prometheus container: a bridge container cannot dial its own host's LAN address at all (hairpin refuses published ports too); the podman gateway answers. Target moved to host.containers.internal:8080; the seam that pinned the LAN address (asserting the wrong doctrine) is re-pinned to the gateway plus a negative pin against the own-host LAN dial. Observability page records the measured reachability law replacing the inferred one.
647 lines
34 KiB
Bash
Executable File
647 lines
34 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the five observability Quadlet units on dankclaw.
|
|
#
|
|
# prometheus, grafana, loki, alloy and node-exporter ran as compose services
|
|
# until 2026-09-11 and did not survive a reboot — the 2026-09-11 reboot for the
|
|
# amd_iommu=off change is what exposed it. They are rootless Quadlet units now.
|
|
# This seam pins the properties a reader cannot check by eye:
|
|
#
|
|
# * boot persistence and self-healing on every unit;
|
|
# * the published ports bind 127.0.0.1 and never a wildcard address, so no
|
|
# unauthenticated metrics/dashboard/log endpoint is reachable off-host —
|
|
# the whole point of moving them off compose, which published on 0.0.0.0;
|
|
# * every port that IS published has a named consumer (the host-side homepage
|
|
# status timer, or Prometheus scraping the host);
|
|
# * the container names still match what Caddy proxies to and what
|
|
# Prometheus' own scrape config targets, so the cutover broke no wiring;
|
|
# * the deployer installs, restarts and health-checks them, and never tries
|
|
# to `enable` a generated unit.
|
|
|
|
load helpers
|
|
|
|
UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager)
|
|
|
|
# container_section <unit> — the [Container] key/value block only
|
|
container_section() {
|
|
awk '/^\[Container\]/{f=1;next} /^\[/{f=0} f' "$1"
|
|
}
|
|
|
|
# service_section <unit> — the [Service] key/value block only
|
|
service_section() {
|
|
awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
|
|
}
|
|
|
|
@test "all seven observability units exist under dankclaw/observability/quadlet/" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
[ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "every observability unit is boot-persistent" {
|
|
# Generated units cannot be enabled on this host's podman quadlet generator;
|
|
# [Install] + linger is the boot-persistence story for the rootless five.
|
|
# Alloy is a ROOT system unit, so its [Install] targets multi-user.target.
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
[[ "$svc" == alloy ]] && continue
|
|
grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc is not boot-persistent"; return 1; }
|
|
done
|
|
grep -q '^WantedBy=multi-user.target$' "$UNIT_DIR/alloy.container" \
|
|
|| { echo "alloy (root unit) is not boot-persistent"; return 1; }
|
|
}
|
|
|
|
@test "every observability unit is self-healing" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
grep -q '^Restart=always$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no Restart policy"; return 1; }
|
|
grep -q '^RestartSec=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no RestartSec"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "no observability unit publishes on a wildcard address" {
|
|
# The compose services published 9090, 3002 and 3100 on 0.0.0.0 with no
|
|
# authentication of any kind. Anything reachable off-host must go through
|
|
# the guarded franksdank.cloud vhosts instead.
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc publishes on a wildcard address"; return 1; }
|
|
done
|
|
! grep -qE '^PublishPort=' "$UNIT_DIR/alloy.container" || true
|
|
}
|
|
|
|
@test "the four HTTP UIs publish on loopback only" {
|
|
grep -q '^PublishPort=127.0.0.1:9090:9090$' "$UNIT_DIR/prometheus.container"
|
|
grep -q '^PublishPort=127.0.0.1:3002:3000$' "$UNIT_DIR/grafana.container"
|
|
grep -q '^PublishPort=127.0.0.1:3100:3100$' "$UNIT_DIR/loki.container"
|
|
grep -q '^PublishPort=127.0.0.1:9093:9093$' "$UNIT_DIR/alertmanager.container"
|
|
}
|
|
|
|
@test "the homepage status timer's probe targets are the published loopback ports" {
|
|
# scripts/homepage-status.py runs on the host and probes the `probe` field
|
|
# of services.json from there, so a probe and a published port that
|
|
# disagree show the card as down forever.
|
|
local registry="$REPO_ROOT/shared/homepage/services.json"
|
|
grep -q '"probe": "127.0.0.1:9090"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:3002"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:3100"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:9100"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:12345"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:9093"' "$registry"
|
|
}
|
|
|
|
@test "node-exporter binds the host namespace and both of its consumers" {
|
|
# It reports the host's own interfaces/cgroups, so it needs the host netns
|
|
# and host PID namespace; and it must listen exactly where its two
|
|
# consumers are: the homepage timer on loopback, Prometheus (in a
|
|
# container) on the LAN address.
|
|
local unit="$UNIT_DIR/node-exporter.container"
|
|
grep -q '^Network=host$' "$unit"
|
|
grep -q -- '--pid=host' "$unit"
|
|
grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
|
|
grep -q -- '--web.listen-address=192.168.0.143:9100' "$unit"
|
|
! grep -qE '^PublishPort=' "$unit"
|
|
}
|
|
|
|
@test "prometheus scrapes node-exporter where node-exporter actually listens" {
|
|
# Cross-file contract: if either side's address changes alone, host metrics
|
|
# silently stop with no error anywhere.
|
|
#
|
|
# This Prometheus is a bridge container (Network=dankclaw_default), and on
|
|
# this netavark host a container can only reach the host's LAN IP at a
|
|
# PUBLISHED port. node-exporter is net=host, so its 192.168.0.143:9100 bind
|
|
# is a host bind, not a published port — unreachable at its own LAN address.
|
|
# The node-dankclaw job therefore scrapes host.containers.internal:9100,
|
|
# which netavark delivers to the host's LAN IP, where node-exporter listens.
|
|
# (node-dankserver is a different machine, so its LAN address is reached
|
|
# over the real network and is scraped directly.)
|
|
local unit="$UNIT_DIR/node-exporter.container"
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
local lan
|
|
lan=$(grep -oE '192\.168\.0\.[0-9]+:9100' "$unit" | head -1)
|
|
[ -n "$lan" ] || { echo "node-exporter has no LAN listen address"; return 1; }
|
|
# The exporter must still bind that LAN address — it is where the
|
|
# host.containers.internal delivery lands.
|
|
grep -q -- "--web.listen-address=$lan" "$unit" \
|
|
|| { echo "node-exporter no longer binds $lan"; return 1; }
|
|
# The local job scrapes via the gateway, not the (unreachable) LAN address.
|
|
grep -q '"host.containers.internal:9100"' "$cfg" \
|
|
|| { echo "node-dankclaw does not scrape host.containers.internal:9100"; return 1; }
|
|
}
|
|
|
|
@test "container names still match what Caddy and Prometheus address by name" {
|
|
# Caddy's post-auth handlers proxy to prometheus:9090 / grafana:3000 /
|
|
# loki:3100 on the shared network, and Prometheus scrapes grafana:3000 and
|
|
# loki:3100 the same way. A renamed container breaks all of it silently.
|
|
grep -qE '^[[:space:]]*@grafana host grafana\.franksdank\.cloud$' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy grafana:3000' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
local svc
|
|
for svc in prometheus grafana loki; do
|
|
grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc unit renames its container"; return 1; }
|
|
grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc is not on the network Caddy and its peers use"; return 1; }
|
|
done
|
|
# Alloy is not on that network: it is a root container on Network=host, the
|
|
# only way to reach Loki's 127.0.0.1:3100 publish from outside a bridge.
|
|
grep -q '^ContainerName=alloy$' "$UNIT_DIR/alloy.container"
|
|
grep -q '^Network=host$' "$UNIT_DIR/alloy.container" \
|
|
|| { echo "alloy must be Network=host to reach Loki on the host loopback"; return 1; }
|
|
grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q 'url = "http://127.0.0.1:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
|
|
}
|
|
|
|
@test "every observability unit has a health check" {
|
|
# A container without one shows as plainly "Up" in podman ps while its
|
|
# process is wedged, and the deploy has nothing to wait on.
|
|
#
|
|
# Four units declare it inline. loki cannot: its image is distroless, so a
|
|
# unit HealthCmd would reach podman as CMD-SHELL and fail for want of
|
|
# /bin/sh. Its health command ships in the image instead.
|
|
local svc
|
|
for svc in prometheus grafana alloy node-exporter alertmanager; do
|
|
grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no HealthCmd"; return 1; }
|
|
grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no HealthRetries"; return 1; }
|
|
done
|
|
! grep -q '^HealthCmd=' "$UNIT_DIR/loki.container"
|
|
}
|
|
|
|
@test "loki's health command is exec form in the image, not a shell string" {
|
|
# The whole reason the loki image exists: distroless has no shell, so the
|
|
# command must be a JSON array that podman execs directly.
|
|
local cf="$REPO_ROOT/dankclaw/loki/Containerfile"
|
|
grep -q '^HEALTHCHECK ' "$cf"
|
|
grep -q 'CMD \["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"\]' "$cf"
|
|
# ... and the busybox it execs has to be in the image.
|
|
grep -q 'COPY --from=tools /bin/busybox.static /usr/local/bin/busybox' "$cf"
|
|
}
|
|
|
|
@test "the loki/alloy/comfyui build uses the docker image format" {
|
|
# Without --format docker the loki healthcheck is silently discarded (OCI
|
|
# has no HEALTHCHECK field) and the container stops reporting health.
|
|
grep -q -- '--format docker' "$REMOTE_DEPLOY"
|
|
grep -q 'podman build --format docker --no-cache' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "grafana gets its OAuth client secret without getting a whole env file" {
|
|
# systemd expands ${GRAFANA_OAUTH_CLIENT_SECRET} for the [Container]
|
|
# Environment= line from the [Service] EnvironmentFile, which keeps the
|
|
# file out of the container entirely. That file is the per-service one the
|
|
# deployer seeds (fleet convention: caddy, pihole, copyparty, romm, seerr)
|
|
# — NOT a repo .env. The repo-root .env the compose service happened to
|
|
# read is a gluetun env file carrying HOME=/root and the WireGuard keys;
|
|
# importing it sets HOME for this service and podman then looks for its
|
|
# configuration under /root and refuses to start.
|
|
local unit="$UNIT_DIR/grafana.container"
|
|
service_section "$unit" | grep -q '^EnvironmentFile=/home/dank/Containers/grafana/env$'
|
|
! service_section "$unit" | grep -q 'serverAppConfigs'
|
|
! container_section "$unit" | grep -q '^EnvironmentFile='
|
|
container_section "$unit" | grep -q '^Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=\${GRAFANA_OAUTH_CLIENT_SECRET}$'
|
|
}
|
|
|
|
@test "the deployer seeds exactly one value into the grafana env file" {
|
|
# The single-variable filter is the point: a whole-file copy is how the
|
|
# HOME=/root failure got in.
|
|
grep -q '^seed_grafana_env() {' "$REMOTE_DEPLOY"
|
|
grep -q '^ local dest=/home/dank/Containers/grafana/env$' "$REMOTE_DEPLOY"
|
|
grep -q "awk -F= '/^GRAFANA_OAUTH_CLIENT_SECRET=/ { print }' dankclaw/.env > \"\$dest\"" "$REMOTE_DEPLOY"
|
|
grep -q "grep -q '\^GRAFANA_OAUTH_CLIENT_SECRET=\.' \"\$dest\"" "$REMOTE_DEPLOY"
|
|
grep -q 'seed_grafana_env || return 1' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "observability units hold no literal secret material" {
|
|
# A secret-named setting is allowed only when its value is a ${VAR}
|
|
# reference resolved from an EnvironmentFile systemd reads; a literal value
|
|
# would be a tracked credential. Keys that merely mention a secret word
|
|
# (GF_AUTH_GENERIC_OAUTH_TOKEN_URL, ..._CLIENT_ID) are not secrets, so the
|
|
# match is anchored to the end of the key.
|
|
local svc unit line key value
|
|
for svc in "${UNITS[@]}"; do
|
|
unit="$UNIT_DIR/$svc.container"
|
|
while IFS= read -r line; do
|
|
key="${line#Environment=}"
|
|
value="${key#*=}"
|
|
key="${key%%=*}"
|
|
case "$key" in
|
|
*_SECRET|*_PASSWORD|*_TOKEN|*_API_KEY)
|
|
case "$value" in
|
|
'${'*) ;;
|
|
*) echo "$svc hardcodes a literal secret in $key"; return 1 ;;
|
|
esac
|
|
;;
|
|
esac
|
|
done < <(container_section "$unit" | grep '^Environment=')
|
|
! container_section "$unit" | grep -q '^EnvironmentFile=' \
|
|
|| { echo "$svc passes a whole env file into the container"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "startup order follows the data dependencies" {
|
|
# Grafana is useless without Prometheus (both rootless, same manager). Alloy
|
|
# used to order on loki.service; as a ROOT unit it lives on the SYSTEM
|
|
# manager and cannot order against a rootless user unit, so it depends only
|
|
# on the network and retries its push until Loki answers.
|
|
grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
|
grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
|
grep -q '^After=network-online.target$' "$UNIT_DIR/alloy.container"
|
|
! grep -q 'loki.service' "$UNIT_DIR/alloy.container"
|
|
}
|
|
|
|
@test "the deployer installs, restarts and health-checks each unit" {
|
|
grep -q '^deploy_observability() {' "$REMOTE_DEPLOY"
|
|
# Installs from the TARGET's own tree, not a hardcoded dankclaw path: the
|
|
# two hosts own different observability units now (Dankclaw five, Dankserver
|
|
# node-exporter), so the source directory has to follow $TARGET.
|
|
grep -q 'local unit_dir="\$TARGET/observability/quadlet"' "$REMOTE_DEPLOY"
|
|
grep -q 'install_quadlet observability "\$unit_dir" "\$service"' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl --user restart "$service.service"' "$REMOTE_DEPLOY"
|
|
local svc url
|
|
while read -r svc url; do
|
|
grep -q "wait_for_health $svc $url" "$REMOTE_DEPLOY" \
|
|
|| { echo "no health wait for $svc at $url"; return 1; }
|
|
done <<'PAIRS'
|
|
prometheus http://127.0.0.1:9090/-/healthy
|
|
grafana http://127.0.0.1:3002/api/health
|
|
loki http://127.0.0.1:3100/ready
|
|
alloy http://127.0.0.1:12345/-/healthy
|
|
node-exporter http://127.0.0.1:9100/metrics
|
|
alertmanager http://127.0.0.1:9093/-/healthy
|
|
cadvisor http://127.0.0.1:8080/healthz
|
|
PAIRS
|
|
}
|
|
|
|
@test "the deployer never enables a generated unit" {
|
|
# `systemctl --user enable X` fails on this host ("transient or generated").
|
|
! grep -q 'systemctl --user enable prometheus.service\|systemctl --user enable grafana.service\|systemctl --user enable loki.service\|systemctl --user enable alloy.service\|systemctl --user enable node-exporter.service' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the deployer rebuilds the locally built images before restarting them" {
|
|
# loki and alloy are franksdank/<svc>:latest built from this repo; a unit
|
|
# change that skipped the build would restart the previous image.
|
|
grep -q 'build_compose_image loki' "$REMOTE_DEPLOY"
|
|
grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "alloy is a root host-network unit that reads every journal" {
|
|
# The whole point of the root cutover: a rootless container cannot open the
|
|
# 0640 journal files. Pin the properties that make the pipeline carry data.
|
|
local u="$UNIT_DIR/alloy.container"
|
|
grep -q '^Network=host$' "$u"
|
|
grep -q '^Volume=/etc/machine-id:/etc/machine-id:ro$' "$u" \
|
|
|| { echo "alloy missing the load-bearing machine-id mount"; return 1; }
|
|
grep -q '^Volume=/var/log/journal:/var/log/journal:ro$' "$u"
|
|
grep -q '^Volume=/run/log/journal:/run/log/journal:ro$' "$u"
|
|
grep -q -- '--server.http.listen-addr=127.0.0.1:12345' "$u"
|
|
# Deployer: system install + root image mirror, not the user quadlet loop.
|
|
grep -q '^deploy_alloy() {' "$REMOTE_DEPLOY"
|
|
grep -q 'install_system_quadlet dankclaw/observability/quadlet alloy' "$REMOTE_DEPLOY"
|
|
grep -q '/etc/containers/systemd/' "$REMOTE_DEPLOY"
|
|
grep -q 'mirror_image_to_root localhost/franksdank/alloy:latest' "$REMOTE_DEPLOY"
|
|
grep -q 'alloy) deploy_alloy ;;' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "no dankclaw compose file competes with the quadlet units" {
|
|
# Two owners for one container name is how a compose run resurrects a
|
|
# stale duplicate next to the quadlet-managed one. This file was retired on
|
|
# 2026-09-16 — every container on the host reports a PODMAN_SYSTEMD_UNIT
|
|
# label — and the invariant is that it stays retired: the four assertions
|
|
# that used to grep it passed against the missing file, which is how a gate
|
|
# goes hollow.
|
|
local cf="$REPO_ROOT/dankclaw/compose.yaml"
|
|
[ ! -f "$cf" ] || {
|
|
echo "$cf is back: every dankclaw container is unit-managed, so a"
|
|
echo "compose path can only spawn a duplicate fighting a unit for a name"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
@test "changes under dankclaw/observability/ run this seam" {
|
|
run "$REPO_ROOT/scripts/test-changed.sh" --list \
|
|
--paths dankclaw/observability/quadlet/prometheus.container
|
|
[ "$status" -eq 0 ]
|
|
echo "$output" | grep -q 'tests/observability_quadlet.bats'
|
|
}
|
|
|
|
@test "the planner routes each unit to its own service" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths "dankclaw/observability/quadlet/$svc.container"
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "$svc" ] || { echo "$svc planned as '$output'"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "prometheus scrapes every LLM endpoint, in its real metric namespace" {
|
|
# Every LLM endpoint that exports /metrics must have a scrape job. As of
|
|
# 2026-10-05 there are TWO runtime namespaces scraped:
|
|
# * llamacpp:* — frank-zealot (llama.cpp) and frank-dragoon (halogen,
|
|
# re-exporting the llama.cpp names), both root host processes on
|
|
# Dankclaw reached via host.containers.internal — this Prometheus is a
|
|
# bridge container and a host-process bind is refused at the LAN IP,
|
|
# which is why frank-zealot read red by construction for days.
|
|
# * vllm:* — frank-archon (radiance) on Dankserver, a different machine,
|
|
# so plain LAN routing reaches 192.168.0.235:7770. It RE-JOINED the
|
|
# scrape set with radiance; the old "job deleted, exports no /metrics"
|
|
# note belonged to the retired FreeToken engine and is false today, so
|
|
# this seam now asserts archon IS scraped (a bare `! grep` here was
|
|
# set -e-exempt and never enforced the opposite).
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-zealot not scraped"; return 1; }
|
|
grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-dragoon not scraped"; return 1; }
|
|
grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-archon not scraped"; return 1; }
|
|
for job in frank-zealot frank-dragoon frank-archon; do
|
|
grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; }
|
|
done
|
|
# No LLM endpoint is reached through a guarded vhost: Prometheus cannot
|
|
# solve an Anubis challenge, so a franksdank.cloud target is red by
|
|
# construction.
|
|
grep -qE '^[[:space:]]*-[[:space:]]*"?https?://[^"]*franksdank' "$cfg" \
|
|
&& { echo "a scrape target points at a guarded vhost"; return 1; }
|
|
true
|
|
}
|
|
|
|
@test "grafana provisioning is declared in git and installed by the deployer" {
|
|
# Datasources belong in the repository, not in grafana.db: a UI-configured
|
|
# one is invisible to review and lost with the data volume.
|
|
local ds="$REPO_ROOT/shared/grafana/provisioning/datasources/prometheus.yml"
|
|
local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json"
|
|
local prov="$REPO_ROOT/shared/grafana/provisioning/dashboards/provider.yml"
|
|
[ -f "$ds" ] && [ -f "$dash" ] && [ -f "$prov" ]
|
|
python3 -c "import json,sys; json.load(open('$dash'))"
|
|
# Container-name address, like every other on-network scrape target: the
|
|
# published port is loopback-only and would be wrong from Grafana.
|
|
grep -q 'url: http://prometheus:9090' "$ds"
|
|
grep -q 'uid: prometheus' "$ds"
|
|
# The unit already mounts the directory; the deployer must fill it.
|
|
grep -q '/opt/grafana/provisioning:/etc/grafana/provisioning:ro' \
|
|
"$REPO_ROOT/dankclaw/observability/quadlet/grafana.container"
|
|
grep -q 'copy_grafana_provisioning' "$REPO_ROOT/scripts/deploy-remote.sh"
|
|
}
|
|
|
|
@test "the planner selects grafana for shared/grafana changes" {
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths shared/grafana/provisioning/datasources/prometheus.yml
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "grafana" ]
|
|
}
|
|
|
|
@test "LLM alerting rules exist and are referenced by the config that ships them" {
|
|
local rules="$REPO_ROOT/shared/prometheus/rules/llm-fleet.yml"
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
[ -f "$rules" ]
|
|
python3 -c "
|
|
import re, yaml
|
|
d = yaml.safe_load(open('$rules'))
|
|
names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rules']]
|
|
for want in ('job:llm_prefix_cache_reuse:rate5m',
|
|
'job:llm_spec_accept_length:rate5m',
|
|
'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
|
|
'LlmSpecDecodeOff', 'LlmEndpointDown',
|
|
'job:vllm_ttft_p95:rate5m', 'VllmRequestsQueueing',
|
|
'VllmPrefixCacheCollapse', 'VllmPreempting', 'VllmTtftSlow'):
|
|
assert want in names, want
|
|
# Every rule must be keyed to a series that actually exists. A rule naming a
|
|
# series nobody exports evaluates nothing forever and errors never — the
|
|
# failure this seam exists for. Two runtimes are scraped in two namespaces:
|
|
# llama.cpp/halogen (llamacpp:*) for frank-zealot and frank-dragoon, and
|
|
# radiance (vllm:*) for frank-archon. Both families were read straight off the
|
|
# endpoints' /metrics; anything else is a guess.
|
|
exprs = [r.get('expr', '') for g in d['groups'] for r in g['rules']]
|
|
live = {
|
|
# llama.cpp / halogen, measured from /metrics 2026-09-16 and 2026-09-20.
|
|
'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
|
|
'llamacpp:requests_deferred', 'llamacpp:requests_processing',
|
|
'llamacpp:spec_decode_num_drafts_total',
|
|
'llamacpp:spec_decode_num_accepted_tokens_total',
|
|
'llamacpp:spec_decode_num_draft_tokens_total',
|
|
'llamacpp:tokens_predicted_total',
|
|
'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
|
|
'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode',
|
|
# radiance (frank-archon), measured from /metrics 2026-10-20.
|
|
'vllm:num_requests_waiting', 'vllm:gpu_prefix_cache_hit_rate',
|
|
'vllm:num_preemptions_total', 'vllm:prompt_tokens_total',
|
|
'vllm:time_to_first_token_seconds_bucket',
|
|
}
|
|
used = {m for e in exprs
|
|
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', e)}
|
|
assert used <= live, used - live
|
|
# The two namespaces must not bleed across groups: a llamacpp: selector
|
|
# evaluates nothing for the radiance job and vice versa, so each group may
|
|
# reference only its own family.
|
|
per = {g['name']: {m.split(':')[0] for r in g['rules']
|
|
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', r.get('expr', ''))}
|
|
for g in d['groups']}
|
|
for name, fams in per.items():
|
|
allowed = {'llamacpp'} if name == 'llm-fleet' else {'vllm'}
|
|
assert fams <= allowed, (name, fams)
|
|
print('rules ok:', len(names))
|
|
"
|
|
# A rule file that nothing loads evaluates nothing, and Prometheus reports
|
|
# no error for a missing rule_files entry — the failure is invisible.
|
|
grep -q '/etc/prometheus/rules/\*.yml' "$cfg"
|
|
# ...and the deployer must install the directory the config points at.
|
|
grep -q 'cp shared/prometheus/rules/llm-fleet.yml' "$REPO_ROOT/scripts/deploy-remote.sh"
|
|
# The dashboard surfaces firing alerts, since nothing else will.
|
|
python3 -c "
|
|
import json
|
|
d = json.load(open('$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json'))
|
|
exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
|
|
assert any('ALERTS' in e for e in exprs), exprs
|
|
"
|
|
}
|
|
|
|
@test "dankserver runs node-exporter as a boot-persistent unit, not from compose" {
|
|
# Dankserver's node-exporter ran only from compose until 2026-09-16 and died
|
|
# at a reboot (`Exited (143)`), leaving Prometheus' node-dankserver job
|
|
# scraping nothing for a day and a half. It also bound the wildcard address,
|
|
# which is the unauthenticated-metrics-on-every-interface posture already
|
|
# removed on Dankclaw.
|
|
local unit="$REPO_ROOT/dankserver/observability/quadlet/node-exporter.container"
|
|
[ -f "$unit" ]
|
|
grep -q '^WantedBy=default.target$' "$unit"
|
|
grep -q '^Restart=always$' "$unit"
|
|
grep -q '^Network=host$' "$unit"
|
|
# Exactly two listen addresses: loopback for host-local probes and this
|
|
# host's LAN address for Prometheus on Dankclaw. Never the wildcard — then
|
|
# fix the half of the sentence the runtime cannot see.
|
|
grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
|
|
grep -q -- '--web.listen-address=192.168.0.235:9100' "$unit"
|
|
! grep -qE -- '--web\.listen-address=(0\.0\.0\.0|:9100|\[::\])' "$unit"
|
|
# One owner per container name: the compose service must be gone, or two
|
|
# definitions fight over the same container.
|
|
! grep -qE '^\s+node-exporter:' "$REPO_ROOT/dankserver/compose.yaml"
|
|
# The deploy path has to know about it on this target.
|
|
grep -q 'dankserver:node-exporter' "$REPO_ROOT/scripts/deploy-remote.sh"
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
|
|
--paths dankserver/observability/quadlet/node-exporter.container
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "node-exporter" ]
|
|
}
|
|
|
|
@test "the observability handler installs units from the target's own tree" {
|
|
# It used to hardcode dankclaw/observability/quadlet, so a dankserver unit
|
|
# could never be installed by the shared dispatcher.
|
|
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
grep -q 'local unit_dir="$TARGET/observability/quadlet"' "$handler"
|
|
grep -q 'install_quadlet observability "$unit_dir" "$service"' "$handler"
|
|
}
|
|
|
|
@test "alertmanager delivers to the Signal relay through the podman host gateway" {
|
|
# The relay listens on the HOST's loopback. From the container that
|
|
# address is only reachable as host.containers.internal (netavark DNATs
|
|
# published ports only — the LAN IP and the gateway are refused), so the
|
|
# webhook URL and the relay's listen port are a cross-file contract: if
|
|
# either half changes alone, every alert is dropped with no error
|
|
# anywhere.
|
|
local am="$REPO_ROOT/shared/prometheus/alertmanager.yml"
|
|
local relay="$REPO_ROOT/scripts/signal-relay.py"
|
|
[ -f "$am" ] && [ -f "$relay" ]
|
|
grep -q 'host.containers.internal:18093/alertmanager' "$am"
|
|
# The relay must listen exactly where the webhook points.
|
|
grep -q '18093' "$relay"
|
|
grep -q '"/alertmanager"' "$relay"
|
|
}
|
|
|
|
@test "prometheus ships alerts to alertmanager on the shared network" {
|
|
# Container-name address, like grafana and loki: the published port is
|
|
# loopback-only and would be wrong from inside Prometheus.
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q 'alertmanagers:' "$cfg"
|
|
grep -q '"alertmanager:9093"' "$cfg"
|
|
}
|
|
|
|
@test "the signal relay is a boot-persistent user unit the deployer installs" {
|
|
# Plain user unit, not a quadlet: it runs python3 from the checkout and
|
|
# can be enabled (generated quadlet units cannot on this host). Linger is
|
|
# on for dank, so [Install] + enable is the whole boot story.
|
|
local unit="$REPO_ROOT/scripts/signal-relay.service"
|
|
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
[ -f "$unit" ]
|
|
grep -q '^WantedBy=default.target$' "$unit"
|
|
grep -q '^Restart=always$' "$unit"
|
|
grep -q '^ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/signal-relay.py$' "$unit"
|
|
grep -q '^install_signal_relay() {' "$handler"
|
|
grep -q 'systemctl --user enable signal-relay.service' "$handler"
|
|
grep -q 'systemctl --user restart signal-relay.service' "$handler"
|
|
# The alertmanager deploy is what reaches the relay on the host.
|
|
grep -q 'install_signal_relay' <(sed -n '/^deploy_observability()/,/^}/p' "$handler")
|
|
}
|
|
|
|
@test "the relay sends through the existing signal-cli account, bridge-subnet only" {
|
|
# The credential is the account state under ~/.local/share/signal-cli;
|
|
# the number and account name are numbers, not secrets. The listener
|
|
# binds the host's LAN IP — the one address a bridge container can
|
|
# reach as host.containers.internal (measured 2026-09-17: loopback
|
|
# refuses, 169.254.1.2 is not local) — and the deployer admits the
|
|
# podman bridge subnet only, so the unauthenticated endpoint stays off
|
|
# the rest of the LAN.
|
|
local relay="$REPO_ROOT/scripts/signal-relay.py"
|
|
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
grep -q 'signal-cli' "$relay"
|
|
grep -q 'SIGNAL_CLI = "/usr/local/bin/signal-cli"' "$relay"
|
|
grep -q 'SIGNAL_ACCOUNT = "+12062476336"' "$relay"
|
|
grep -q 'LISTEN_ADDR = ("192.168.0.143", 18093)' "$relay"
|
|
grep -q -- '--dport 18093 -s 127.0.0.1 -j ACCEPT' "$handler"
|
|
grep -q -- '--dport 18093 -s 192.168.0.143 -j ACCEPT' "$handler"
|
|
grep -q -- '--dport 18093 -s 10.89.0.0/16 -j ACCEPT' "$handler"
|
|
grep -q -- '--dport 18093 -j DROP' "$handler"
|
|
}
|
|
|
|
@test "the planner selects alertmanager for its config and relay files" {
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths shared/prometheus/alertmanager.yml
|
|
[ "$status" -eq 0 ]
|
|
echo "$output" | grep -qx 'alertmanager'
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths scripts/signal-relay.py
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "alertmanager" ]
|
|
}
|
|
|
|
@test "cadvisor exports container metrics only because of --cgroupns=host, and binds exactly its two consumers" {
|
|
# Podman starts every container in a private cgroup namespace, so a cadvisor
|
|
# without --cgroupns=host sees only its own cgroup root and exports a single
|
|
# series (measured 2026-10-20: 1 series before the flag, 996 after). That
|
|
# flag and the host /sys mount are the entire mechanism — pin them, plus the
|
|
# publish set: loopback (host probe + health gate) and this host's LAN
|
|
# address (Prometheus reaches a host-published port there). Never a wildcard.
|
|
local unit lan
|
|
for pair in "dankclaw/observability/quadlet:192.168.0.143" "dankserver/observability/quadlet:192.168.0.235"; do
|
|
unit="$REPO_ROOT/${pair%%:*}/cadvisor.container"
|
|
lan="${pair##*:}"
|
|
[ -f "$unit" ] || { echo "missing $unit"; return 1; }
|
|
grep -q -- '--cgroupns=host' "$unit" || { echo "$unit lacks --cgroupns=host"; return 1; }
|
|
grep -q -- '--pid=host' "$unit" || { echo "$unit lacks --pid=host"; return 1; }
|
|
grep -q '^Volume=/sys:/sys:ro$' "$unit" || { echo "$unit lacks the host /sys mount"; return 1; }
|
|
grep -q '^WantedBy=default.target$' "$unit" || { echo "$unit is not boot-persistent"; return 1; }
|
|
grep -q '^Restart=always$' "$unit" || { echo "$unit is not self-healing"; return 1; }
|
|
grep -q '^HealthCmd=' "$unit" || { echo "$unit has no health check"; return 1; }
|
|
grep -q "^PublishPort=127.0.0.1:8080:8080\$" "$unit" || { echo "$unit must publish loopback :8080"; return 1; }
|
|
grep -q "^PublishPort=$lan:8080:8080\$" "$unit" || { echo "$unit must publish $lan:8080"; return 1; }
|
|
! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$unit" || { echo "$unit publishes on a wildcard"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "Prometheus scrapes cadvisor where it is published and relabels the cgroup path to a container name" {
|
|
# Cross-file contract, measured 2026-10-21 from inside the Prometheus
|
|
# container: a bridge container cannot dial its OWN host's LAN IP — the
|
|
# hairpin refuses published ports exactly like host-net binds, so
|
|
# cadvisor-dankclaw must scrape via the podman gateway (the original pin
|
|
# here demanded the LAN address and shipped a permanently-down target).
|
|
# dankserver is a different machine: plain LAN there. And the whole
|
|
# dashboard depends on the id -> container relabel: cadvisor names nothing
|
|
# itself, the quadlet unit name is only in the cgroup path. If either half
|
|
# changes alone, the Containers dashboard is blank.
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q 'job_name: "cadvisor-dankclaw"' "$cfg" || { echo "no cadvisor-dankclaw job"; return 1; }
|
|
grep -q 'job_name: "cadvisor-dankserver"' "$cfg" || { echo "no cadvisor-dankserver job"; return 1; }
|
|
grep -q '"host.containers.internal:8080"' "$cfg" \
|
|
|| { echo "cadvisor-dankclaw does not scrape via the podman gateway"; return 1; }
|
|
! grep -q '"192.168.0.143:8080"' "$cfg" \
|
|
|| { echo "cadvisor-dankclaw dials its own host LAN IP — hairpin refuses it"; return 1; }
|
|
grep -q '"192.168.0.235:8080"' "$cfg" || { echo "cadvisor-dankserver does not scrape the published LAN port"; return 1; }
|
|
# The relabel that turns …/app.slice/<name>.service… into a `container`.
|
|
grep -q 'app\\.slice/(\[\^\.\]+)\\.service' "$cfg" || { echo "no cgroup-path -> container relabel"; return 1; }
|
|
grep -q 'target_label: container' "$cfg" || { echo "relabel does not set the container label"; return 1; }
|
|
}
|
|
|
|
@test "the Containers dashboard is provisioned from git" {
|
|
local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/containers.json"
|
|
[ -f "$dash" ]
|
|
python3 -c "
|
|
import json
|
|
d = json.load(open('$dash'))
|
|
assert d['uid'] == 'containers', d['uid']
|
|
names = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
|
|
# Only families cadvisor actually exports for rootless quadlet containers are
|
|
# graphed; network/fs are absent for them, so a network/fs panel would be a
|
|
# permanent empty lie.
|
|
assert any('container_cpu_usage_seconds_total' in e for e in names), names
|
|
assert any('container_memory_working_set_bytes' in e for e in names), names
|
|
assert not any('container_network' in e or 'container_fs_' in e for e in names), names
|
|
"
|
|
}
|
|
|
|
@test "the planner selects cadvisor for its unit on each target" {
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths dankclaw/observability/quadlet/cadvisor.container
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "cadvisor" ]
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
|
|
--paths dankserver/observability/quadlet/cadvisor.container
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "cadvisor" ]
|
|
}
|