Frank
31908c623f
Per-container CPU/memory was invisible for the quadlet app stack — Grafana could only see the host (node_exporter) and the LLM engines. This adds a rootless cadvisor quadlet on each host and a `containers` dashboard. cadvisor works on this rootless-Podman / cgroup-v2 host on one condition: --cgroupns=host. Podman starts every container in a private cgroup namespace, so without it cadvisor sees only its own cgroup root and exports a single series (measured 2026-10-20: 1 series off, 996 on). The libpod socket does not name the rootless app containers (it resolved 2 of 34), so naming comes from elsewhere: the quadlet unit name is carried in the cgroup path (app.slice/<name>.service/libpod-payload-<hex>), and Prometheus relabels it into a `container` label. No podman socket needed. cadvisor cannot bind per-interface like node_exporter, so each unit is a bridge container publishing exactly 127.0.0.1:8080 and the host LAN address — never a wildcard. Prometheus (a bridge container) reaches a host-published port at the LAN address, so cadvisor-dankclaw/dankserver scrape 192.168.0.143:8080 / 192.168.0.235:8080. Only the families cadvisor actually exports for these containers are graphed — CPU usage, memory working-set/limit, OOM — not network or filesystem, which are absent under the host-PID layout. Wired through the deployer (allowlist, deploy_observability case + /healthz gate, dispatcher), the planner (both targets), and observability_quadlet.bats.
641 lines
34 KiB
Bash
Executable File
641 lines
34 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the five observability Quadlet units on dankclaw.
|
|
#
|
|
# prometheus, grafana, loki, alloy and node-exporter ran as compose services
|
|
# until 2026-09-11 and did not survive a reboot — the 2026-09-11 reboot for the
|
|
# amd_iommu=off change is what exposed it. They are rootless Quadlet units now.
|
|
# This seam pins the properties a reader cannot check by eye:
|
|
#
|
|
# * boot persistence and self-healing on every unit;
|
|
# * the published ports bind 127.0.0.1 and never a wildcard address, so no
|
|
# unauthenticated metrics/dashboard/log endpoint is reachable off-host —
|
|
# the whole point of moving them off compose, which published on 0.0.0.0;
|
|
# * every port that IS published has a named consumer (the host-side homepage
|
|
# status timer, or Prometheus scraping the host);
|
|
# * the container names still match what Caddy proxies to and what
|
|
# Prometheus' own scrape config targets, so the cutover broke no wiring;
|
|
# * the deployer installs, restarts and health-checks them, and never tries
|
|
# to `enable` a generated unit.
|
|
|
|
load helpers
|
|
|
|
UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager)
|
|
|
|
# container_section <unit> — the [Container] key/value block only
|
|
container_section() {
|
|
awk '/^\[Container\]/{f=1;next} /^\[/{f=0} f' "$1"
|
|
}
|
|
|
|
# service_section <unit> — the [Service] key/value block only
|
|
service_section() {
|
|
awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
|
|
}
|
|
|
|
@test "all seven observability units exist under dankclaw/observability/quadlet/" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
[ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "every observability unit is boot-persistent" {
|
|
# Generated units cannot be enabled on this host's podman quadlet generator;
|
|
# [Install] + linger is the boot-persistence story for the rootless five.
|
|
# Alloy is a ROOT system unit, so its [Install] targets multi-user.target.
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
[[ "$svc" == alloy ]] && continue
|
|
grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc is not boot-persistent"; return 1; }
|
|
done
|
|
grep -q '^WantedBy=multi-user.target$' "$UNIT_DIR/alloy.container" \
|
|
|| { echo "alloy (root unit) is not boot-persistent"; return 1; }
|
|
}
|
|
|
|
@test "every observability unit is self-healing" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
grep -q '^Restart=always$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no Restart policy"; return 1; }
|
|
grep -q '^RestartSec=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no RestartSec"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "no observability unit publishes on a wildcard address" {
|
|
# The compose services published 9090, 3002 and 3100 on 0.0.0.0 with no
|
|
# authentication of any kind. Anything reachable off-host must go through
|
|
# the guarded franksdank.cloud vhosts instead.
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc publishes on a wildcard address"; return 1; }
|
|
done
|
|
! grep -qE '^PublishPort=' "$UNIT_DIR/alloy.container" || true
|
|
}
|
|
|
|
@test "the four HTTP UIs publish on loopback only" {
|
|
grep -q '^PublishPort=127.0.0.1:9090:9090$' "$UNIT_DIR/prometheus.container"
|
|
grep -q '^PublishPort=127.0.0.1:3002:3000$' "$UNIT_DIR/grafana.container"
|
|
grep -q '^PublishPort=127.0.0.1:3100:3100$' "$UNIT_DIR/loki.container"
|
|
grep -q '^PublishPort=127.0.0.1:9093:9093$' "$UNIT_DIR/alertmanager.container"
|
|
}
|
|
|
|
@test "the homepage status timer's probe targets are the published loopback ports" {
|
|
# scripts/homepage-status.py runs on the host and probes the `probe` field
|
|
# of services.json from there, so a probe and a published port that
|
|
# disagree show the card as down forever.
|
|
local registry="$REPO_ROOT/shared/homepage/services.json"
|
|
grep -q '"probe": "127.0.0.1:9090"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:3002"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:3100"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:9100"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:12345"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:9093"' "$registry"
|
|
}
|
|
|
|
@test "node-exporter binds the host namespace and both of its consumers" {
|
|
# It reports the host's own interfaces/cgroups, so it needs the host netns
|
|
# and host PID namespace; and it must listen exactly where its two
|
|
# consumers are: the homepage timer on loopback, Prometheus (in a
|
|
# container) on the LAN address.
|
|
local unit="$UNIT_DIR/node-exporter.container"
|
|
grep -q '^Network=host$' "$unit"
|
|
grep -q -- '--pid=host' "$unit"
|
|
grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
|
|
grep -q -- '--web.listen-address=192.168.0.143:9100' "$unit"
|
|
! grep -qE '^PublishPort=' "$unit"
|
|
}
|
|
|
|
@test "prometheus scrapes node-exporter where node-exporter actually listens" {
|
|
# Cross-file contract: if either side's address changes alone, host metrics
|
|
# silently stop with no error anywhere.
|
|
#
|
|
# This Prometheus is a bridge container (Network=dankclaw_default), and on
|
|
# this netavark host a container can only reach the host's LAN IP at a
|
|
# PUBLISHED port. node-exporter is net=host, so its 192.168.0.143:9100 bind
|
|
# is a host bind, not a published port — unreachable at its own LAN address.
|
|
# The node-dankclaw job therefore scrapes host.containers.internal:9100,
|
|
# which netavark delivers to the host's LAN IP, where node-exporter listens.
|
|
# (node-dankserver is a different machine, so its LAN address is reached
|
|
# over the real network and is scraped directly.)
|
|
local unit="$UNIT_DIR/node-exporter.container"
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
local lan
|
|
lan=$(grep -oE '192\.168\.0\.[0-9]+:9100' "$unit" | head -1)
|
|
[ -n "$lan" ] || { echo "node-exporter has no LAN listen address"; return 1; }
|
|
# The exporter must still bind that LAN address — it is where the
|
|
# host.containers.internal delivery lands.
|
|
grep -q -- "--web.listen-address=$lan" "$unit" \
|
|
|| { echo "node-exporter no longer binds $lan"; return 1; }
|
|
# The local job scrapes via the gateway, not the (unreachable) LAN address.
|
|
grep -q '"host.containers.internal:9100"' "$cfg" \
|
|
|| { echo "node-dankclaw does not scrape host.containers.internal:9100"; return 1; }
|
|
}
|
|
|
|
@test "container names still match what Caddy and Prometheus address by name" {
|
|
# Caddy's post-auth handlers proxy to prometheus:9090 / grafana:3000 /
|
|
# loki:3100 on the shared network, and Prometheus scrapes grafana:3000 and
|
|
# loki:3100 the same way. A renamed container breaks all of it silently.
|
|
grep -qE '^[[:space:]]*@grafana host grafana\.franksdank\.cloud$' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy grafana:3000' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
local svc
|
|
for svc in prometheus grafana loki; do
|
|
grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc unit renames its container"; return 1; }
|
|
grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc is not on the network Caddy and its peers use"; return 1; }
|
|
done
|
|
# Alloy is not on that network: it is a root container on Network=host, the
|
|
# only way to reach Loki's 127.0.0.1:3100 publish from outside a bridge.
|
|
grep -q '^ContainerName=alloy$' "$UNIT_DIR/alloy.container"
|
|
grep -q '^Network=host$' "$UNIT_DIR/alloy.container" \
|
|
|| { echo "alloy must be Network=host to reach Loki on the host loopback"; return 1; }
|
|
grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q 'url = "http://127.0.0.1:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
|
|
}
|
|
|
|
@test "every observability unit has a health check" {
|
|
# A container without one shows as plainly "Up" in podman ps while its
|
|
# process is wedged, and the deploy has nothing to wait on.
|
|
#
|
|
# Four units declare it inline. loki cannot: its image is distroless, so a
|
|
# unit HealthCmd would reach podman as CMD-SHELL and fail for want of
|
|
# /bin/sh. Its health command ships in the image instead.
|
|
local svc
|
|
for svc in prometheus grafana alloy node-exporter alertmanager; do
|
|
grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no HealthCmd"; return 1; }
|
|
grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no HealthRetries"; return 1; }
|
|
done
|
|
! grep -q '^HealthCmd=' "$UNIT_DIR/loki.container"
|
|
}
|
|
|
|
@test "loki's health command is exec form in the image, not a shell string" {
|
|
# The whole reason the loki image exists: distroless has no shell, so the
|
|
# command must be a JSON array that podman execs directly.
|
|
local cf="$REPO_ROOT/dankclaw/loki/Containerfile"
|
|
grep -q '^HEALTHCHECK ' "$cf"
|
|
grep -q 'CMD \["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"\]' "$cf"
|
|
# ... and the busybox it execs has to be in the image.
|
|
grep -q 'COPY --from=tools /bin/busybox.static /usr/local/bin/busybox' "$cf"
|
|
}
|
|
|
|
@test "the loki/alloy/comfyui build uses the docker image format" {
|
|
# Without --format docker the loki healthcheck is silently discarded (OCI
|
|
# has no HEALTHCHECK field) and the container stops reporting health.
|
|
grep -q -- '--format docker' "$REMOTE_DEPLOY"
|
|
grep -q 'podman build --format docker --no-cache' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "grafana gets its OAuth client secret without getting a whole env file" {
|
|
# systemd expands ${GRAFANA_OAUTH_CLIENT_SECRET} for the [Container]
|
|
# Environment= line from the [Service] EnvironmentFile, which keeps the
|
|
# file out of the container entirely. That file is the per-service one the
|
|
# deployer seeds (fleet convention: caddy, pihole, copyparty, romm, seerr)
|
|
# — NOT a repo .env. The repo-root .env the compose service happened to
|
|
# read is a gluetun env file carrying HOME=/root and the WireGuard keys;
|
|
# importing it sets HOME for this service and podman then looks for its
|
|
# configuration under /root and refuses to start.
|
|
local unit="$UNIT_DIR/grafana.container"
|
|
service_section "$unit" | grep -q '^EnvironmentFile=/home/dank/Containers/grafana/env$'
|
|
! service_section "$unit" | grep -q 'serverAppConfigs'
|
|
! container_section "$unit" | grep -q '^EnvironmentFile='
|
|
container_section "$unit" | grep -q '^Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=\${GRAFANA_OAUTH_CLIENT_SECRET}$'
|
|
}
|
|
|
|
@test "the deployer seeds exactly one value into the grafana env file" {
|
|
# The single-variable filter is the point: a whole-file copy is how the
|
|
# HOME=/root failure got in.
|
|
grep -q '^seed_grafana_env() {' "$REMOTE_DEPLOY"
|
|
grep -q '^ local dest=/home/dank/Containers/grafana/env$' "$REMOTE_DEPLOY"
|
|
grep -q "awk -F= '/^GRAFANA_OAUTH_CLIENT_SECRET=/ { print }' dankclaw/.env > \"\$dest\"" "$REMOTE_DEPLOY"
|
|
grep -q "grep -q '\^GRAFANA_OAUTH_CLIENT_SECRET=\.' \"\$dest\"" "$REMOTE_DEPLOY"
|
|
grep -q 'seed_grafana_env || return 1' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "observability units hold no literal secret material" {
|
|
# A secret-named setting is allowed only when its value is a ${VAR}
|
|
# reference resolved from an EnvironmentFile systemd reads; a literal value
|
|
# would be a tracked credential. Keys that merely mention a secret word
|
|
# (GF_AUTH_GENERIC_OAUTH_TOKEN_URL, ..._CLIENT_ID) are not secrets, so the
|
|
# match is anchored to the end of the key.
|
|
local svc unit line key value
|
|
for svc in "${UNITS[@]}"; do
|
|
unit="$UNIT_DIR/$svc.container"
|
|
while IFS= read -r line; do
|
|
key="${line#Environment=}"
|
|
value="${key#*=}"
|
|
key="${key%%=*}"
|
|
case "$key" in
|
|
*_SECRET|*_PASSWORD|*_TOKEN|*_API_KEY)
|
|
case "$value" in
|
|
'${'*) ;;
|
|
*) echo "$svc hardcodes a literal secret in $key"; return 1 ;;
|
|
esac
|
|
;;
|
|
esac
|
|
done < <(container_section "$unit" | grep '^Environment=')
|
|
! container_section "$unit" | grep -q '^EnvironmentFile=' \
|
|
|| { echo "$svc passes a whole env file into the container"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "startup order follows the data dependencies" {
|
|
# Grafana is useless without Prometheus (both rootless, same manager). Alloy
|
|
# used to order on loki.service; as a ROOT unit it lives on the SYSTEM
|
|
# manager and cannot order against a rootless user unit, so it depends only
|
|
# on the network and retries its push until Loki answers.
|
|
grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
|
grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
|
grep -q '^After=network-online.target$' "$UNIT_DIR/alloy.container"
|
|
! grep -q 'loki.service' "$UNIT_DIR/alloy.container"
|
|
}
|
|
|
|
@test "the deployer installs, restarts and health-checks each unit" {
|
|
grep -q '^deploy_observability() {' "$REMOTE_DEPLOY"
|
|
# Installs from the TARGET's own tree, not a hardcoded dankclaw path: the
|
|
# two hosts own different observability units now (Dankclaw five, Dankserver
|
|
# node-exporter), so the source directory has to follow $TARGET.
|
|
grep -q 'local unit_dir="\$TARGET/observability/quadlet"' "$REMOTE_DEPLOY"
|
|
grep -q 'install_quadlet observability "\$unit_dir" "\$service"' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl --user restart "$service.service"' "$REMOTE_DEPLOY"
|
|
local svc url
|
|
while read -r svc url; do
|
|
grep -q "wait_for_health $svc $url" "$REMOTE_DEPLOY" \
|
|
|| { echo "no health wait for $svc at $url"; return 1; }
|
|
done <<'PAIRS'
|
|
prometheus http://127.0.0.1:9090/-/healthy
|
|
grafana http://127.0.0.1:3002/api/health
|
|
loki http://127.0.0.1:3100/ready
|
|
alloy http://127.0.0.1:12345/-/healthy
|
|
node-exporter http://127.0.0.1:9100/metrics
|
|
alertmanager http://127.0.0.1:9093/-/healthy
|
|
cadvisor http://127.0.0.1:8080/healthz
|
|
PAIRS
|
|
}
|
|
|
|
@test "the deployer never enables a generated unit" {
|
|
# `systemctl --user enable X` fails on this host ("transient or generated").
|
|
! grep -q 'systemctl --user enable prometheus.service\|systemctl --user enable grafana.service\|systemctl --user enable loki.service\|systemctl --user enable alloy.service\|systemctl --user enable node-exporter.service' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the deployer rebuilds the locally built images before restarting them" {
|
|
# loki and alloy are franksdank/<svc>:latest built from this repo; a unit
|
|
# change that skipped the build would restart the previous image.
|
|
grep -q 'build_compose_image loki' "$REMOTE_DEPLOY"
|
|
grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "alloy is a root host-network unit that reads every journal" {
|
|
# The whole point of the root cutover: a rootless container cannot open the
|
|
# 0640 journal files. Pin the properties that make the pipeline carry data.
|
|
local u="$UNIT_DIR/alloy.container"
|
|
grep -q '^Network=host$' "$u"
|
|
grep -q '^Volume=/etc/machine-id:/etc/machine-id:ro$' "$u" \
|
|
|| { echo "alloy missing the load-bearing machine-id mount"; return 1; }
|
|
grep -q '^Volume=/var/log/journal:/var/log/journal:ro$' "$u"
|
|
grep -q '^Volume=/run/log/journal:/run/log/journal:ro$' "$u"
|
|
grep -q -- '--server.http.listen-addr=127.0.0.1:12345' "$u"
|
|
# Deployer: system install + root image mirror, not the user quadlet loop.
|
|
grep -q '^deploy_alloy() {' "$REMOTE_DEPLOY"
|
|
grep -q 'install_system_quadlet dankclaw/observability/quadlet alloy' "$REMOTE_DEPLOY"
|
|
grep -q '/etc/containers/systemd/' "$REMOTE_DEPLOY"
|
|
grep -q 'mirror_image_to_root localhost/franksdank/alloy:latest' "$REMOTE_DEPLOY"
|
|
grep -q 'alloy) deploy_alloy ;;' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "no dankclaw compose file competes with the quadlet units" {
|
|
# Two owners for one container name is how a compose run resurrects a
|
|
# stale duplicate next to the quadlet-managed one. This file was retired on
|
|
# 2026-09-16 — every container on the host reports a PODMAN_SYSTEMD_UNIT
|
|
# label — and the invariant is that it stays retired: the four assertions
|
|
# that used to grep it passed against the missing file, which is how a gate
|
|
# goes hollow.
|
|
local cf="$REPO_ROOT/dankclaw/compose.yaml"
|
|
[ ! -f "$cf" ] || {
|
|
echo "$cf is back: every dankclaw container is unit-managed, so a"
|
|
echo "compose path can only spawn a duplicate fighting a unit for a name"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
@test "changes under dankclaw/observability/ run this seam" {
|
|
run "$REPO_ROOT/scripts/test-changed.sh" --list \
|
|
--paths dankclaw/observability/quadlet/prometheus.container
|
|
[ "$status" -eq 0 ]
|
|
echo "$output" | grep -q 'tests/observability_quadlet.bats'
|
|
}
|
|
|
|
@test "the planner routes each unit to its own service" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths "dankclaw/observability/quadlet/$svc.container"
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "$svc" ] || { echo "$svc planned as '$output'"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "prometheus scrapes every LLM endpoint, in its real metric namespace" {
|
|
# Every LLM endpoint that exports /metrics must have a scrape job. As of
|
|
# 2026-10-05 there are TWO runtime namespaces scraped:
|
|
# * llamacpp:* — frank-zealot (llama.cpp) and frank-dragoon (halogen,
|
|
# re-exporting the llama.cpp names), both root host processes on
|
|
# Dankclaw reached via host.containers.internal — this Prometheus is a
|
|
# bridge container and a host-process bind is refused at the LAN IP,
|
|
# which is why frank-zealot read red by construction for days.
|
|
# * vllm:* — frank-archon (radiance) on Dankserver, a different machine,
|
|
# so plain LAN routing reaches 192.168.0.235:7770. It RE-JOINED the
|
|
# scrape set with radiance; the old "job deleted, exports no /metrics"
|
|
# note belonged to the retired FreeToken engine and is false today, so
|
|
# this seam now asserts archon IS scraped (a bare `! grep` here was
|
|
# set -e-exempt and never enforced the opposite).
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-zealot not scraped"; return 1; }
|
|
grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-dragoon not scraped"; return 1; }
|
|
grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-archon not scraped"; return 1; }
|
|
for job in frank-zealot frank-dragoon frank-archon; do
|
|
grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; }
|
|
done
|
|
# No LLM endpoint is reached through a guarded vhost: Prometheus cannot
|
|
# solve an Anubis challenge, so a franksdank.cloud target is red by
|
|
# construction.
|
|
grep -qE '^[[:space:]]*-[[:space:]]*"?https?://[^"]*franksdank' "$cfg" \
|
|
&& { echo "a scrape target points at a guarded vhost"; return 1; }
|
|
true
|
|
}
|
|
|
|
@test "grafana provisioning is declared in git and installed by the deployer" {
|
|
# Datasources belong in the repository, not in grafana.db: a UI-configured
|
|
# one is invisible to review and lost with the data volume.
|
|
local ds="$REPO_ROOT/shared/grafana/provisioning/datasources/prometheus.yml"
|
|
local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json"
|
|
local prov="$REPO_ROOT/shared/grafana/provisioning/dashboards/provider.yml"
|
|
[ -f "$ds" ] && [ -f "$dash" ] && [ -f "$prov" ]
|
|
python3 -c "import json,sys; json.load(open('$dash'))"
|
|
# Container-name address, like every other on-network scrape target: the
|
|
# published port is loopback-only and would be wrong from Grafana.
|
|
grep -q 'url: http://prometheus:9090' "$ds"
|
|
grep -q 'uid: prometheus' "$ds"
|
|
# The unit already mounts the directory; the deployer must fill it.
|
|
grep -q '/opt/grafana/provisioning:/etc/grafana/provisioning:ro' \
|
|
"$REPO_ROOT/dankclaw/observability/quadlet/grafana.container"
|
|
grep -q 'copy_grafana_provisioning' "$REPO_ROOT/scripts/deploy-remote.sh"
|
|
}
|
|
|
|
@test "the planner selects grafana for shared/grafana changes" {
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths shared/grafana/provisioning/datasources/prometheus.yml
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "grafana" ]
|
|
}
|
|
|
|
@test "LLM alerting rules exist and are referenced by the config that ships them" {
|
|
local rules="$REPO_ROOT/shared/prometheus/rules/llm-fleet.yml"
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
[ -f "$rules" ]
|
|
python3 -c "
|
|
import re, yaml
|
|
d = yaml.safe_load(open('$rules'))
|
|
names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rules']]
|
|
for want in ('job:llm_prefix_cache_reuse:rate5m',
|
|
'job:llm_spec_accept_length:rate5m',
|
|
'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
|
|
'LlmSpecDecodeOff', 'LlmEndpointDown',
|
|
'job:vllm_ttft_p95:rate5m', 'VllmRequestsQueueing',
|
|
'VllmPrefixCacheCollapse', 'VllmPreempting', 'VllmTtftSlow'):
|
|
assert want in names, want
|
|
# Every rule must be keyed to a series that actually exists. A rule naming a
|
|
# series nobody exports evaluates nothing forever and errors never — the
|
|
# failure this seam exists for. Two runtimes are scraped in two namespaces:
|
|
# llama.cpp/halogen (llamacpp:*) for frank-zealot and frank-dragoon, and
|
|
# radiance (vllm:*) for frank-archon. Both families were read straight off the
|
|
# endpoints' /metrics; anything else is a guess.
|
|
exprs = [r.get('expr', '') for g in d['groups'] for r in g['rules']]
|
|
live = {
|
|
# llama.cpp / halogen, measured from /metrics 2026-09-16 and 2026-09-20.
|
|
'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
|
|
'llamacpp:requests_deferred', 'llamacpp:requests_processing',
|
|
'llamacpp:spec_decode_num_drafts_total',
|
|
'llamacpp:spec_decode_num_accepted_tokens_total',
|
|
'llamacpp:spec_decode_num_draft_tokens_total',
|
|
'llamacpp:tokens_predicted_total',
|
|
'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
|
|
'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode',
|
|
# radiance (frank-archon), measured from /metrics 2026-10-20.
|
|
'vllm:num_requests_waiting', 'vllm:gpu_prefix_cache_hit_rate',
|
|
'vllm:num_preemptions_total', 'vllm:prompt_tokens_total',
|
|
'vllm:time_to_first_token_seconds_bucket',
|
|
}
|
|
used = {m for e in exprs
|
|
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', e)}
|
|
assert used <= live, used - live
|
|
# The two namespaces must not bleed across groups: a llamacpp: selector
|
|
# evaluates nothing for the radiance job and vice versa, so each group may
|
|
# reference only its own family.
|
|
per = {g['name']: {m.split(':')[0] for r in g['rules']
|
|
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', r.get('expr', ''))}
|
|
for g in d['groups']}
|
|
for name, fams in per.items():
|
|
allowed = {'llamacpp'} if name == 'llm-fleet' else {'vllm'}
|
|
assert fams <= allowed, (name, fams)
|
|
print('rules ok:', len(names))
|
|
"
|
|
# A rule file that nothing loads evaluates nothing, and Prometheus reports
|
|
# no error for a missing rule_files entry — the failure is invisible.
|
|
grep -q '/etc/prometheus/rules/\*.yml' "$cfg"
|
|
# ...and the deployer must install the directory the config points at.
|
|
grep -q 'cp shared/prometheus/rules/llm-fleet.yml' "$REPO_ROOT/scripts/deploy-remote.sh"
|
|
# The dashboard surfaces firing alerts, since nothing else will.
|
|
python3 -c "
|
|
import json
|
|
d = json.load(open('$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json'))
|
|
exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
|
|
assert any('ALERTS' in e for e in exprs), exprs
|
|
"
|
|
}
|
|
|
|
@test "dankserver runs node-exporter as a boot-persistent unit, not from compose" {
|
|
# Dankserver's node-exporter ran only from compose until 2026-09-16 and died
|
|
# at a reboot (`Exited (143)`), leaving Prometheus' node-dankserver job
|
|
# scraping nothing for a day and a half. It also bound the wildcard address,
|
|
# which is the unauthenticated-metrics-on-every-interface posture already
|
|
# removed on Dankclaw.
|
|
local unit="$REPO_ROOT/dankserver/observability/quadlet/node-exporter.container"
|
|
[ -f "$unit" ]
|
|
grep -q '^WantedBy=default.target$' "$unit"
|
|
grep -q '^Restart=always$' "$unit"
|
|
grep -q '^Network=host$' "$unit"
|
|
# Exactly two listen addresses: loopback for host-local probes and this
|
|
# host's LAN address for Prometheus on Dankclaw. Never the wildcard — then
|
|
# fix the half of the sentence the runtime cannot see.
|
|
grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
|
|
grep -q -- '--web.listen-address=192.168.0.235:9100' "$unit"
|
|
! grep -qE -- '--web\.listen-address=(0\.0\.0\.0|:9100|\[::\])' "$unit"
|
|
# One owner per container name: the compose service must be gone, or two
|
|
# definitions fight over the same container.
|
|
! grep -qE '^\s+node-exporter:' "$REPO_ROOT/dankserver/compose.yaml"
|
|
# The deploy path has to know about it on this target.
|
|
grep -q 'dankserver:node-exporter' "$REPO_ROOT/scripts/deploy-remote.sh"
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
|
|
--paths dankserver/observability/quadlet/node-exporter.container
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "node-exporter" ]
|
|
}
|
|
|
|
@test "the observability handler installs units from the target's own tree" {
|
|
# It used to hardcode dankclaw/observability/quadlet, so a dankserver unit
|
|
# could never be installed by the shared dispatcher.
|
|
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
grep -q 'local unit_dir="$TARGET/observability/quadlet"' "$handler"
|
|
grep -q 'install_quadlet observability "$unit_dir" "$service"' "$handler"
|
|
}
|
|
|
|
@test "alertmanager delivers to the Signal relay through the podman host gateway" {
|
|
# The relay listens on the HOST's loopback. From the container that
|
|
# address is only reachable as host.containers.internal (netavark DNATs
|
|
# published ports only — the LAN IP and the gateway are refused), so the
|
|
# webhook URL and the relay's listen port are a cross-file contract: if
|
|
# either half changes alone, every alert is dropped with no error
|
|
# anywhere.
|
|
local am="$REPO_ROOT/shared/prometheus/alertmanager.yml"
|
|
local relay="$REPO_ROOT/scripts/signal-relay.py"
|
|
[ -f "$am" ] && [ -f "$relay" ]
|
|
grep -q 'host.containers.internal:18093/alertmanager' "$am"
|
|
# The relay must listen exactly where the webhook points.
|
|
grep -q '18093' "$relay"
|
|
grep -q '"/alertmanager"' "$relay"
|
|
}
|
|
|
|
@test "prometheus ships alerts to alertmanager on the shared network" {
|
|
# Container-name address, like grafana and loki: the published port is
|
|
# loopback-only and would be wrong from inside Prometheus.
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q 'alertmanagers:' "$cfg"
|
|
grep -q '"alertmanager:9093"' "$cfg"
|
|
}
|
|
|
|
@test "the signal relay is a boot-persistent user unit the deployer installs" {
|
|
# Plain user unit, not a quadlet: it runs python3 from the checkout and
|
|
# can be enabled (generated quadlet units cannot on this host). Linger is
|
|
# on for dank, so [Install] + enable is the whole boot story.
|
|
local unit="$REPO_ROOT/scripts/signal-relay.service"
|
|
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
[ -f "$unit" ]
|
|
grep -q '^WantedBy=default.target$' "$unit"
|
|
grep -q '^Restart=always$' "$unit"
|
|
grep -q '^ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/signal-relay.py$' "$unit"
|
|
grep -q '^install_signal_relay() {' "$handler"
|
|
grep -q 'systemctl --user enable signal-relay.service' "$handler"
|
|
grep -q 'systemctl --user restart signal-relay.service' "$handler"
|
|
# The alertmanager deploy is what reaches the relay on the host.
|
|
grep -q 'install_signal_relay' <(sed -n '/^deploy_observability()/,/^}/p' "$handler")
|
|
}
|
|
|
|
@test "the relay sends through the existing signal-cli account, bridge-subnet only" {
|
|
# The credential is the account state under ~/.local/share/signal-cli;
|
|
# the number and account name are numbers, not secrets. The listener
|
|
# binds the host's LAN IP — the one address a bridge container can
|
|
# reach as host.containers.internal (measured 2026-09-17: loopback
|
|
# refuses, 169.254.1.2 is not local) — and the deployer admits the
|
|
# podman bridge subnet only, so the unauthenticated endpoint stays off
|
|
# the rest of the LAN.
|
|
local relay="$REPO_ROOT/scripts/signal-relay.py"
|
|
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
grep -q 'signal-cli' "$relay"
|
|
grep -q 'SIGNAL_CLI = "/usr/local/bin/signal-cli"' "$relay"
|
|
grep -q 'SIGNAL_ACCOUNT = "+12062476336"' "$relay"
|
|
grep -q 'LISTEN_ADDR = ("192.168.0.143", 18093)' "$relay"
|
|
grep -q -- '--dport 18093 -s 127.0.0.1 -j ACCEPT' "$handler"
|
|
grep -q -- '--dport 18093 -s 192.168.0.143 -j ACCEPT' "$handler"
|
|
grep -q -- '--dport 18093 -s 10.89.0.0/16 -j ACCEPT' "$handler"
|
|
grep -q -- '--dport 18093 -j DROP' "$handler"
|
|
}
|
|
|
|
@test "the planner selects alertmanager for its config and relay files" {
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths shared/prometheus/alertmanager.yml
|
|
[ "$status" -eq 0 ]
|
|
echo "$output" | grep -qx 'alertmanager'
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths scripts/signal-relay.py
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "alertmanager" ]
|
|
}
|
|
|
|
@test "cadvisor exports container metrics only because of --cgroupns=host, and binds exactly its two consumers" {
|
|
# Podman starts every container in a private cgroup namespace, so a cadvisor
|
|
# without --cgroupns=host sees only its own cgroup root and exports a single
|
|
# series (measured 2026-10-20: 1 series before the flag, 996 after). That
|
|
# flag and the host /sys mount are the entire mechanism — pin them, plus the
|
|
# publish set: loopback (host probe + health gate) and this host's LAN
|
|
# address (Prometheus reaches a host-published port there). Never a wildcard.
|
|
local unit lan
|
|
for pair in "dankclaw/observability/quadlet:192.168.0.143" "dankserver/observability/quadlet:192.168.0.235"; do
|
|
unit="$REPO_ROOT/${pair%%:*}/cadvisor.container"
|
|
lan="${pair##*:}"
|
|
[ -f "$unit" ] || { echo "missing $unit"; return 1; }
|
|
grep -q -- '--cgroupns=host' "$unit" || { echo "$unit lacks --cgroupns=host"; return 1; }
|
|
grep -q -- '--pid=host' "$unit" || { echo "$unit lacks --pid=host"; return 1; }
|
|
grep -q '^Volume=/sys:/sys:ro$' "$unit" || { echo "$unit lacks the host /sys mount"; return 1; }
|
|
grep -q '^WantedBy=default.target$' "$unit" || { echo "$unit is not boot-persistent"; return 1; }
|
|
grep -q '^Restart=always$' "$unit" || { echo "$unit is not self-healing"; return 1; }
|
|
grep -q '^HealthCmd=' "$unit" || { echo "$unit has no health check"; return 1; }
|
|
grep -q "^PublishPort=127.0.0.1:8080:8080\$" "$unit" || { echo "$unit must publish loopback :8080"; return 1; }
|
|
grep -q "^PublishPort=$lan:8080:8080\$" "$unit" || { echo "$unit must publish $lan:8080"; return 1; }
|
|
! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$unit" || { echo "$unit publishes on a wildcard"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "Prometheus scrapes cadvisor where it is published and relabels the cgroup path to a container name" {
|
|
# Cross-file contract: cadvisor publishes the LAN port; a cadvisor scrape
|
|
# job pointing anywhere else reads nothing (unlike node-exporter's host-net
|
|
# bind, a published port IS reachable from this bridge container at the LAN
|
|
# address). And the whole dashboard depends on the id -> container relabel:
|
|
# cadvisor names nothing itself, the quadlet unit name is only in the cgroup
|
|
# path. If either half changes alone, the Containers dashboard is blank.
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q 'job_name: "cadvisor-dankclaw"' "$cfg" || { echo "no cadvisor-dankclaw job"; return 1; }
|
|
grep -q 'job_name: "cadvisor-dankserver"' "$cfg" || { echo "no cadvisor-dankserver job"; return 1; }
|
|
grep -q '"192.168.0.143:8080"' "$cfg" || { echo "cadvisor-dankclaw does not scrape the published LAN port"; return 1; }
|
|
grep -q '"192.168.0.235:8080"' "$cfg" || { echo "cadvisor-dankserver does not scrape the published LAN port"; return 1; }
|
|
# The relabel that turns …/app.slice/<name>.service… into a `container`.
|
|
grep -q 'app\\.slice/(\[\^\.\]+)\\.service' "$cfg" || { echo "no cgroup-path -> container relabel"; return 1; }
|
|
grep -q 'target_label: container' "$cfg" || { echo "relabel does not set the container label"; return 1; }
|
|
}
|
|
|
|
@test "the Containers dashboard is provisioned from git" {
|
|
local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/containers.json"
|
|
[ -f "$dash" ]
|
|
python3 -c "
|
|
import json
|
|
d = json.load(open('$dash'))
|
|
assert d['uid'] == 'containers', d['uid']
|
|
names = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
|
|
# Only families cadvisor actually exports for rootless quadlet containers are
|
|
# graphed; network/fs are absent for them, so a network/fs panel would be a
|
|
# permanent empty lie.
|
|
assert any('container_cpu_usage_seconds_total' in e for e in names), names
|
|
assert any('container_memory_working_set_bytes' in e for e in names), names
|
|
assert not any('container_network' in e or 'container_fs_' in e for e in names), names
|
|
"
|
|
}
|
|
|
|
@test "the planner selects cadvisor for its unit on each target" {
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths dankclaw/observability/quadlet/cadvisor.container
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "cadvisor" ]
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
|
|
--paths dankserver/observability/quadlet/cadvisor.container
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "cadvisor" ]
|
|
}
|