Frank
87721ab0fe
Cross-cutting rename of the current llama.cpp endpoint on the RTX 3090
(port 7771) from frank-small to frank-secondary:
- Service trio: scripts/{frank-small.env,frank-small.service,
serve-frank-small.sh,download-frank-small.sh} → frank-secondary.*
- Test: tests/frank_small_unit.bats → tests/frank_secondary_unit.bats
- Runbooks: serverAppConfigs-docs/LLM/frank-small-*.md →
frank-secondary-*.md (9 files); the FreeToken runbook stays
freetoken-frank-small.md (it documents the retired CUDA-era service)
- Identifiers: FRANK_SMALL→FRANK_SECONDARY, llm_small→llm_secondary,
LLM_SMALL_*→LLM_SECONDARY_*, deploy_llm_small→deploy_llm_secondary
- Historical references to the OLD frank-small (FreeToken/CUDA on the
3090, and the Dankserver service that became frank-medium) are
preserved verbatim — they describe what the service was called
at the time.
526 lines
27 KiB
Bash
Executable File
526 lines
27 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the five observability Quadlet units on dankclaw.
|
|
#
|
|
# prometheus, grafana, loki, alloy and node-exporter ran as compose services
|
|
# until 2026-09-11 and did not survive a reboot — the 2026-09-11 reboot for the
|
|
# amd_iommu=off change is what exposed it. They are rootless Quadlet units now.
|
|
# This seam pins the properties a reader cannot check by eye:
|
|
#
|
|
# * boot persistence and self-healing on every unit;
|
|
# * the published ports bind 127.0.0.1 and never a wildcard address, so no
|
|
# unauthenticated metrics/dashboard/log endpoint is reachable off-host —
|
|
# the whole point of moving them off compose, which published on 0.0.0.0;
|
|
# * every port that IS published has a named consumer (the host-side homepage
|
|
# status timer, or Prometheus scraping the host);
|
|
# * the container names still match what Caddy proxies to and what
|
|
# Prometheus' own scrape config targets, so the cutover broke no wiring;
|
|
# * the deployer installs, restarts and health-checks them, and never tries
|
|
# to `enable` a generated unit.
|
|
|
|
load helpers
|
|
|
|
UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
UNITS=(prometheus grafana loki alloy node-exporter alertmanager)
|
|
|
|
# container_section <unit> — the [Container] key/value block only
|
|
container_section() {
|
|
awk '/^\[Container\]/{f=1;next} /^\[/{f=0} f' "$1"
|
|
}
|
|
|
|
# service_section <unit> — the [Service] key/value block only
|
|
service_section() {
|
|
awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
|
|
}
|
|
|
|
@test "all six observability units exist under dankclaw/observability/quadlet/" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
[ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "every observability unit is boot-persistent" {
|
|
# Generated units cannot be enabled on this host's podman 5.x v1 quadlet
|
|
# generator; [Install] + linger is the entire boot-persistence story.
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc is not boot-persistent"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "every observability unit is self-healing" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
grep -q '^Restart=always$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no Restart policy"; return 1; }
|
|
grep -q '^RestartSec=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no RestartSec"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "no observability unit publishes on a wildcard address" {
|
|
# The compose services published 9090, 3002 and 3100 on 0.0.0.0 with no
|
|
# authentication of any kind. Anything reachable off-host must go through
|
|
# the guarded franksdank.cloud vhosts instead.
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc publishes on a wildcard address"; return 1; }
|
|
done
|
|
! grep -qE '^PublishPort=' "$UNIT_DIR/alloy.container" || true
|
|
}
|
|
|
|
@test "the four HTTP UIs publish on loopback only" {
|
|
grep -q '^PublishPort=127.0.0.1:9090:9090$' "$UNIT_DIR/prometheus.container"
|
|
grep -q '^PublishPort=127.0.0.1:3002:3000$' "$UNIT_DIR/grafana.container"
|
|
grep -q '^PublishPort=127.0.0.1:3100:3100$' "$UNIT_DIR/loki.container"
|
|
grep -q '^PublishPort=127.0.0.1:9093:9093$' "$UNIT_DIR/alertmanager.container"
|
|
}
|
|
|
|
@test "the homepage status timer's probe targets are the published loopback ports" {
|
|
# scripts/homepage-status.py runs on the host and probes the `probe` field
|
|
# of services.json from there, so a probe and a published port that
|
|
# disagree show the card as down forever.
|
|
local registry="$REPO_ROOT/shared/homepage/services.json"
|
|
grep -q '"probe": "127.0.0.1:9090"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:3002"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:3100"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:9100"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:12345"' "$registry"
|
|
grep -q '"probe": "127.0.0.1:9093"' "$registry"
|
|
}
|
|
|
|
@test "node-exporter binds the host namespace and both of its consumers" {
|
|
# It reports the host's own interfaces/cgroups, so it needs the host netns
|
|
# and host PID namespace; and it must listen exactly where its two
|
|
# consumers are: the homepage timer on loopback, Prometheus (in a
|
|
# container) on the LAN address.
|
|
local unit="$UNIT_DIR/node-exporter.container"
|
|
grep -q '^Network=host$' "$unit"
|
|
grep -q -- '--pid=host' "$unit"
|
|
grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
|
|
grep -q -- '--web.listen-address=192.168.0.143:9100' "$unit"
|
|
! grep -qE '^PublishPort=' "$unit"
|
|
}
|
|
|
|
@test "prometheus scrapes node-exporter where node-exporter actually listens" {
|
|
# Cross-file contract: if either side's address changes alone, host metrics
|
|
# silently stop with no error anywhere.
|
|
#
|
|
# This Prometheus is a bridge container (Network=dankclaw_default), and on
|
|
# this netavark host a container can only reach the host's LAN IP at a
|
|
# PUBLISHED port. node-exporter is net=host, so its 192.168.0.143:9100 bind
|
|
# is a host bind, not a published port — unreachable at its own LAN address.
|
|
# The node-dankclaw job therefore scrapes host.containers.internal:9100,
|
|
# which netavark delivers to the host's LAN IP, where node-exporter listens.
|
|
# (node-dankserver is a different machine, so its LAN address is reached
|
|
# over the real network and is scraped directly.)
|
|
local unit="$UNIT_DIR/node-exporter.container"
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
local lan
|
|
lan=$(grep -oE '192\.168\.0\.[0-9]+:9100' "$unit" | head -1)
|
|
[ -n "$lan" ] || { echo "node-exporter has no LAN listen address"; return 1; }
|
|
# The exporter must still bind that LAN address — it is where the
|
|
# host.containers.internal delivery lands.
|
|
grep -q -- "--web.listen-address=$lan" "$unit" \
|
|
|| { echo "node-exporter no longer binds $lan"; return 1; }
|
|
# The local job scrapes via the gateway, not the (unreachable) LAN address.
|
|
grep -q '"host.containers.internal:9100"' "$cfg" \
|
|
|| { echo "node-dankclaw does not scrape host.containers.internal:9100"; return 1; }
|
|
}
|
|
|
|
@test "container names still match what Caddy and Prometheus address by name" {
|
|
# Caddy's post-auth handlers proxy to prometheus:9090 / grafana:3000 /
|
|
# loki:3100 on the shared network, and Prometheus scrapes grafana:3000 and
|
|
# loki:3100 the same way. A renamed container breaks all of it silently.
|
|
grep -qE '^[[:space:]]*@grafana host grafana\.franksdank\.cloud$' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy grafana:3000' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile"
|
|
local svc
|
|
for svc in prometheus grafana loki alloy; do
|
|
grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc unit renames its container"; return 1; }
|
|
grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc is not on the network Caddy and its peers use"; return 1; }
|
|
done
|
|
grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q 'url = "http://loki:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
|
|
}
|
|
|
|
@test "every observability unit has a health check" {
|
|
# A container without one shows as plainly "Up" in podman ps while its
|
|
# process is wedged, and the deploy has nothing to wait on.
|
|
#
|
|
# Four units declare it inline. loki cannot: its image is distroless, so a
|
|
# unit HealthCmd would reach podman as CMD-SHELL and fail for want of
|
|
# /bin/sh. Its health command ships in the image instead.
|
|
local svc
|
|
for svc in prometheus grafana alloy node-exporter alertmanager; do
|
|
grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no HealthCmd"; return 1; }
|
|
grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \
|
|
|| { echo "$svc has no HealthRetries"; return 1; }
|
|
done
|
|
! grep -q '^HealthCmd=' "$UNIT_DIR/loki.container"
|
|
}
|
|
|
|
@test "loki's health command is exec form in the image, not a shell string" {
|
|
# The whole reason the loki image exists: distroless has no shell, so the
|
|
# command must be a JSON array that podman execs directly.
|
|
local cf="$REPO_ROOT/dankclaw/loki/Containerfile"
|
|
grep -q '^HEALTHCHECK ' "$cf"
|
|
grep -q 'CMD \["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"\]' "$cf"
|
|
# ... and the busybox it execs has to be in the image.
|
|
grep -q 'COPY --from=tools /bin/busybox.static /usr/local/bin/busybox' "$cf"
|
|
}
|
|
|
|
@test "the loki/alloy/comfyui build uses the docker image format" {
|
|
# Without --format docker the loki healthcheck is silently discarded (OCI
|
|
# has no HEALTHCHECK field) and the container stops reporting health.
|
|
grep -q -- '--format docker' "$REMOTE_DEPLOY"
|
|
grep -q 'podman build --format docker --no-cache' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "grafana gets its OAuth client secret without getting a whole env file" {
|
|
# systemd expands ${GRAFANA_OAUTH_CLIENT_SECRET} for the [Container]
|
|
# Environment= line from the [Service] EnvironmentFile, which keeps the
|
|
# file out of the container entirely. That file is the per-service one the
|
|
# deployer seeds (fleet convention: caddy, pihole, copyparty, romm, seerr)
|
|
# — NOT a repo .env. The repo-root .env the compose service happened to
|
|
# read is a gluetun env file carrying HOME=/root and the WireGuard keys;
|
|
# importing it sets HOME for this service and podman then looks for its
|
|
# configuration under /root and refuses to start.
|
|
local unit="$UNIT_DIR/grafana.container"
|
|
service_section "$unit" | grep -q '^EnvironmentFile=/home/dank/Containers/grafana/env$'
|
|
! service_section "$unit" | grep -q 'serverAppConfigs'
|
|
! container_section "$unit" | grep -q '^EnvironmentFile='
|
|
container_section "$unit" | grep -q '^Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=\${GRAFANA_OAUTH_CLIENT_SECRET}$'
|
|
}
|
|
|
|
@test "the deployer seeds exactly one value into the grafana env file" {
|
|
# The single-variable filter is the point: a whole-file copy is how the
|
|
# HOME=/root failure got in.
|
|
grep -q '^seed_grafana_env() {' "$REMOTE_DEPLOY"
|
|
grep -q '^ local dest=/home/dank/Containers/grafana/env$' "$REMOTE_DEPLOY"
|
|
grep -q "awk -F= '/^GRAFANA_OAUTH_CLIENT_SECRET=/ { print }' dankclaw/.env > \"\$dest\"" "$REMOTE_DEPLOY"
|
|
grep -q "grep -q '\^GRAFANA_OAUTH_CLIENT_SECRET=\.' \"\$dest\"" "$REMOTE_DEPLOY"
|
|
grep -q 'seed_grafana_env || return 1' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "observability units hold no literal secret material" {
|
|
# A secret-named setting is allowed only when its value is a ${VAR}
|
|
# reference resolved from an EnvironmentFile systemd reads; a literal value
|
|
# would be a tracked credential. Keys that merely mention a secret word
|
|
# (GF_AUTH_GENERIC_OAUTH_TOKEN_URL, ..._CLIENT_ID) are not secrets, so the
|
|
# match is anchored to the end of the key.
|
|
local svc unit line key value
|
|
for svc in "${UNITS[@]}"; do
|
|
unit="$UNIT_DIR/$svc.container"
|
|
while IFS= read -r line; do
|
|
key="${line#Environment=}"
|
|
value="${key#*=}"
|
|
key="${key%%=*}"
|
|
case "$key" in
|
|
*_SECRET|*_PASSWORD|*_TOKEN|*_API_KEY)
|
|
case "$value" in
|
|
'${'*) ;;
|
|
*) echo "$svc hardcodes a literal secret in $key"; return 1 ;;
|
|
esac
|
|
;;
|
|
esac
|
|
done < <(container_section "$unit" | grep '^Environment=')
|
|
! container_section "$unit" | grep -q '^EnvironmentFile=' \
|
|
|| { echo "$svc passes a whole env file into the container"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "startup order follows the data dependencies" {
|
|
# Grafana is useless without Prometheus and Alloy drops logs without Loki;
|
|
# both pairs are expressed as systemd ordering, not launch luck.
|
|
grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
|
grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
|
grep -q 'After=network-online.target loki.service' "$UNIT_DIR/alloy.container"
|
|
grep -q 'Wants=network-online.target loki.service' "$UNIT_DIR/alloy.container"
|
|
}
|
|
|
|
@test "the deployer installs, restarts and health-checks each unit" {
|
|
grep -q '^deploy_observability() {' "$REMOTE_DEPLOY"
|
|
# Installs from the TARGET's own tree, not a hardcoded dankclaw path: the
|
|
# two hosts own different observability units now (Dankclaw five, Dankserver
|
|
# node-exporter), so the source directory has to follow $TARGET.
|
|
grep -q 'local unit_dir="\$TARGET/observability/quadlet"' "$REMOTE_DEPLOY"
|
|
grep -q 'install_quadlet observability "\$unit_dir" "\$service"' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl --user restart "$service.service"' "$REMOTE_DEPLOY"
|
|
local svc url
|
|
while read -r svc url; do
|
|
grep -q "wait_for_health $svc $url" "$REMOTE_DEPLOY" \
|
|
|| { echo "no health wait for $svc at $url"; return 1; }
|
|
done <<'PAIRS'
|
|
prometheus http://127.0.0.1:9090/-/healthy
|
|
grafana http://127.0.0.1:3002/api/health
|
|
loki http://127.0.0.1:3100/ready
|
|
alloy http://127.0.0.1:12345/-/healthy
|
|
node-exporter http://127.0.0.1:9100/metrics
|
|
alertmanager http://127.0.0.1:9093/-/healthy
|
|
PAIRS
|
|
}
|
|
|
|
@test "the deployer never enables a generated unit" {
|
|
# `systemctl --user enable X` fails on this host ("transient or generated").
|
|
! grep -q 'systemctl --user enable prometheus.service\|systemctl --user enable grafana.service\|systemctl --user enable loki.service\|systemctl --user enable alloy.service\|systemctl --user enable node-exporter.service' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the deployer rebuilds the locally built images before restarting them" {
|
|
# loki and alloy are franksdank/<svc>:latest built from this repo; a unit
|
|
# change that skipped the build would restart the previous image.
|
|
grep -q 'build_compose_image loki' "$REMOTE_DEPLOY"
|
|
grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "no dankclaw compose file competes with the quadlet units" {
|
|
# Two owners for one container name is how a compose run resurrects a
|
|
# stale duplicate next to the quadlet-managed one. This file was retired on
|
|
# 2026-09-16 — every container on the host reports a PODMAN_SYSTEMD_UNIT
|
|
# label — and the invariant is that it stays retired: the four assertions
|
|
# that used to grep it passed against the missing file, which is how a gate
|
|
# goes hollow.
|
|
local cf="$REPO_ROOT/dankclaw/compose.yaml"
|
|
[ ! -f "$cf" ] || {
|
|
echo "$cf is back: every dankclaw container is unit-managed, so a"
|
|
echo "compose path can only spawn a duplicate fighting a unit for a name"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
@test "changes under dankclaw/observability/ run this seam" {
|
|
run "$REPO_ROOT/scripts/test-changed.sh" --list \
|
|
--paths dankclaw/observability/quadlet/prometheus.container
|
|
[ "$status" -eq 0 ]
|
|
echo "$output" | grep -q 'tests/observability_quadlet.bats'
|
|
}
|
|
|
|
@test "the planner routes each unit to its own service" {
|
|
local svc
|
|
for svc in "${UNITS[@]}"; do
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths "dankclaw/observability/quadlet/$svc.container"
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "$svc" ] || { echo "$svc planned as '$output'"; return 1; }
|
|
done
|
|
}
|
|
|
|
@test "prometheus scrapes every LLM endpoint that exports metrics" {
|
|
# The LLM endpoints are the services whose behaviour this fleet argues about
|
|
# most (KV pressure, prefix-cache reuse, queue depth). Both Dankclaw
|
|
# endpoints export the llamacpp:* family. frank-primary LEFT the set
|
|
# 2026-10-05: the FreeToken PR587 ROCm engine has NO /metrics endpoint —
|
|
# scraping it was a structural never-green (404), so the job was deleted
|
|
# rather than left vacuous (the Lucebox DS4.1 precedent, 2026-09-26).
|
|
# The two Dankclaw endpoints are HOST processes reached through
|
|
# host.containers.internal: this Prometheus is a bridge-networked
|
|
# container, and on this netavark host a container can only reach the
|
|
# host's LAN IP at a PUBLISHED port — a host-process bind like
|
|
# llama-server's 0.0.0.0:7770 is refused at 192.168.0.143:7770 while it
|
|
# answers at 169.254.1.2:7770 (why frank-secondary stayed red by construction
|
|
# for days until the 2026-09-20 /targets audit).
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-secondary not scraped"; return 1; }
|
|
grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-large not scraped"; return 1; }
|
|
! grep -q '"192.168.0.235:7770"' "$cfg"
|
|
grep -q '"host.containers.internal:7770,192.168.0.235:7770"' "$cfg" \
|
|
&& { echo "frank-primary folded into a Dankclaw gateway target"; return 1; }
|
|
for job in frank-secondary frank-large; do
|
|
grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; }
|
|
done
|
|
! grep -q 'job_name: "frank-primary"' "$cfg"
|
|
# None of them is reached through a guarded vhost: Prometheus cannot solve
|
|
# an Anubis challenge, so a franksdank.cloud target is red by construction.
|
|
! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg"
|
|
}
|
|
|
|
@test "grafana provisioning is declared in git and installed by the deployer" {
|
|
# Datasources belong in the repository, not in grafana.db: a UI-configured
|
|
# one is invisible to review and lost with the data volume.
|
|
local ds="$REPO_ROOT/shared/grafana/provisioning/datasources/prometheus.yml"
|
|
local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json"
|
|
local prov="$REPO_ROOT/shared/grafana/provisioning/dashboards/provider.yml"
|
|
[ -f "$ds" ] && [ -f "$dash" ] && [ -f "$prov" ]
|
|
python3 -c "import json,sys; json.load(open('$dash'))"
|
|
# Container-name address, like every other on-network scrape target: the
|
|
# published port is loopback-only and would be wrong from Grafana.
|
|
grep -q 'url: http://prometheus:9090' "$ds"
|
|
grep -q 'uid: prometheus' "$ds"
|
|
# The unit already mounts the directory; the deployer must fill it.
|
|
grep -q '/opt/grafana/provisioning:/etc/grafana/provisioning:ro' \
|
|
"$REPO_ROOT/dankclaw/observability/quadlet/grafana.container"
|
|
grep -q 'copy_grafana_provisioning' "$REPO_ROOT/scripts/deploy-remote.sh"
|
|
}
|
|
|
|
@test "the planner selects grafana for shared/grafana changes" {
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths shared/grafana/provisioning/datasources/prometheus.yml
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "grafana" ]
|
|
}
|
|
|
|
@test "LLM alerting rules exist and are referenced by the config that ships them" {
|
|
local rules="$REPO_ROOT/shared/prometheus/rules/llm-fleet.yml"
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
[ -f "$rules" ]
|
|
python3 -c "
|
|
import re, yaml
|
|
d = yaml.safe_load(open('$rules'))
|
|
names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rules']]
|
|
for want in ('job:llm_prefix_cache_reuse:rate5m',
|
|
'job:llm_spec_accept_length:rate5m',
|
|
'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
|
|
'LlmSpecDecodeOff', 'LlmEndpointDown'):
|
|
assert want in names, want
|
|
# Every rule must be keyed to a series that actually exists. A rule naming a
|
|
# series nobody exports evaluates nothing forever and errors never — the
|
|
# failure this seam exists for. Since the 2026-09-26 cutover the only scraped
|
|
# LLM runtime is llama.cpp (frank-secondary, frank-large); the vllm: family left
|
|
# with frank-medium's radiance runtime and must not survive in any expr.
|
|
exprs = [r.get('expr', '') for g in d['groups'] for r in g['rules']]
|
|
# llama.cpp families measured from /metrics on 2026-09-16 and 2026-09-20
|
|
# (frank-secondary, frank-large). Anything else is a guess.
|
|
live = {'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
|
|
'llamacpp:requests_deferred', 'llamacpp:requests_processing',
|
|
'llamacpp:spec_decode_num_drafts_total',
|
|
'llamacpp:spec_decode_num_accepted_tokens_total',
|
|
'llamacpp:spec_decode_num_draft_tokens_total',
|
|
'llamacpp:tokens_predicted_total',
|
|
'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
|
|
'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode'}
|
|
used = {m for e in exprs
|
|
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', e)}
|
|
assert used <= live, used - live
|
|
assert not any(':vllm' in e for e in exprs), 'vllm: series in rules but no vllm endpoint is scraped'
|
|
print('rules ok:', len(names))
|
|
"
|
|
# A rule file that nothing loads evaluates nothing, and Prometheus reports
|
|
# no error for a missing rule_files entry — the failure is invisible.
|
|
grep -q '/etc/prometheus/rules/\*.yml' "$cfg"
|
|
# ...and the deployer must install the directory the config points at.
|
|
grep -q 'cp shared/prometheus/rules/llm-fleet.yml' "$REPO_ROOT/scripts/deploy-remote.sh"
|
|
# The dashboard surfaces firing alerts, since nothing else will.
|
|
python3 -c "
|
|
import json
|
|
d = json.load(open('$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json'))
|
|
exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
|
|
assert any('ALERTS' in e for e in exprs), exprs
|
|
"
|
|
}
|
|
|
|
@test "dankserver runs node-exporter as a boot-persistent unit, not from compose" {
|
|
# Dankserver's node-exporter ran only from compose until 2026-09-16 and died
|
|
# at a reboot (`Exited (143)`), leaving Prometheus' node-dankserver job
|
|
# scraping nothing for a day and a half. It also bound the wildcard address,
|
|
# which is the unauthenticated-metrics-on-every-interface posture already
|
|
# removed on Dankclaw.
|
|
local unit="$REPO_ROOT/dankserver/observability/quadlet/node-exporter.container"
|
|
[ -f "$unit" ]
|
|
grep -q '^WantedBy=default.target$' "$unit"
|
|
grep -q '^Restart=always$' "$unit"
|
|
grep -q '^Network=host$' "$unit"
|
|
# Exactly two listen addresses: loopback for host-local probes and this
|
|
# host's LAN address for Prometheus on Dankclaw. Never the wildcard — then
|
|
# fix the half of the sentence the runtime cannot see.
|
|
grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
|
|
grep -q -- '--web.listen-address=192.168.0.235:9100' "$unit"
|
|
! grep -qE -- '--web\.listen-address=(0\.0\.0\.0|:9100|\[::\])' "$unit"
|
|
# One owner per container name: the compose service must be gone, or two
|
|
# definitions fight over the same container.
|
|
! grep -qE '^\s+node-exporter:' "$REPO_ROOT/dankserver/compose.yaml"
|
|
# The deploy path has to know about it on this target.
|
|
grep -q 'dankserver:node-exporter' "$REPO_ROOT/scripts/deploy-remote.sh"
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
|
|
--paths dankserver/observability/quadlet/node-exporter.container
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "node-exporter" ]
|
|
}
|
|
|
|
@test "the observability handler installs units from the target's own tree" {
|
|
# It used to hardcode dankclaw/observability/quadlet, so a dankserver unit
|
|
# could never be installed by the shared dispatcher.
|
|
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
grep -q 'local unit_dir="$TARGET/observability/quadlet"' "$handler"
|
|
grep -q 'install_quadlet observability "$unit_dir" "$service"' "$handler"
|
|
}
|
|
|
|
@test "alertmanager delivers to the Signal relay through the podman host gateway" {
|
|
# The relay listens on the HOST's loopback. From the container that
|
|
# address is only reachable as host.containers.internal (netavark DNATs
|
|
# published ports only — the LAN IP and the gateway are refused), so the
|
|
# webhook URL and the relay's listen port are a cross-file contract: if
|
|
# either half changes alone, every alert is dropped with no error
|
|
# anywhere.
|
|
local am="$REPO_ROOT/shared/prometheus/alertmanager.yml"
|
|
local relay="$REPO_ROOT/scripts/signal-relay.py"
|
|
[ -f "$am" ] && [ -f "$relay" ]
|
|
grep -q 'host.containers.internal:18093/alertmanager' "$am"
|
|
# The relay must listen exactly where the webhook points.
|
|
grep -q '18093' "$relay"
|
|
grep -q '"/alertmanager"' "$relay"
|
|
}
|
|
|
|
@test "prometheus ships alerts to alertmanager on the shared network" {
|
|
# Container-name address, like grafana and loki: the published port is
|
|
# loopback-only and would be wrong from inside Prometheus.
|
|
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
|
grep -q 'alertmanagers:' "$cfg"
|
|
grep -q '"alertmanager:9093"' "$cfg"
|
|
}
|
|
|
|
@test "the signal relay is a boot-persistent user unit the deployer installs" {
|
|
# Plain user unit, not a quadlet: it runs python3 from the checkout and
|
|
# can be enabled (generated quadlet units cannot on this host). Linger is
|
|
# on for dank, so [Install] + enable is the whole boot story.
|
|
local unit="$REPO_ROOT/scripts/signal-relay.service"
|
|
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
[ -f "$unit" ]
|
|
grep -q '^WantedBy=default.target$' "$unit"
|
|
grep -q '^Restart=always$' "$unit"
|
|
grep -q '^ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/signal-relay.py$' "$unit"
|
|
grep -q '^install_signal_relay() {' "$handler"
|
|
grep -q 'systemctl --user enable signal-relay.service' "$handler"
|
|
grep -q 'systemctl --user restart signal-relay.service' "$handler"
|
|
# The alertmanager deploy is what reaches the relay on the host.
|
|
grep -q 'install_signal_relay' <(sed -n '/^deploy_observability()/,/^}/p' "$handler")
|
|
}
|
|
|
|
@test "the relay sends through the existing signal-cli account, bridge-subnet only" {
|
|
# The credential is the account state under ~/.local/share/signal-cli;
|
|
# the number and account name are numbers, not secrets. The listener
|
|
# binds the host's LAN IP — the one address a bridge container can
|
|
# reach as host.containers.internal (measured 2026-09-17: loopback
|
|
# refuses, 169.254.1.2 is not local) — and the deployer admits the
|
|
# podman bridge subnet only, so the unauthenticated endpoint stays off
|
|
# the rest of the LAN.
|
|
local relay="$REPO_ROOT/scripts/signal-relay.py"
|
|
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
grep -q 'signal-cli' "$relay"
|
|
grep -q 'SIGNAL_CLI = "/usr/local/bin/signal-cli"' "$relay"
|
|
grep -q 'SIGNAL_ACCOUNT = "+12062476336"' "$relay"
|
|
grep -q 'LISTEN_ADDR = ("192.168.0.143", 18093)' "$relay"
|
|
grep -q -- '--dport 18093 -s 127.0.0.1 -j ACCEPT' "$handler"
|
|
grep -q -- '--dport 18093 -s 192.168.0.143 -j ACCEPT' "$handler"
|
|
grep -q -- '--dport 18093 -s 10.89.0.0/16 -j ACCEPT' "$handler"
|
|
grep -q -- '--dport 18093 -j DROP' "$handler"
|
|
}
|
|
|
|
@test "the planner selects alertmanager for its config and relay files" {
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths shared/prometheus/alertmanager.yml
|
|
[ "$status" -eq 0 ]
|
|
echo "$output" | grep -qx 'alertmanager'
|
|
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
|
--paths scripts/signal-relay.py
|
|
[ "$status" -eq 0 ]
|
|
[ "$output" = "alertmanager" ]
|
|
}
|