serverAppConfigs/tests/observability_quadlet.bats

#!/usr/bin/env bats
# Seam: the five observability Quadlet units on dankclaw.
#
# prometheus, grafana, loki, alloy and node-exporter ran as compose services
# until 2026-09-11 and did not survive a reboot — the 2026-09-11 reboot for the
# amd_iommu=off change is what exposed it. They are rootless Quadlet units now.
# This seam pins the properties a reader cannot check by eye:
#
#   * boot persistence and self-healing on every unit;
#   * the published ports bind 127.0.0.1 and never a wildcard address, so no
#     unauthenticated metrics/dashboard/log endpoint is reachable off-host —
#     the whole point of moving them off compose, which published on 0.0.0.0;
#   * every port that IS published has a named consumer (the host-side homepage
#     status timer, or Prometheus scraping the host);
#   * the container names still match what Caddy proxies to and what
#     Prometheus' own scrape config targets, so the cutover broke no wiring;
#   * the deployer installs, restarts and health-checks them, and never tries
#     to `enable` a generated unit.

load helpers

UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
UNITS=(prometheus grafana loki alloy node-exporter)

# container_section <unit> — the [Container] key/value block only
container_section() {
    awk '/^\[Container\]/{f=1;next} /^\[/{f=0} f' "$1"
}

# service_section <unit> — the [Service] key/value block only
service_section() {
    awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
}

@test "all five observability units exist under dankclaw/observability/quadlet/" {
    local svc
    for svc in "${UNITS[@]}"; do
        [ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
    done
}

@test "every observability unit is boot-persistent" {
    # Generated units cannot be enabled on this host's podman 5.x v1 quadlet
    # generator; [Install] + linger is the entire boot-persistence story.
    local svc
    for svc in "${UNITS[@]}"; do
        grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \
            || { echo "$svc is not boot-persistent"; return 1; }
    done
}

@test "every observability unit is self-healing" {
    local svc
    for svc in "${UNITS[@]}"; do
        grep -q '^Restart=always$' "$UNIT_DIR/$svc.container" \
            || { echo "$svc has no Restart policy"; return 1; }
        grep -q '^RestartSec=' "$UNIT_DIR/$svc.container" \
            || { echo "$svc has no RestartSec"; return 1; }
    done
}

@test "no observability unit publishes on a wildcard address" {
    # The compose services published 9090, 3002 and 3100 on 0.0.0.0 with no
    # authentication of any kind. Anything reachable off-host must go through
    # the guarded franksdank.cloud vhosts instead.
    local svc
    for svc in "${UNITS[@]}"; do
        ! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$UNIT_DIR/$svc.container" \
            || { echo "$svc publishes on a wildcard address"; return 1; }
    done
    ! grep -qE '^PublishPort=' "$UNIT_DIR/alloy.container" || true
}

@test "the three HTTP UIs publish on loopback only" {
    grep -q '^PublishPort=127.0.0.1:9090:9090$' "$UNIT_DIR/prometheus.container"
    grep -q '^PublishPort=127.0.0.1:3002:3000$' "$UNIT_DIR/grafana.container"
    grep -q '^PublishPort=127.0.0.1:3100:3100$' "$UNIT_DIR/loki.container"
}

@test "the homepage status timer's probe targets are the published loopback ports" {
    # scripts/homepage-status.py runs on the host and probes the `probe` field
    # of services.json from there, so a probe and a published port that
    # disagree show the card as down forever.
    local registry="$REPO_ROOT/shared/homepage/services.json"
    grep -q '"probe": "127.0.0.1:9090"' "$registry"
    grep -q '"probe": "127.0.0.1:3002"' "$registry"
    grep -q '"probe": "127.0.0.1:3100"' "$registry"
    grep -q '"probe": "127.0.0.1:9100"' "$registry"
    grep -q '"probe": "127.0.0.1:12345"' "$registry"
}

@test "node-exporter binds the host namespace and both of its consumers" {
    # It reports the host's own interfaces/cgroups, so it needs the host netns
    # and host PID namespace; and it must listen exactly where its two
    # consumers are: the homepage timer on loopback, Prometheus (in a
    # container) on the LAN address.
    local unit="$UNIT_DIR/node-exporter.container"
    grep -q '^Network=host$' "$unit"
    grep -q -- '--pid=host' "$unit"
    grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
    grep -q -- '--web.listen-address=192.168.0.143:9100' "$unit"
    ! grep -qE '^PublishPort=' "$unit"
}

@test "prometheus scrapes node-exporter where node-exporter actually listens" {
    # Cross-file contract: if either side's address changes alone, host metrics
    # silently stop with no error anywhere.
    local unit="$UNIT_DIR/node-exporter.container"
    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
    local addr
    addr=$(grep -oE '192\.168\.0\.[0-9]+:9100' "$unit" | head -1)
    [ -n "$addr" ] || { echo "node-exporter has no LAN listen address"; return 1; }
    grep -q "\"$addr\"" "$cfg" || { echo "prometheus does not scrape $addr"; return 1; }
}

@test "container names still match what Caddy and Prometheus address by name" {
    # Caddy's post-auth handlers proxy to prometheus:9090 / grafana:3000 /
    # loki:3100 on the shared network, and Prometheus scrapes grafana:3000 and
    # loki:3100 the same way. A renamed container breaks all of it silently.
    grep -qE '^[[:space:]]*@grafana host grafana\.franksdank\.cloud$' "$REPO_ROOT/dankclaw/Caddyfile"
    grep -q 'reverse_proxy grafana:3000' "$REPO_ROOT/dankclaw/Caddyfile"
    grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile"
    grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile"
    local svc
    for svc in prometheus grafana loki alloy; do
        grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \
            || { echo "$svc unit renames its container"; return 1; }
        grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \
            || { echo "$svc is not on the network Caddy and its peers use"; return 1; }
    done
    grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
    grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
    grep -q 'url = "http://loki:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
}

@test "every observability unit has a health check" {
    # A container without one shows as plainly "Up" in podman ps while its
    # process is wedged, and the deploy has nothing to wait on.
    #
    # Four units declare it inline. loki cannot: its image is distroless, so a
    # unit HealthCmd would reach podman as CMD-SHELL and fail for want of
    # /bin/sh. Its health command ships in the image instead.
    local svc
    for svc in prometheus grafana alloy node-exporter; do
        grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \
            || { echo "$svc has no HealthCmd"; return 1; }
        grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \
            || { echo "$svc has no HealthRetries"; return 1; }
    done
    ! grep -q '^HealthCmd=' "$UNIT_DIR/loki.container"
}

@test "loki's health command is exec form in the image, not a shell string" {
    # The whole reason the loki image exists: distroless has no shell, so the
    # command must be a JSON array that podman execs directly.
    local cf="$REPO_ROOT/dankclaw/loki/Containerfile"
    grep -q '^HEALTHCHECK ' "$cf"
    grep -q 'CMD \["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"\]' "$cf"
    # ... and the busybox it execs has to be in the image.
    grep -q 'COPY --from=tools /bin/busybox.static /usr/local/bin/busybox' "$cf"
}

@test "the loki/alloy/comfyui build uses the docker image format" {
    # Without --format docker the loki healthcheck is silently discarded (OCI
    # has no HEALTHCHECK field) and the container stops reporting health.
    grep -q -- '--format docker' "$REMOTE_DEPLOY"
    grep -q 'podman build --format docker --no-cache' "$REMOTE_DEPLOY"
}

@test "grafana gets its OAuth client secret without getting a whole env file" {
    # systemd expands ${GRAFANA_OAUTH_CLIENT_SECRET} for the [Container]
    # Environment= line from the [Service] EnvironmentFile, which keeps the
    # file out of the container entirely. That file is the per-service one the
    # deployer seeds (fleet convention: caddy, pihole, copyparty, romm, seerr)
    # — NOT a repo .env. The repo-root .env the compose service happened to
    # read is a gluetun env file carrying HOME=/root and the WireGuard keys;
    # importing it sets HOME for this service and podman then looks for its
    # configuration under /root and refuses to start.
    local unit="$UNIT_DIR/grafana.container"
    service_section "$unit" | grep -q '^EnvironmentFile=/home/dank/Containers/grafana/env$'
    ! service_section "$unit" | grep -q 'serverAppConfigs'
    ! container_section "$unit" | grep -q '^EnvironmentFile='
    container_section "$unit" | grep -q '^Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=\${GRAFANA_OAUTH_CLIENT_SECRET}$'
}

@test "the deployer seeds exactly one value into the grafana env file" {
    # The single-variable filter is the point: a whole-file copy is how the
    # HOME=/root failure got in.
    grep -q '^seed_grafana_env() {' "$REMOTE_DEPLOY"
    grep -q '^    local dest=/home/dank/Containers/grafana/env$' "$REMOTE_DEPLOY"
    grep -q "awk -F= '/^GRAFANA_OAUTH_CLIENT_SECRET=/ { print }' dankclaw/.env > \"\$dest\"" "$REMOTE_DEPLOY"
    grep -q "grep -q '\^GRAFANA_OAUTH_CLIENT_SECRET=\.' \"\$dest\"" "$REMOTE_DEPLOY"
    grep -q 'seed_grafana_env || return 1' "$REMOTE_DEPLOY"
}

@test "observability units hold no literal secret material" {
    # A secret-named setting is allowed only when its value is a ${VAR}
    # reference resolved from an EnvironmentFile systemd reads; a literal value
    # would be a tracked credential. Keys that merely mention a secret word
    # (GF_AUTH_GENERIC_OAUTH_TOKEN_URL, ..._CLIENT_ID) are not secrets, so the
    # match is anchored to the end of the key.
    local svc unit line key value
    for svc in "${UNITS[@]}"; do
        unit="$UNIT_DIR/$svc.container"
        while IFS= read -r line; do
            key="${line#Environment=}"
            value="${key#*=}"
            key="${key%%=*}"
            case "$key" in
                *_SECRET|*_PASSWORD|*_TOKEN|*_API_KEY)
                    case "$value" in
                        '${'*) ;;
                        *) echo "$svc hardcodes a literal secret in $key"; return 1 ;;
                    esac
                    ;;
            esac
        done < <(container_section "$unit" | grep '^Environment=')
        ! container_section "$unit" | grep -q '^EnvironmentFile=' \
            || { echo "$svc passes a whole env file into the container"; return 1; }
    done
}

@test "startup order follows the data dependencies" {
    # Grafana is useless without Prometheus and Alloy drops logs without Loki;
    # both pairs are expressed as systemd ordering, not launch luck.
    grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
    grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
    grep -q 'After=network-online.target loki.service' "$UNIT_DIR/alloy.container"
    grep -q 'Wants=network-online.target loki.service' "$UNIT_DIR/alloy.container"
}

@test "the deployer installs, restarts and health-checks each unit" {
    grep -q '^deploy_observability() {' "$REMOTE_DEPLOY"
    # Installs from the TARGET's own tree, not a hardcoded dankclaw path: the
    # two hosts own different observability units now (Dankclaw five, Dankserver
    # node-exporter), so the source directory has to follow $TARGET.
    grep -q 'local unit_dir="\$TARGET/observability/quadlet"' "$REMOTE_DEPLOY"
    grep -q 'install_quadlet observability "\$unit_dir" "\$service"' "$REMOTE_DEPLOY"
    grep -q 'systemctl --user restart "$service.service"' "$REMOTE_DEPLOY"
    local svc url
    while read -r svc url; do
        grep -q "wait_for_health $svc $url" "$REMOTE_DEPLOY" \
            || { echo "no health wait for $svc at $url"; return 1; }
    done <<'PAIRS'
prometheus http://127.0.0.1:9090/-/healthy
grafana http://127.0.0.1:3002/api/health
loki http://127.0.0.1:3100/ready
alloy http://127.0.0.1:12345/-/healthy
node-exporter http://127.0.0.1:9100/metrics
PAIRS
}

@test "the deployer never enables a generated unit" {
    # `systemctl --user enable X` fails on this host ("transient or generated").
    ! grep -q 'systemctl --user enable prometheus.service\|systemctl --user enable grafana.service\|systemctl --user enable loki.service\|systemctl --user enable alloy.service\|systemctl --user enable node-exporter.service' "$REMOTE_DEPLOY"
}

@test "the deployer rebuilds the locally built images before restarting them" {
    # loki and alloy are franksdank/<svc>:latest built from this repo; a unit
    # change that skipped the build would restart the previous image.
    grep -q 'build_compose_image loki' "$REMOTE_DEPLOY"
    grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY"
}

@test "no dankclaw compose file competes with the quadlet units" {
    # Two owners for one container name is how a compose run resurrects a
    # stale duplicate next to the quadlet-managed one. This file was retired on
    # 2026-09-16 — every container on the host reports a PODMAN_SYSTEMD_UNIT
    # label — and the invariant is that it stays retired: the four assertions
    # that used to grep it passed against the missing file, which is how a gate
    # goes hollow.
    local cf="$REPO_ROOT/dankclaw/compose.yaml"
    [ ! -f "$cf" ] || {
        echo "$cf is back: every dankclaw container is unit-managed, so a"
        echo "compose path can only spawn a duplicate fighting a unit for a name"
        return 1
    }
}

@test "changes under dankclaw/observability/ run this seam" {
    run "$REPO_ROOT/scripts/test-changed.sh" --list \
        --paths dankclaw/observability/quadlet/prometheus.container
    [ "$status" -eq 0 ]
    echo "$output" | grep -q 'tests/observability_quadlet.bats'
}

@test "the planner routes each unit to its own service" {
    local svc
    for svc in "${UNITS[@]}"; do
        run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
            --paths "dankclaw/observability/quadlet/$svc.container"
        [ "$status" -eq 0 ]
        [ "$output" = "$svc" ] || { echo "$svc planned as '$output'"; return 1; }
    done
}

@test "prometheus scrapes every LLM endpoint that exports metrics" {
    # The LLM endpoints are the services whose behaviour this fleet argues about
    # most (KV pressure, prefix-cache reuse, queue depth), and vLLM exports all
    # of it. If a scrape target drifts from the endpoint the units publish,
    # nobody notices until someone needs the graph during an incident.
    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
    grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-medium not scraped"; return 1; }
    grep -q '"192.168.0.143:7771"' "$cfg" || { echo "frank-small not scraped"; return 1; }
    # The medium endpoint is exported by the proxy on the host network; the
    # quadlet's own contract test pins that port, so name the job too.
    grep -q 'job_name: "frank-medium"' "$cfg"
    grep -q 'job_name: "frank-small"' "$cfg"
    # frank-large is NOT scraped on purpose: the halogen engine serves
    # /health, /cache and /v1/* and no metrics endpoint, so a target here would
    # be red by construction and would fire LlmEndpointDown forever. Assert the
    # absence, so a well-meaning edit cannot re-add a permanently-red probe.
    ! grep -q '192.168.0.143:7770' "$cfg"
    ! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg"
}

@test "grafana provisioning is declared in git and installed by the deployer" {
    # Datasources belong in the repository, not in grafana.db: a UI-configured
    # one is invisible to review and lost with the data volume.
    local ds="$REPO_ROOT/shared/grafana/provisioning/datasources/prometheus.yml"
    local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json"
    local prov="$REPO_ROOT/shared/grafana/provisioning/dashboards/provider.yml"
    [ -f "$ds" ] && [ -f "$dash" ] && [ -f "$prov" ]
    python3 -c "import json,sys; json.load(open('$dash'))"
    # Container-name address, like every other on-network scrape target: the
    # published port is loopback-only and would be wrong from Grafana.
    grep -q 'url: http://prometheus:9090' "$ds"
    grep -q 'uid: prometheus' "$ds"
    # The unit already mounts the directory; the deployer must fill it.
    grep -q '/opt/grafana/provisioning:/etc/grafana/provisioning:ro' \
        "$REPO_ROOT/dankclaw/observability/quadlet/grafana.container"
    grep -q 'copy_grafana_provisioning' "$REPO_ROOT/scripts/deploy-remote.sh"
}

@test "the planner selects grafana for shared/grafana changes" {
    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
        --paths shared/grafana/provisioning/datasources/prometheus.yml
    [ "$status" -eq 0 ]
    [ "$output" = "grafana" ]
}

@test "LLM alerting rules exist and are referenced by the config that ships them" {
    local rules="$REPO_ROOT/shared/prometheus/rules/llm-fleet.yml"
    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
    [ -f "$rules" ]
    python3 -c "
import yaml
d = yaml.safe_load(open('$rules'))
names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rules']]
assert 'record' or True
for want in ('LlmKvPoolSaturated', 'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
             'LlmEndpointDown', 'LlmLatencyHigh'):
    assert want in names, want
print('rules ok:', len(names))
"
    # A rule file that nothing loads evaluates nothing, and Prometheus reports
    # no error for a missing rule_files entry — the failure is invisible.
    grep -q '/etc/prometheus/rules/\*.yml' "$cfg"
    # ...and the deployer must install the directory the config points at.
    grep -q 'cp shared/prometheus/rules/llm-fleet.yml' "$REPO_ROOT/scripts/deploy-remote.sh"
    # The dashboard surfaces firing alerts, since nothing else will.
    python3 -c "
import json
d = json.load(open('$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json'))
exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
assert any('ALERTS' in e for e in exprs), exprs
"
}

@test "dankserver runs node-exporter as a boot-persistent unit, not from compose" {
    # Dankserver's node-exporter ran only from compose until 2026-09-16 and died
    # at a reboot (`Exited (143)`), leaving Prometheus' node-dankserver job
    # scraping nothing for a day and a half. It also bound the wildcard address,
    # which is the unauthenticated-metrics-on-every-interface posture already
    # removed on Dankclaw.
    local unit="$REPO_ROOT/dankserver/observability/quadlet/node-exporter.container"
    [ -f "$unit" ]
    grep -q '^WantedBy=default.target$' "$unit"
    grep -q '^Restart=always$' "$unit"
    grep -q '^Network=host$' "$unit"
    # Exactly two listen addresses: loopback for host-local probes and this
    # host's LAN address for Prometheus on Dankclaw. Never the wildcard — then
    # fix the half of the sentence the runtime cannot see.
    grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
    grep -q -- '--web.listen-address=192.168.0.235:9100' "$unit"
    ! grep -qE -- '--web\.listen-address=(0\.0\.0\.0|:9100|\[::\])' "$unit"
    # One owner per container name: the compose service must be gone, or two
    # definitions fight over the same container.
    ! grep -qE '^\s+node-exporter:' "$REPO_ROOT/dankserver/compose.yaml"
    # The deploy path has to know about it on this target.
    grep -q 'dankserver:node-exporter' "$REPO_ROOT/scripts/deploy-remote.sh"
    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
        --paths dankserver/observability/quadlet/node-exporter.container
    [ "$status" -eq 0 ]
    [ "$output" = "node-exporter" ]
}

@test "the observability handler installs units from the target's own tree" {
    # It used to hardcode dankclaw/observability/quadlet, so a dankserver unit
    # could never be installed by the shared dispatcher.
    local handler="$REPO_ROOT/scripts/deploy-remote.sh"
    grep -q 'local unit_dir="$TARGET/observability/quadlet"' "$handler"
    grep -q 'install_quadlet observability "$unit_dir" "$service"' "$handler"
}