#!/usr/bin/env bats
# Seam: the five observability Quadlet units on dankclaw.
#
# prometheus, grafana, loki, alloy and node-exporter ran as compose services
# until 2026-09-11 and did not survive a reboot — the 2026-09-11 reboot for the
# amd_iommu=off change is what exposed it. They are rootless Quadlet units now.
# This seam pins the properties a reader cannot check by eye:
#
#   * boot persistence and self-healing on every unit;
#   * the published ports bind 127.0.0.1 and never a wildcard address, so no
#     unauthenticated metrics/dashboard/log endpoint is reachable off-host —
#     the whole point of moving them off compose, which published on 0.0.0.0;
#   * every port that IS published has a named consumer (the host-side homepage
#     status timer, or Prometheus scraping the host);
#   * the container names still match what Caddy proxies to and what
#     Prometheus' own scrape config targets, so the cutover broke no wiring;
#   * the deployer installs, restarts and health-checks them, and never tries
#     to `enable` a generated unit.

load helpers

UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
UNITS=(prometheus grafana loki alloy node-exporter alertmanager)

# container_section <unit> — the [Container] key/value block only
container_section() {
    awk '/^\[Container\]/{f=1;next} /^\[/{f=0} f' "$1"
}

# service_section <unit> — the [Service] key/value block only
service_section() {
    awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
}

@test "all six observability units exist under dankclaw/observability/quadlet/" {
    local svc
    for svc in "${UNITS[@]}"; do
        [ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
    done
}

@test "every observability unit is boot-persistent" {
    # Generated units cannot be enabled on this host's podman 5.x v1 quadlet
    # generator; [Install] + linger is the entire boot-persistence story.
    local svc
    for svc in "${UNITS[@]}"; do
        grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \
            || { echo "$svc is not boot-persistent"; return 1; }
    done
}

@test "every observability unit is self-healing" {
    local svc
    for svc in "${UNITS[@]}"; do
        grep -q '^Restart=always$' "$UNIT_DIR/$svc.container" \
            || { echo "$svc has no Restart policy"; return 1; }
        grep -q '^RestartSec=' "$UNIT_DIR/$svc.container" \
            || { echo "$svc has no RestartSec"; return 1; }
    done
}

@test "no observability unit publishes on a wildcard address" {
    # The compose services published 9090, 3002 and 3100 on 0.0.0.0 with no
    # authentication of any kind. Anything reachable off-host must go through
    # the guarded franksdank.cloud vhosts instead.
    local svc
    for svc in "${UNITS[@]}"; do
        ! grep -qE '^PublishPort=(0\.0\.0\.0|:)' "$UNIT_DIR/$svc.container" \
            || { echo "$svc publishes on a wildcard address"; return 1; }
    done
    ! grep -qE '^PublishPort=' "$UNIT_DIR/alloy.container" || true
}

@test "the four HTTP UIs publish on loopback only" {
    grep -q '^PublishPort=127.0.0.1:9090:9090$' "$UNIT_DIR/prometheus.container"
    grep -q '^PublishPort=127.0.0.1:3002:3000$' "$UNIT_DIR/grafana.container"
    grep -q '^PublishPort=127.0.0.1:3100:3100$' "$UNIT_DIR/loki.container"
    grep -q '^PublishPort=127.0.0.1:9093:9093$' "$UNIT_DIR/alertmanager.container"
}

@test "the homepage status timer's probe targets are the published loopback ports" {
    # scripts/homepage-status.py runs on the host and probes the `probe` field
    # of services.json from there, so a probe and a published port that
    # disagree show the card as down forever.
    local registry="$REPO_ROOT/shared/homepage/services.json"
    grep -q '"probe": "127.0.0.1:9090"' "$registry"
    grep -q '"probe": "127.0.0.1:3002"' "$registry"
    grep -q '"probe": "127.0.0.1:3100"' "$registry"
    grep -q '"probe": "127.0.0.1:9100"' "$registry"
    grep -q '"probe": "127.0.0.1:12345"' "$registry"
    grep -q '"probe": "127.0.0.1:9093"' "$registry"
}

@test "node-exporter binds the host namespace and both of its consumers" {
    # It reports the host's own interfaces/cgroups, so it needs the host netns
    # and host PID namespace; and it must listen exactly where its two
    # consumers are: the homepage timer on loopback, Prometheus (in a
    # container) on the LAN address.
    local unit="$UNIT_DIR/node-exporter.container"
    grep -q '^Network=host$' "$unit"
    grep -q -- '--pid=host' "$unit"
    grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
    grep -q -- '--web.listen-address=192.168.0.143:9100' "$unit"
    ! grep -qE '^PublishPort=' "$unit"
}

@test "prometheus scrapes node-exporter where node-exporter actually listens" {
    # Cross-file contract: if either side's address changes alone, host metrics
    # silently stop with no error anywhere.
    #
    # This Prometheus is a bridge container (Network=dankclaw_default), and on
    # this netavark host a container can only reach the host's LAN IP at a
    # PUBLISHED port. node-exporter is net=host, so its 192.168.0.143:9100 bind
    # is a host bind, not a published port — unreachable at its own LAN address.
    # The node-dankclaw job therefore scrapes host.containers.internal:9100,
    # which netavark delivers to the host's LAN IP, where node-exporter listens.
    # (node-dankserver is a different machine, so its LAN address is reached
    # over the real network and is scraped directly.)
    local unit="$UNIT_DIR/node-exporter.container"
    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
    local lan
    lan=$(grep -oE '192\.168\.0\.[0-9]+:9100' "$unit" | head -1)
    [ -n "$lan" ] || { echo "node-exporter has no LAN listen address"; return 1; }
    # The exporter must still bind that LAN address — it is where the
    # host.containers.internal delivery lands.
    grep -q -- "--web.listen-address=$lan" "$unit" \
        || { echo "node-exporter no longer binds $lan"; return 1; }
    # The local job scrapes via the gateway, not the (unreachable) LAN address.
    grep -q '"host.containers.internal:9100"' "$cfg" \
        || { echo "node-dankclaw does not scrape host.containers.internal:9100"; return 1; }
}

@test "container names still match what Caddy and Prometheus address by name" {
    # Caddy's post-auth handlers proxy to prometheus:9090 / grafana:3000 /
    # loki:3100 on the shared network, and Prometheus scrapes grafana:3000 and
    # loki:3100 the same way. A renamed container breaks all of it silently.
    grep -qE '^[[:space:]]*@grafana host grafana\.franksdank\.cloud$' "$REPO_ROOT/dankclaw/Caddyfile"
    grep -q 'reverse_proxy grafana:3000' "$REPO_ROOT/dankclaw/Caddyfile"
    grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile"
    grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile"
    local svc
    for svc in prometheus grafana loki alloy; do
        grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \
            || { echo "$svc unit renames its container"; return 1; }
        grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \
            || { echo "$svc is not on the network Caddy and its peers use"; return 1; }
    done
    grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
    grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
    grep -q 'url = "http://loki:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
}

@test "every observability unit has a health check" {
    # A container without one shows as plainly "Up" in podman ps while its
    # process is wedged, and the deploy has nothing to wait on.
    #
    # Four units declare it inline. loki cannot: its image is distroless, so a
    # unit HealthCmd would reach podman as CMD-SHELL and fail for want of
    # /bin/sh. Its health command ships in the image instead.
    local svc
    for svc in prometheus grafana alloy node-exporter alertmanager; do
        grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \
            || { echo "$svc has no HealthCmd"; return 1; }
        grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \
            || { echo "$svc has no HealthRetries"; return 1; }
    done
    ! grep -q '^HealthCmd=' "$UNIT_DIR/loki.container"
}

@test "loki's health command is exec form in the image, not a shell string" {
    # The whole reason the loki image exists: distroless has no shell, so the
    # command must be a JSON array that podman execs directly.
    local cf="$REPO_ROOT/dankclaw/loki/Containerfile"
    grep -q '^HEALTHCHECK ' "$cf"
    grep -q 'CMD \["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"\]' "$cf"
    # ... and the busybox it execs has to be in the image.
    grep -q 'COPY --from=tools /bin/busybox.static /usr/local/bin/busybox' "$cf"
}

@test "the loki/alloy/comfyui build uses the docker image format" {
    # Without --format docker the loki healthcheck is silently discarded (OCI
    # has no HEALTHCHECK field) and the container stops reporting health.
    grep -q -- '--format docker' "$REMOTE_DEPLOY"
    grep -q 'podman build --format docker --no-cache' "$REMOTE_DEPLOY"
}

@test "grafana gets its OAuth client secret without getting a whole env file" {
    # systemd expands ${GRAFANA_OAUTH_CLIENT_SECRET} for the [Container]
    # Environment= line from the [Service] EnvironmentFile, which keeps the
    # file out of the container entirely. That file is the per-service one the
    # deployer seeds (fleet convention: caddy, pihole, copyparty, romm, seerr)
    # — NOT a repo .env. The repo-root .env the compose service happened to
    # read is a gluetun env file carrying HOME=/root and the WireGuard keys;
    # importing it sets HOME for this service and podman then looks for its
    # configuration under /root and refuses to start.
    local unit="$UNIT_DIR/grafana.container"
    service_section "$unit" | grep -q '^EnvironmentFile=/home/dank/Containers/grafana/env$'
    ! service_section "$unit" | grep -q 'serverAppConfigs'
    ! container_section "$unit" | grep -q '^EnvironmentFile='
    container_section "$unit" | grep -q '^Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=\${GRAFANA_OAUTH_CLIENT_SECRET}$'
}

@test "the deployer seeds exactly one value into the grafana env file" {
    # The single-variable filter is the point: a whole-file copy is how the
    # HOME=/root failure got in.
    grep -q '^seed_grafana_env() {' "$REMOTE_DEPLOY"
    grep -q '^    local dest=/home/dank/Containers/grafana/env$' "$REMOTE_DEPLOY"
    grep -q "awk -F= '/^GRAFANA_OAUTH_CLIENT_SECRET=/ { print }' dankclaw/.env > \"\$dest\"" "$REMOTE_DEPLOY"
    grep -q "grep -q '\^GRAFANA_OAUTH_CLIENT_SECRET=\.' \"\$dest\"" "$REMOTE_DEPLOY"
    grep -q 'seed_grafana_env || return 1' "$REMOTE_DEPLOY"
}

@test "observability units hold no literal secret material" {
    # A secret-named setting is allowed only when its value is a ${VAR}
    # reference resolved from an EnvironmentFile systemd reads; a literal value
    # would be a tracked credential. Keys that merely mention a secret word
    # (GF_AUTH_GENERIC_OAUTH_TOKEN_URL, ..._CLIENT_ID) are not secrets, so the
    # match is anchored to the end of the key.
    local svc unit line key value
    for svc in "${UNITS[@]}"; do
        unit="$UNIT_DIR/$svc.container"
        while IFS= read -r line; do
            key="${line#Environment=}"
            value="${key#*=}"
            key="${key%%=*}"
            case "$key" in
                *_SECRET|*_PASSWORD|*_TOKEN|*_API_KEY)
                    case "$value" in
                        '${'*) ;;
                        *) echo "$svc hardcodes a literal secret in $key"; return 1 ;;
                    esac
                    ;;
            esac
        done < <(container_section "$unit" | grep '^Environment=')
        ! container_section "$unit" | grep -q '^EnvironmentFile=' \
            || { echo "$svc passes a whole env file into the container"; return 1; }
    done
}

@test "startup order follows the data dependencies" {
    # Grafana is useless without Prometheus and Alloy drops logs without Loki;
    # both pairs are expressed as systemd ordering, not launch luck.
    grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
    grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
    grep -q 'After=network-online.target loki.service' "$UNIT_DIR/alloy.container"
    grep -q 'Wants=network-online.target loki.service' "$UNIT_DIR/alloy.container"
}

@test "the deployer installs, restarts and health-checks each unit" {
    grep -q '^deploy_observability() {' "$REMOTE_DEPLOY"
    # Installs from the TARGET's own tree, not a hardcoded dankclaw path: the
    # two hosts own different observability units now (Dankclaw five, Dankserver
    # node-exporter), so the source directory has to follow $TARGET.
    grep -q 'local unit_dir="\$TARGET/observability/quadlet"' "$REMOTE_DEPLOY"
    grep -q 'install_quadlet observability "\$unit_dir" "\$service"' "$REMOTE_DEPLOY"
    grep -q 'systemctl --user restart "$service.service"' "$REMOTE_DEPLOY"
    local svc url
    while read -r svc url; do
        grep -q "wait_for_health $svc $url" "$REMOTE_DEPLOY" \
            || { echo "no health wait for $svc at $url"; return 1; }
    done <<'PAIRS'
prometheus http://127.0.0.1:9090/-/healthy
grafana http://127.0.0.1:3002/api/health
loki http://127.0.0.1:3100/ready
alloy http://127.0.0.1:12345/-/healthy
node-exporter http://127.0.0.1:9100/metrics
alertmanager http://127.0.0.1:9093/-/healthy
PAIRS
}

@test "the deployer never enables a generated unit" {
    # `systemctl --user enable X` fails on this host ("transient or generated").
    ! grep -q 'systemctl --user enable prometheus.service\|systemctl --user enable grafana.service\|systemctl --user enable loki.service\|systemctl --user enable alloy.service\|systemctl --user enable node-exporter.service' "$REMOTE_DEPLOY"
}

@test "the deployer rebuilds the locally built images before restarting them" {
    # loki and alloy are franksdank/<svc>:latest built from this repo; a unit
    # change that skipped the build would restart the previous image.
    grep -q 'build_compose_image loki' "$REMOTE_DEPLOY"
    grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY"
}

@test "no dankclaw compose file competes with the quadlet units" {
    # Two owners for one container name is how a compose run resurrects a
    # stale duplicate next to the quadlet-managed one. This file was retired on
    # 2026-09-16 — every container on the host reports a PODMAN_SYSTEMD_UNIT
    # label — and the invariant is that it stays retired: the four assertions
    # that used to grep it passed against the missing file, which is how a gate
    # goes hollow.
    local cf="$REPO_ROOT/dankclaw/compose.yaml"
    [ ! -f "$cf" ] || {
        echo "$cf is back: every dankclaw container is unit-managed, so a"
        echo "compose path can only spawn a duplicate fighting a unit for a name"
        return 1
    }
}

@test "changes under dankclaw/observability/ run this seam" {
    run "$REPO_ROOT/scripts/test-changed.sh" --list \
        --paths dankclaw/observability/quadlet/prometheus.container
    [ "$status" -eq 0 ]
    echo "$output" | grep -q 'tests/observability_quadlet.bats'
}

@test "the planner routes each unit to its own service" {
    local svc
    for svc in "${UNITS[@]}"; do
        run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
            --paths "dankclaw/observability/quadlet/$svc.container"
        [ "$status" -eq 0 ]
        [ "$output" = "$svc" ] || { echo "$svc planned as '$output'"; return 1; }
    done
}

@test "prometheus scrapes every LLM endpoint that exports metrics" {
    # The LLM endpoints are the services whose behaviour this fleet argues about
    # most (KV pressure, prefix-cache reuse, queue depth). Both Dankclaw
    # endpoints export the llamacpp:* family. frank-archon LEFT the set
    # 2026-10-05: the FreeToken PR587 ROCm engine has NO /metrics endpoint —
    # scraping it was a structural never-green (404), so the job was deleted
    # rather than left vacuous (the Lucebox DS4.1 precedent, 2026-09-26).
    # The two Dankclaw endpoints are HOST processes reached through
    # host.containers.internal: this Prometheus is a bridge-networked
    # container, and on this netavark host a container can only reach the
    # host's LAN IP at a PUBLISHED port — a host-process bind like
    # llama-server's 0.0.0.0:7770 is refused at 192.168.0.143:7770 while it
    # answers at 169.254.1.2:7770 (why frank-zealot stayed red by construction
    # for days until the 2026-09-20 /targets audit).
    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
    grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-zealot not scraped"; return 1; }
    grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-dragoon not scraped"; return 1; }
    ! grep -q '"192.168.0.235:7770"' "$cfg"
    grep -q '"host.containers.internal:7770,192.168.0.235:7770"' "$cfg" \
        && { echo "frank-archon folded into a Dankclaw gateway target"; return 1; }
    for job in frank-zealot frank-dragoon; do
        grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; }
    done
    ! grep -q 'job_name: "frank-archon"' "$cfg"
    # None of them is reached through a guarded vhost: Prometheus cannot solve
    # an Anubis challenge, so a franksdank.cloud target is red by construction.
    ! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg"
}

@test "grafana provisioning is declared in git and installed by the deployer" {
    # Datasources belong in the repository, not in grafana.db: a UI-configured
    # one is invisible to review and lost with the data volume.
    local ds="$REPO_ROOT/shared/grafana/provisioning/datasources/prometheus.yml"
    local dash="$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json"
    local prov="$REPO_ROOT/shared/grafana/provisioning/dashboards/provider.yml"
    [ -f "$ds" ] && [ -f "$dash" ] && [ -f "$prov" ]
    python3 -c "import json,sys; json.load(open('$dash'))"
    # Container-name address, like every other on-network scrape target: the
    # published port is loopback-only and would be wrong from Grafana.
    grep -q 'url: http://prometheus:9090' "$ds"
    grep -q 'uid: prometheus' "$ds"
    # The unit already mounts the directory; the deployer must fill it.
    grep -q '/opt/grafana/provisioning:/etc/grafana/provisioning:ro' \
        "$REPO_ROOT/dankclaw/observability/quadlet/grafana.container"
    grep -q 'copy_grafana_provisioning' "$REPO_ROOT/scripts/deploy-remote.sh"
}

@test "the planner selects grafana for shared/grafana changes" {
    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
        --paths shared/grafana/provisioning/datasources/prometheus.yml
    [ "$status" -eq 0 ]
    [ "$output" = "grafana" ]
}

@test "LLM alerting rules exist and are referenced by the config that ships them" {
    local rules="$REPO_ROOT/shared/prometheus/rules/llm-fleet.yml"
    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
    [ -f "$rules" ]
    python3 -c "
import re, yaml
d = yaml.safe_load(open('$rules'))
names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rules']]
for want in ('job:llm_prefix_cache_reuse:rate5m',
             'job:llm_spec_accept_length:rate5m',
             'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
             'LlmSpecDecodeOff', 'LlmEndpointDown'):
    assert want in names, want
# Every rule must be keyed to a series that actually exists. A rule naming a
# series nobody exports evaluates nothing forever and errors never — the
# failure this seam exists for. Since the 2026-09-26 cutover the only scraped
# LLM runtime is llama.cpp (frank-zealot, frank-dragoon); the vllm: family left
# with frank-medium's radiance runtime and must not survive in any expr.
exprs = [r.get('expr', '') for g in d['groups'] for r in g['rules']]
# llama.cpp families measured from /metrics on 2026-09-16 and 2026-09-20
# (frank-zealot, frank-dragoon). Anything else is a guess.
live = {'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
        'llamacpp:requests_deferred', 'llamacpp:requests_processing',
        'llamacpp:spec_decode_num_drafts_total',
        'llamacpp:spec_decode_num_accepted_tokens_total',
        'llamacpp:spec_decode_num_draft_tokens_total',
        'llamacpp:tokens_predicted_total',
        'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
        'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode'}
used = {m for e in exprs
        for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', e)}
assert used <= live, used - live
assert not any(':vllm' in e for e in exprs), 'vllm: series in rules but no vllm endpoint is scraped'
print('rules ok:', len(names))
"
    # A rule file that nothing loads evaluates nothing, and Prometheus reports
    # no error for a missing rule_files entry — the failure is invisible.
    grep -q '/etc/prometheus/rules/\*.yml' "$cfg"
    # ...and the deployer must install the directory the config points at.
    grep -q 'cp shared/prometheus/rules/llm-fleet.yml' "$REPO_ROOT/scripts/deploy-remote.sh"
    # The dashboard surfaces firing alerts, since nothing else will.
    python3 -c "
import json
d = json.load(open('$REPO_ROOT/shared/grafana/provisioning/dashboards/llm-fleet.json'))
exprs = [t['expr'] for p in d['panels'] for t in p.get('targets', [])]
assert any('ALERTS' in e for e in exprs), exprs
"
}

@test "dankserver runs node-exporter as a boot-persistent unit, not from compose" {
    # Dankserver's node-exporter ran only from compose until 2026-09-16 and died
    # at a reboot (`Exited (143)`), leaving Prometheus' node-dankserver job
    # scraping nothing for a day and a half. It also bound the wildcard address,
    # which is the unauthenticated-metrics-on-every-interface posture already
    # removed on Dankclaw.
    local unit="$REPO_ROOT/dankserver/observability/quadlet/node-exporter.container"
    [ -f "$unit" ]
    grep -q '^WantedBy=default.target$' "$unit"
    grep -q '^Restart=always$' "$unit"
    grep -q '^Network=host$' "$unit"
    # Exactly two listen addresses: loopback for host-local probes and this
    # host's LAN address for Prometheus on Dankclaw. Never the wildcard — then
    # fix the half of the sentence the runtime cannot see.
    grep -q -- '--web.listen-address=127.0.0.1:9100' "$unit"
    grep -q -- '--web.listen-address=192.168.0.235:9100' "$unit"
    ! grep -qE -- '--web\.listen-address=(0\.0\.0\.0|:9100|\[::\])' "$unit"
    # One owner per container name: the compose service must be gone, or two
    # definitions fight over the same container.
    ! grep -qE '^\s+node-exporter:' "$REPO_ROOT/dankserver/compose.yaml"
    # The deploy path has to know about it on this target.
    grep -q 'dankserver:node-exporter' "$REPO_ROOT/scripts/deploy-remote.sh"
    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
        --paths dankserver/observability/quadlet/node-exporter.container
    [ "$status" -eq 0 ]
    [ "$output" = "node-exporter" ]
}

@test "the observability handler installs units from the target's own tree" {
    # It used to hardcode dankclaw/observability/quadlet, so a dankserver unit
    # could never be installed by the shared dispatcher.
    local handler="$REPO_ROOT/scripts/deploy-remote.sh"
    grep -q 'local unit_dir="$TARGET/observability/quadlet"' "$handler"
    grep -q 'install_quadlet observability "$unit_dir" "$service"' "$handler"
}

@test "alertmanager delivers to the Signal relay through the podman host gateway" {
    # The relay listens on the HOST's loopback. From the container that
    # address is only reachable as host.containers.internal (netavark DNATs
    # published ports only — the LAN IP and the gateway are refused), so the
    # webhook URL and the relay's listen port are a cross-file contract: if
    # either half changes alone, every alert is dropped with no error
    # anywhere.
    local am="$REPO_ROOT/shared/prometheus/alertmanager.yml"
    local relay="$REPO_ROOT/scripts/signal-relay.py"
    [ -f "$am" ] && [ -f "$relay" ]
    grep -q 'host.containers.internal:18093/alertmanager' "$am"
    # The relay must listen exactly where the webhook points.
    grep -q '18093' "$relay"
    grep -q '"/alertmanager"' "$relay"
}

@test "prometheus ships alerts to alertmanager on the shared network" {
    # Container-name address, like grafana and loki: the published port is
    # loopback-only and would be wrong from inside Prometheus.
    local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
    grep -q 'alertmanagers:' "$cfg"
    grep -q '"alertmanager:9093"' "$cfg"
}

@test "the signal relay is a boot-persistent user unit the deployer installs" {
    # Plain user unit, not a quadlet: it runs python3 from the checkout and
    # can be enabled (generated quadlet units cannot on this host). Linger is
    # on for dank, so [Install] + enable is the whole boot story.
    local unit="$REPO_ROOT/scripts/signal-relay.service"
    local handler="$REPO_ROOT/scripts/deploy-remote.sh"
    [ -f "$unit" ]
    grep -q '^WantedBy=default.target$' "$unit"
    grep -q '^Restart=always$' "$unit"
    grep -q '^ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/signal-relay.py$' "$unit"
    grep -q '^install_signal_relay() {' "$handler"
    grep -q 'systemctl --user enable signal-relay.service' "$handler"
    grep -q 'systemctl --user restart signal-relay.service' "$handler"
    # The alertmanager deploy is what reaches the relay on the host.
    grep -q 'install_signal_relay' <(sed -n '/^deploy_observability()/,/^}/p' "$handler")
}

@test "the relay sends through the existing signal-cli account, bridge-subnet only" {
    # The credential is the account state under ~/.local/share/signal-cli;
    # the number and account name are numbers, not secrets. The listener
    # binds the host's LAN IP — the one address a bridge container can
    # reach as host.containers.internal (measured 2026-09-17: loopback
    # refuses, 169.254.1.2 is not local) — and the deployer admits the
    # podman bridge subnet only, so the unauthenticated endpoint stays off
    # the rest of the LAN.
    local relay="$REPO_ROOT/scripts/signal-relay.py"
    local handler="$REPO_ROOT/scripts/deploy-remote.sh"
    grep -q 'signal-cli' "$relay"
    grep -q 'SIGNAL_CLI = "/usr/local/bin/signal-cli"' "$relay"
    grep -q 'SIGNAL_ACCOUNT = "+12062476336"' "$relay"
    grep -q 'LISTEN_ADDR = ("192.168.0.143", 18093)' "$relay"
    grep -q -- '--dport 18093 -s 127.0.0.1 -j ACCEPT' "$handler"
    grep -q -- '--dport 18093 -s 192.168.0.143 -j ACCEPT' "$handler"
    grep -q -- '--dport 18093 -s 10.89.0.0/16 -j ACCEPT' "$handler"
    grep -q -- '--dport 18093 -j DROP' "$handler"
}

@test "the planner selects alertmanager for its config and relay files" {
    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
        --paths shared/prometheus/alertmanager.yml
    [ "$status" -eq 0 ]
    echo "$output" | grep -qx 'alertmanager'
    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
        --paths scripts/signal-relay.py
    [ "$status" -eq 0 ]
    [ "$output" = "alertmanager" ]
}