serverAppConfigs/tests/llm_units.bats

#!/usr/bin/env bash
# Seam: LLM systemd unit integrity.
# Every LLM unit (frank-*.service) must point at files that exist in
# this repo. The homepage-status unit is a different pattern (it installs to
# /usr/local/bin and runs from /srv at deploy time) and is not checked here.
# The units reference absolute paths on the target checkout
# (/home/dank/serverAppConfigs/...); the relative part of each path must
# resolve to a tracked file here. A unit that points at a retired checkout
# path (the old vllmScripts layout) fails this seam.

load helpers

TARGET_DIR="/home/dank/serverAppConfigs"

# unit_repo_path <unit-file> — for each absolute path in the unit's
# ExecStart and Environment lines, print the repo-relative path if it is
# rooted at $TARGET_DIR, or "MISSING:<abs>" if the relative file does not
# exist in the repo, or "FOREIGN:<abs>" if it is rooted elsewhere.
unit_repo_path() {
    local unit="$1"
    local line path rel
    while IFS= read -r line; do
        path="${line#*=}"
        path="${path#Environment=\"}"
        path="${path%\"}"
        # keep only the value after the last '=' for VAR=/abs/path pairs
        if [[ "$path" == *=* && "$path" != "$TARGET_DIR"* ]]; then
            path="${path##*=}"
        fi
        if [[ "$path" == "$TARGET_DIR"/* ]]; then
            rel="${path#"$TARGET_DIR"/}"
            if [ -e "$REPO_ROOT/$rel" ]; then
                printf 'OK\n'
            else
                printf 'MISSING:%s\n' "$path"
            fi
        else
            printf 'FOREIGN:%s\n' "$path"
        fi
    done < <(grep -E '^(ExecStart=|Environment=)' "$unit")
}

@test "every LLM unit points at the merged repo checkout" {
    local any=0 unit bad
    while IFS= read -r unit; do
        any=1
        bad=$(unit_repo_path "$unit" | grep -v '^OK$' || true)
        if [ -n "$bad" ]; then
            echo "unit $(basename "$unit") has bad paths:"
            echo "$bad"
            return 1
        fi
    done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
    [ "$any" -eq 1 ] || skip "no LLM units yet"
}

@test "every LLM unit references an env file that exists in the repo" {
    local unit envfile
    while IFS= read -r unit; do
        envfile=$(grep -oE 'Environment="[^"]*ENV_FILE=[^"]*"' "$unit" \
            | head -1 | sed -E 's/Environment="[^=]*=//; s/"$//')
        [ -n "$envfile" ] || continue
        local rel="${envfile#"$TARGET_DIR"/}"
        [ -f "$REPO_ROOT/$rel" ] || {
            echo "unit $(basename "$unit") env file $envfile not in repo"
            return 1
        }
    done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
}

@test "LLM serve scripts exist for every unit" {
    local unit svc script
    while IFS= read -r unit; do
        svc=$(basename "$unit" .service)
        script=$(grep -E '^ExecStart=' "$unit" | head -1 | sed 's/^ExecStart=//')
        local rel="${script#"$TARGET_DIR"/}"
        [ -f "$REPO_ROOT/$rel" ] || {
            echo "unit $svc: ExecStart script $script not in repo"
            return 1
        }
    done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
}

@test "every LLM root unit is self-healing against a voluntary exit" {
    # Restart=always, not on-failure. Every endpoint here is a container
    # runner (podman run) or a server whose front end can exit 0 on a
    # control-plane event; systemd treats exit 0 as a CLEAN stop and
    # on-failure leaves the endpoint dark until the next boot, with the
    # unit reporting success. No LLM unit may ship the weaker policy: a
    # single endpoint on on-failure is a daemon that goes dark silently.
    local unit
    while IFS= read -r unit; do
        grep -qx 'Restart=always' "$unit" || {
            echo "$(basename "$unit"): $(grep '^Restart=' "$unit") (must be always)"
            return 1
        }
    done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
}

@test "every LLM root unit restarts promptly and starts at boot" {
    # RestartSec=10 is the fleet's cadence: fast enough that a transient
    # fault is invisible to traffic, slow enough that a hard fault does not
    # spin the engine (each radiance boot re-pins ~147 GiB).
    # WantedBy=multi-user.target is boot persistence for a ROOT unit —
    # default.target is the USER-manager's target and means nothing here.
    local unit
    while IFS= read -r unit; do
        grep -qE '^RestartSec=([1-9]|1[0-9]|20)$' "$unit" || {
            echo "$(basename "$unit"): no RestartSec in 1..20"
            return 1
        }
        grep -qx 'WantedBy=multi-user.target' "$unit" || {
            echo "$(basename "$unit"): $(grep '^WantedBy=' "$unit")"
            return 1
        }
    done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
}