serverAppConfigs/tests/frank_primary_unit.bats

#!/usr/bin/env bats
# Seam: frank-primary — the Dankserver LLM endpoint on :7770.
#
# This file asserts the tcclaviger RDNA4 vLLM ROOT-UNIT shape (the 2026-09-29
# cutover that replaced the R9V radiance fork): the serve script drives
# `podman run` DIRECTLY (the image entrypoint execs `vllm serve`), with every
# engine flag qualified by boot probe on 2026-09-29 and carried in the repo
# env file. The seam pins what the flags alone do not say: the qualification
# law (600k ctx / 3 sessions / q8-q8 KV were MEASURED to fit — 729,545 tokens
# of capacity at 7.2 GiB per rank — and KV_BYTES is PER RANK), the self-healing
# law (the run returns before the engine is up, so the serve script must wait
# on the container for Restart=on-failure to mean anything), the memlock law
# that makes the tiered-expert pin legal, the deployer's image/payload gates
# with the false-green MainPID check, and the prefetch law (expert prefetch
# measured at 36% precision — OFF is the incumbent; a regression that
# reintroduces it silently costs ~40-50% decode).

load helpers

UNIT="$REPO_ROOT/scripts/frank-primary.service"
SERVE="$REPO_ROOT/scripts/serve-frank-primary.sh"
ENV_FILE="$REPO_ROOT/scripts/frank-primary.env"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh"

@test "frank-primary unit, serve script and env exist in scripts/" {
    [ -f "$UNIT" ]
    [ -x "$SERVE" ]
    [ -f "$ENV_FILE" ]
}

@test "frank-primary is boot-persistent, self-healing and unprivileged" {
    # Root unit parity with frank-large/frank-small: enable is the [Install]
    # section, the restart policy survives a crash, and the tiered-expert
    # pin needs an unbounded memlock (a root unit grants it via PID1; the
    # dank session's own hard limit is 8 MiB and cannot be raised without
    # restarting user@1000.service).
    grep -q '^WantedBy=multi-user.target$' "$UNIT"
    grep -q '^Restart=on-failure$' "$UNIT"
    grep -q '^RestartSec=' "$UNIT"
    grep -q '^User=dank$' "$UNIT"
    grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
}

@test "frank-primary runs the repo's own serve script with the repo's env" {
    grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh$' "$UNIT"
    grep -q 'FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env' "$UNIT"
}

@test "the unit stops the container child explicitly" {
    # The serve script blocks on podman wait, which cannot stop the
    # container child; systemd stop goes through ExecStop or the engine
    # outlives the unit and holds both GPUs. The container name must match
    # the env file's identity.
    grep -q '^ExecStop=/usr/bin/podman stop' "$UNIT"
    grep -q 'frank-primary-qwen' "$UNIT"
    grep -q '^FRANK_PRIMARY_CONTAINER=frank-primary-qwen$' "$ENV_FILE"
}

@test "the serve shape is direct podman run, not a launcher or shim" {
    # The tcclaviger stack is fully expressed by flags; the R9V launcher,
    # docker->podman shim and setup-state layering are the RETIRED shape.
    grep -q 'podman run' "$SERVE"
    ! grep -q 'launch.sh' "$SERVE"
    ! grep -q 'podman-shim' "$SERVE"
    ! grep -q 'R9V_STATE_FILE' "$SERVE"
    ! grep -q -- '--model-draft' "$SERVE"
    ! grep -q -- '--spec-type' "$SERVE"
}

@test "the qualified engine shape is in the env file" {
    # 2026-09-29 boot probe: MAXLEN 200000 x SEQS 3 (600k total) at KV q8/q8
    # fits — the engine measured "GPU KV cache size: 729,545 tokens,
    # Maximum concurrency for 200,000 tokens per request: 3.65x" at
    # KV_BYTES 7.2 GiB per rank. KV_BYTES is PER RANK: sizing it to the
    # global requirement OOMs the worker (measured twice). fp8 KV is the
    # requested q8/q8 (e4m3 for both K and V together). MTP4 spec decode,
    # util 0.98 (0.99 fails the free-memory wall).
    grep -q '^FRANK_PRIMARY_MAXLEN=200000$' "$ENV_FILE"
    grep -q '^FRANK_PRIMARY_SEQS=3$' "$ENV_FILE"
    grep -q '^FRANK_PRIMARY_KV_BYTES=7730934784$' "$ENV_FILE"
    grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE"
    grep -q '^FRANK_PRIMARY_SPEC=4$' "$ENV_FILE"
    grep -q '^FRANK_PRIMARY_UTIL=0.98$' "$ENV_FILE"
}

@test "the serve script wires the engine flags from the env" {
    grep -q -- '--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE"' "$SERVE"
    grep -q -- '--kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES"' "$SERVE"
    grep -q -- '--max-num-seqs "$FRANK_PRIMARY_SEQS"' "$SERVE"
    grep -q -- '--max-model-len "$FRANK_PRIMARY_MAXLEN"' "$SERVE"
    grep -q -- '--tensor-parallel-size 2' "$SERVE"
    grep -q 'num_speculative_tokens' "$SERVE"
    grep -q -- '--enable-prefix-caching' "$SERVE"
    grep -q -- '--trust-remote-code' "$SERVE"
}

@test "expert prefetch is absent BY LAW, not by omission" {
    # Telemetry (2026-09-29): the prefetcher served 13,369 of 36,720
    # prefetched slabs (36% precision) — roughly half the gather traffic was
    # waste, and the "savings" would have been installs anyway. Deleting the
    # flag moved decode 26.8 -> 40.6 tok/s. Any future flag must come with a
    # fresh A/B, so the serve script must NOT wire it unconditionally.
    ! grep -q -- '--expert-prefetch' "$SERVE"
    ! grep -q 'expert_prefetch' "$ENV_FILE"
    grep -q 'PREFETCH IS ABSENT BY LAW' "$ENV_FILE"
}

@test "TunableOp stays off" {
    # CLAV_TUNABLEOP_SWEEP silently skips under expert offload unless
    # CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts numerics
    # (repetition loops, early halts, invalid measurements).
    ! grep -q 'TUNABLEOP' "$ENV_FILE"
    ! grep -q 'TUNABLEOP' "$SERVE"
}

@test "the serve script gates the digest-pinned image and the payload" {
    # The image is pinned by OCI manifest digest (the floating :latest moved
    # once already). The payload gate checks config.json and the exact shard
    # count from the env — a missing shard means a guaranteed long-timeout
    # boot instead of a fast failure.
    grep -q 'FRANK_PRIMARY_IMAGE' "$SERVE"
    grep -q 'RepoDigests' "$SERVE"
    grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$SERVE"
    grep -q 'config.json' "$SERVE"
    grep -q '/dev/kfd' "$SERVE"
    grep -q -- '--device /dev/dri' "$SERVE"
    ! grep -q 'tcclaviger/vllm:latest' "$ENV_FILE"
}

@test "the serve script clears stale containers and self-heals via podman wait" {
    # A crashed unit can leave the container name registered forever after;
    # the legacy identities (the retired R9V container and the ad hoc
    # experiment container) hold the same GPUs and die here too. Then block
    # on the container so a crash is a nonzero ExecStart.
    grep -q 'FRANK_PRIMARY_LEGACY_CONTAINERS' "$ENV_FILE"
    grep -q 'frank-primary-r9v' "$ENV_FILE"
    grep -q 'qwen-fp8' "$ENV_FILE"
    grep -q 'exec podman wait "$FRANK_PRIMARY_CONTAINER"' "$SERVE"
}

@test "frank-primary holds the single fleet port under its own alias" {
    # The scratch identity (qwen3.8-flash-next on :7773) must never leak
    # into production: LAN publish of :7770, the harness's model id.
    grep -q '^FRANK_PRIMARY_PUBLISH=7770:7770$' "$ENV_FILE"
    grep -q '^FRANK_PRIMARY_SERVED_NAME=frank-primary$' "$ENV_FILE"
}

@test "the model id matches the payload's served alias wiring" {
    # The payload dir and its shard count are the fetch script's output
    # layout; the fetch script must write where the env points.
    grep -q '^FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8$' "$ENV_FILE"
    grep -q '^FRANK_PRIMARY_MODEL_SHARDS=131$' "$ENV_FILE"
    grep -q 'Qwen3.8-Flash-Next-Uncensored-FP8' "$DOWNLOAD"
    grep -q 'orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8' "$DOWNLOAD"
}

@test "no bare HIP-reserved env var is defined" {
    # HIP reads bare TEMP for its JIT cache path (2026-08-13: TEMP=1.0
    # segfaulted every load), and HOST/GPU_MAX_HEAP_SIZE are the same
    # class. Every key here is FRANK_PRIMARY_-prefixed.
    ! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
    ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
}

@test "the deployer gates the image digest and payload, then health-checks" {
    # The deployer refuses when the pinned image is absent from rootless
    # storage or the payload is missing/wrong-sharded — then clears stale
    # containers (including the legacy identities), waits out the port,
    # relinks/enables/restarts the root unit and health-gates it.
    grep -q 'LLM_PRIMARY_CONTAINER=frank-primary-qwen' "$REMOTE_DEPLOY"
    grep -q 'FRANK_PRIMARY_IMAGE' "$REMOTE_DEPLOY"
    grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY"
    grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$REMOTE_DEPLOY"
    grep -q 'config.json' "$REMOTE_DEPLOY"
    grep -q 'qwen-fp8' "$REMOTE_DEPLOY"
    grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
    grep -q 'relink_unit frank-primary' "$REMOTE_DEPLOY"
    grep -q 'systemctl enable frank-primary.service' "$REMOTE_DEPLOY"
    grep -q 'systemctl restart frank-primary.service' "$REMOTE_DEPLOY"
    grep -q 'wait_for_health frank-primary http://127.0.0.1:7770/health 1800' "$REMOTE_DEPLOY"
}

@test "the deployer will not report green on a stray port-holder" {
    # The 2026-09-26 false green: a leftover probe held :7770 and satisfied
    # the health curl while the real unit was stopped. The deploy is only
    # green when the unit's own MainPID is live behind the health answer.
    grep -q 'systemctl show -p MainPID --value frank-primary.service' "$REMOTE_DEPLOY"
    grep -q 'stray process holds the port' "$REMOTE_DEPLOY"
}