serverAppConfigs/tests/frank_primary_unit.bats

#!/usr/bin/env bats
# Seam: frank-primary — the Dankserver LLM endpoint on :7770.
#
# This file asserts the vLLM 0.30 ROOT-UNIT shape (the 2026-09-27 swap that
# replaced the llama.cpp b11030 endpoint): a locally built ROCm gfx1201
# image (localhost/frank-primary:vllm030) serving Qwen3.8-Flash-Next-FP8
# (Qwen4ExpForConditionalGeneration, 131 shards, 172.78 GiB) on the two
# R9700s with the UVA offload shape. The seam pins what the flags alone do
# not say: the offload budget arithmetic (the bandwidth-huge tensors — the
# 47.75 GiB PLE family and the 114.86 GiB routed experts — are pinned in
# host RAM and read in place, never streamed), the memlock law that makes
# the pin legal, the parser contract that keeps the harness wire formats
# (tool_calls + the reasoning channel) working, HIP env hygiene, and the
# deployer's image/payload gates with the false-green MainPID check.

load helpers

UNIT="$REPO_ROOT/scripts/frank-primary.service"
SERVE="$REPO_ROOT/scripts/serve-frank-primary.sh"
ENV_FILE="$REPO_ROOT/scripts/frank-primary.env"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"

@test "frank-primary unit, serve script and env exist in scripts/" {
    [ -f "$UNIT" ]
    [ -x "$SERVE" ]
    [ -f "$ENV_FILE" ]
}

@test "frank-primary is boot-persistent, self-healing and unprivileged" {
    # Root unit parity with frank-large/frank-small: enable is the [Install]
    # section, the restart policy survives a crash, and the ~164 GiB UVA pin
    # needs an unbounded memlock (a root unit grants it via PID1; the dank
    # session's own hard limit is 8 MiB and cannot be raised without
    # restarting user@1000.service).
    grep -q '^WantedBy=multi-user.target$' "$UNIT"
    grep -q '^Restart=on-failure$' "$UNIT"
    grep -q '^RestartSec=' "$UNIT"
    grep -q '^User=dank$' "$UNIT"
    grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
}

@test "frank-primary runs the repo's own serve script with the repo's env" {
    grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh$' "$UNIT"
    grep -q 'FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env' "$UNIT"
}

@test "the serve shape is vLLM: no llama.cpp-ism survives the cutover" {
    # Clean cutover: the GGUF era's flags would silently no-op or crash a vLLM
    # server, so their absence is part of the contract.
    ! grep -q -- '--model-draft' "$SERVE"
    ! grep -q -- '--spec-type' "$SERVE"
    ! grep -q -- '--mmproj' "$SERVE"
    ! grep -q -- '--override-tensor' "$SERVE"
    ! grep -q '\.gguf' "$SERVE"
    ! grep -q '\.gguf' "$ENV_FILE"
}

@test "the bandwidth-huge tensors are pinned in host RAM, not streamed" {
    # The UVA shape (the x4-pair law: streaming experts over PCIe loses,
    # computing them in host RAM does not care): --offload-backend uva pins
    # the matched weights with zero-copy reads. Measured from the payload
    # headers (2026-09-26): `ple` matches 138 tensors (131 n-gram table
    # shards + 7 per-layer conv/proj weights) = 47.75 GiB; `experts` matches
    # 150,528 tensors = 114.86 GiB; combined 162.60 GiB. cpu-offload-gb is
    # PER GPU: under TP=2 each rank needs ~81.3 GiB, hence 82 — 80/card
    # would strand 2.6 GiB of experts on the cards.
    grep -q '^OFFLOAD_BACKEND=uva$' "$ENV_FILE"
    grep -q '^OFFLOAD_GB=82$' "$ENV_FILE"
    grep -q '^OFFLOAD_PARAMS=ple,experts$' "$ENV_FILE"
    grep -q -- '--offload-backend "$OFFLOAD_BACKEND"' "$SERVE"
    grep -q -- '--cpu-offload-gb "$OFFLOAD_GB"' "$SERVE"
    grep -q -- '--cpu-offload-params "$OFFLOAD_PARAMS"' "$SERVE"
    # auto/prefetch are the copy paths: this deployment is only qualified
    # for uva, so the serve script refuses anything else.
    grep -q 'only qualified for uva' "$SERVE"
}

@test "the engine shape: TP=2 over both R9700s, full native window, batch 8" {
    grep -q '^TP=2$' "$ENV_FILE"
    grep -q '^GPU_UTIL=0.97$' "$ENV_FILE"
    grep -q '^CTX_SIZE=262144$' "$ENV_FILE"
    grep -q '^MAX_SEQS=8$' "$ENV_FILE"
    grep -q -- '--tensor-parallel-size "$TP"' "$SERVE"
    grep -q -- '--gpu-memory-utilization "$GPU_UTIL"' "$SERVE"
    grep -q -- '--max-model-len "$CTX_SIZE"' "$SERVE"
    grep -q -- '--max-num-seqs "$MAX_SEQS"' "$SERVE"
}

@test "the payload is the FP8 tree, vision served from the model's own config" {
    # Qwen4ExpForConditionalGeneration: vLLM auto-loads the ViT and the
    # mtp.* weights from the payload config — no explicit projector flag
    # exists in this runtime (the llama.cpp --mmproj trap does not apply).
    grep -q '^MODEL_DIR=/models/Qwen3.8-Flash-Next-FP8$' "$ENV_FILE"
    grep -q '^IMAGE=localhost/frank-primary:vllm030$' "$ENV_FILE"
}

@test "frank-primary holds the single fleet port under its own alias" {
    grep -q '^PORT=7770$' "$ENV_FILE"
    grep -q '^BIND_HOST=0.0.0.0$' "$ENV_FILE"
    grep -q '^SERVED_NAME=frank-primary$' "$ENV_FILE"
    grep -q -- '--served-model-name "$SERVED_NAME"' "$SERVE"
    grep -q -- '--host "$BIND_HOST"' "$SERVE"
    grep -q -- '--port "$PORT"' "$SERVE"
}

@test "tool_calls and the reasoning channel are wired for the harness" {
    # The wire contract: vLLM 0.30 emits the thinking channel as `reasoning`
    # (the deprecated `reasoning_content` is renamed server-side), the qwen3
    # parser extracts the template's think block into it, and qwen3_xml +
    # auto-tool-choice structure tool calls. Precedent: the host's own
    # Ornith-era scripts (~/vllm/failedAttempts/OrnithFP8.sh); the FP8
    # template consumes enable_thinking/preserve_thinking (verified in
    # tokenizer_config.json).
    grep -q '^TOOL_PARSER=qwen3_xml$' "$ENV_FILE"
    grep -q '^REASONING_PARSER=qwen3$' "$ENV_FILE"
    grep -q -- '--enable-auto-tool-choice --tool-call-parser "$TOOL_PARSER"' "$SERVE"
    grep -q -- '--reasoning-parser "$REASONING_PARSER"' "$SERVE"
    # The template kwargs are JSON: the env value must be single-quoted or
    # bash splits it and CHAT_TEMPLATE_KWARGS arrives empty (the -ot lesson).
    grep -q "^CHAT_TEMPLATE_KWARGS='{\"enable_thinking\": true, \"preserve_thinking\": true}'$" "$ENV_FILE"
    grep -q -- '--default-chat-template-kwargs "$CHAT_TEMPLATE_KWARGS"' "$SERVE"
}

@test "speculative decode is env-wired and OFF at bring-up" {
    # MTP drafting comes from the model's own mtp.* weights via
    # --speculative-config; bring-up measures the base shape clean, so
    # SPEC_CONFIG is empty and the flag is only added when non-empty.
    grep -q '^SPEC_CONFIG=$' "$ENV_FILE"
    grep -q -- '--speculative-config "$SPEC_CONFIG"' "$SERVE"
}

@test "both R9700s are visible and the Raphael iGPU is not" {
    grep -q '^ROCR_VISIBLE_DEVICES=0,1$' "$ENV_FILE"
    grep -q '^HIP_VISIBLE_DEVICES=0,1$' "$ENV_FILE"
    grep -q -- '-e ROCR_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"' "$SERVE"
    grep -q -- '-e HIP_VISIBLE_DEVICES="$HIP_VISIBLE_DEVICES"' "$SERVE"
}

@test "no bare HIP-reserved env var is defined" {
    # HIP reads bare TEMP for its JIT cache path (2026-08-13: TEMP=1.0
    # segfaulted every load), and HOST/GPU_MAX_HEAP_SIZE are the same
    # class. The bind address is BIND_HOST, not HOST.
    ! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
    grep -q '^BIND_HOST=' "$ENV_FILE"
    ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
}

@test "the serve script refuses to start on a half-finished payload or missing image" {
    # A payload missing its wrapper config or shard index turns the
    # container start into a guaranteed 20-minute timeout; the checks fail
    # fast with a pointer at the fetch script instead.
    grep -q 'config.json or index missing' "$SERVE"
    grep -q 'download-frank-primary-fp8.sh' "$SERVE"
    grep -q 'podman image inspect' "$SERVE"
}

@test "the deployer gates the image and the full shard set before starting" {
    # The deployer refuses when the image or any index-named shard is
    # absent (never serve a partial download), clears any stale container
    # that would break `podman run --name`, and waits out the port before
    # the new engine binds it.
    grep -q 'LLM_PRIMARY_IMAGE=localhost/frank-primary:vllm030' "$REMOTE_DEPLOY"
    grep -q 'LLM_PRIMARY_ROOT=/home/dank/models/Qwen3.8-Flash-Next-FP8' "$REMOTE_DEPLOY"
    grep -q 'weight_map' "$REMOTE_DEPLOY"
    grep -q 'download-frank-primary-fp8.sh' "$REMOTE_DEPLOY"
    grep -q "podman rm -f \"\$LLM_PRIMARY_CONTAINER\"" "$REMOTE_DEPLOY"
    grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
    grep -q 'relink_unit frank-primary' "$REMOTE_DEPLOY"
    grep -q 'systemctl enable frank-primary.service' "$REMOTE_DEPLOY"
    grep -q 'systemctl restart frank-primary.service' "$REMOTE_DEPLOY"
    grep -q 'wait_for_health frank-primary http://127.0.0.1:7770/health 1800' "$REMOTE_DEPLOY"
}

@test "the deployer will not report green on a stray port-holder" {
    # The 2026-09-26 false green: a leftover probe held :7770 and satisfied
    # the health curl while the real unit was stopped. The deploy is only
    # green when the unit's own MainPID is live behind the health answer.
    grep -q 'systemctl show -p MainPID --value frank-primary.service' "$REMOTE_DEPLOY"
    grep -q 'stray process holds the port' "$REMOTE_DEPLOY"
}