serverAppConfigs/tests/frank_large_unit.bats

#!/usr/bin/env bats
# Seam: frank-large serves peonist-ai/halogen-qwen3.8-27b (a .hgn halogen
# checkpoint) on the Strix Halo iGPU through the halogen CONTAINER
# (ghcr.io/peonist-ai/halogen, digest-pinned), driven by the root unit's
# serve script — one conversation of 200,700 tokens with the DFlash2
# drafter ON. 2026-10-05 cutover from the ROCmFPX llama.cpp shape.
#
# This seam exists because the artifacts are a triangle that must agree on
# exact strings the engine reads: the image DIGEST, the checkpoint file
# name AND byte count inside the /models mount, and the tokenizer's
# flat-file shape (a HF cache snapshot's symlinks dangle inside the
# container). A serve gate and a deployer gate that disagree is the
# 2026-10-01 crash-loop; both sides are pinned here, plus the one engine
# law the fleet must never "optimize" away: HALOGEN_KV_SLOTS=1 is what
# keeps speculative decoding on.
#
# Everything is tested from what an operator or supervisor can observe —
# the command line podman would run, the exit code, the deployer's error
# text — never from anything inside the engine.

setup() {
    ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
    SERVE="$ROOT/scripts/serve-frank-large.sh"
    ENV_FILE="$ROOT/scripts/frank-large.env"
    UNIT="$ROOT/scripts/frank-large.service"
    DEPLOYER="$ROOT/scripts/deploy-remote.sh"
    BUILD="$ROOT/scripts/build-rocmfpx.sh"
    TEST_TMP="$(mktemp -d)"
    touch "$TEST_TMP/empty.env"
}

teardown() {
    rm -rf "$TEST_TMP"
}

# podman_stub — a fake podman on PATH that prints the argv of `run` and
# `wait` and answers `images` with whatever digests the test names, so
# "what would the supervisor run" is observable output and the digest
# gate is testable without a 3.5 GB image.
podman_stub() {
    mkdir -p "$TEST_TMP/bin"
    cat > "$TEST_TMP/bin/podman" <<'STUB'
#!/usr/bin/env bash
case "$1" in
    images) printf '%s\n' "${STUB_DIGESTS:-}" ;;
    rm)     exit 0 ;;
    run|wait) echo "PODMAN $*"; exit "${STUB_WAIT_RC:-0}" ;;
    *)      exit 0 ;;
esac
STUB
    chmod +x "$TEST_TMP/bin/podman"
}

# good_env — the minimum env the serve script demands, with the digest
# stub agrees on and zero-byte artifacts passing the size gate.
good_env() {
    : > "$TEST_TMP/empty.env"
    touch "$TEST_TMP/qwen.hgn" "$TEST_TMP/tok.gguf"
    GOOD_ENV=(
        "FRANK_LARGE_ENV_FILE=$TEST_TMP/empty.env"
        "FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen@sha256:cafe"
        "FRANK_LARGE_CONTAINER=frank-large-halogen"
        "FRANK_LARGE_LEGACY_CONTAINERS=frank-large"
        "FRANK_LARGE_DEV_KFD=$TEST_TMP/qwen.hgn"
        "FRANK_LARGE_DEV_DRI=$TEST_TMP"
        "FRANK_LARGE_LOG=$TEST_TMP/serve.log"
        "FRANK_LARGE_PUBLISH=7770:8731"
        "FRANK_LARGE_MODEL_DIR=$TEST_TMP"
        "FRANK_LARGE_CHECKPOINT=qwen.hgn"
        "FRANK_LARGE_CHECKPOINT_BYTES=0"
        "FRANK_LARGE_TOKENIZER=tok.gguf"
        "HALOGEN_MODEL_ID=frank-large"
        "HALOGEN_SLOT_CTX=200700"
        "HALOGEN_KV_SLOTS=1"
        "HALOGEN_DRAFTER=2"
        "STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen@sha256:cafe"
    )
}

@test "the env file defines the halogen deployment values" {
    grep -qE '^FRANK_LARGE_IMAGE=ghcr\.io/peonist-ai/halogen@sha256:[0-9a-f]{64}$' "$ENV_FILE"
    grep -qE '^FRANK_LARGE_CONTAINER=frank-large-halogen$' "$ENV_FILE"
    grep -qE '^FRANK_LARGE_PUBLISH=7770:8731$' "$ENV_FILE"
    grep -qE '^FRANK_LARGE_CHECKPOINT=.*\.hgn$' "$ENV_FILE"
    grep -qE '^FRANK_LARGE_CHECKPOINT_BYTES=[0-9]+$' "$ENV_FILE"
    grep -qE '^HALOGEN_MODEL_ID=frank-large$' "$ENV_FILE"
    grep -qE '^HALOGEN_SLOT_CTX=[0-9]+$' "$ENV_FILE"
}

@test "the drafter stays on: slots must stay 1" {
    # halogen makes batching and speculation mutually exclusive — with
    # more than one slot the drafter is DISABLED and per-stream decode
    # falls to serial speed (the sibling engine measured 41 -> 14 t/s
    # when slots went up). frank-large was a one-session endpoint before
    # this swap too (PARALLEL=1), so slots=1 preserves the contract;
    # raising it is a product decision, never a default change.
    grep -qE '^HALOGEN_KV_SLOTS=1$' "$ENV_FILE"
    grep -qE '^HALOGEN_DRAFTER=2$' "$ENV_FILE"
    # The cap/timeout pair is coupled (FLAGS.md table): a request at the
    # cap must not outlast the queue timeout, or everyone queued behind
    # it eats a 503.
    grep -qE '^HALOGEN_MAX_TOKENS_CAP=[0-9]+$' "$ENV_FILE"
    grep -qE '^HALOGEN_QUEUE_TIMEOUT=[0-9]+$' "$ENV_FILE"
}

@test "the serve script refuses when there is no GPU to serve on" {
    podman_stub
    good_env
    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
        FRANK_LARGE_DEV_KFD="$TEST_TMP/absent-node" "$SERVE"
    [ "$status" -ne 0 ]
    [[ "$output" == *"no gfx1151 to serve on"* ]]
}

@test "the KV budget fits unified memory with margin" {
    # Pool arithmetic is the engine's documented product: slots x
    # slot_ctx x 64 KiB. Against it: 33.4 GiB of mapped weights, the
    # engine working set + auto prompt cache (~25 GiB of headroom
    # allowed), host baseline and frank-small co-tenancy. The box has
    # 121 GiB unified; the 2026-09-16 halogen era ran the same engine
    # class here on a LARGER pool with less co-tenancy slack.
    ctx="$(sed -nE 's/^HALOGEN_SLOT_CTX=([0-9]+)$/\1/p' "$ENV_FILE")"
    slots="$(sed -nE 's/^HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")"
    pool_gib=$(( slots * ctx * 65536 / 1073741824 ))
    total_gib=$(( 34 + pool_gib + 25 + 15 ))
    [ "$total_gib" -le 121 ]
}

@test "the serve script refuses when the image is not in storage" {
    podman_stub
    good_env
    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" STUB_DIGESTS="" "$SERVE"
    [ "$status" -ne 0 ]
    [[ "$output" == *"not in podman storage"* ]]
    [[ "$output" == *"podman pull"* ]]
}

@test "the serve script refuses a missing checkpoint and names the fetch script" {
    podman_stub
    good_env
    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
        FRANK_LARGE_CHECKPOINT=absent.hgn "$SERVE"
    [ "$status" -ne 0 ]
    [[ "$output" == *"checkpoint"*"missing"* ]]
    [[ "$output" == *"download-frank-large-halogen.sh"* ]]
}

@test "the serve script refuses a checkpoint of the wrong byte count" {
    # A truncated curl exit-0 artifact must not boot; the model card's
    # byte count is the contract (fleet fetch law).
    podman_stub
    good_env
    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
        FRANK_LARGE_CHECKPOINT_BYTES=42 "$SERVE"
    [ "$status" -ne 0 ]
    [[ "$output" == *"expected 42"* ]]
}

@test "the serve script refuses a symlinked tokenizer" {
    # A HuggingFace cache snapshot's entries are symlinks into blobs/ and
    # dangle inside the container mount.
    podman_stub
    good_env
    ln -sf /nowhere/x "$TEST_TMP/link.gguf"
    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
        FRANK_LARGE_TOKENIZER=link.gguf "$SERVE"
    [ "$status" -ne 0 ]
    [[ "$output" == *"missing or a symlink"* ]]
}

@test "the render names the container, the port map and the engine shape" {
    podman_stub
    good_env
    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "$SERVE"
    [ "$status" -eq 0 ]
    # `podman run` output is redirected to the log by design (the unit
    # journal must not absorb engine chatter); the stub's printed argv
    # lands there, so the render assertions read the same log.
    output="$output
$(cat "$TEST_TMP/serve.log")"
    [[ "$output" == *"--name frank-large-halogen"* ]]
    [[ "$output" == *"--publish 7770:8731"* ]]
    [[ "$output" == *"-e HALOGEN_MODEL_ID=frank-large"* ]]
    [[ "$output" == *"-e HALOGEN_SLOT_CTX=200700"* ]]
    [[ "$output" == *"-e HALOGEN_KV_SLOTS=1"* ]]
    [[ "$output" == *"-e HALOGEN_DRAFTER=2"* ]]
    [[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]]
    [[ "$output" == *"--device $TEST_TMP/qwen.hgn"* ]]
    [[ "$output" == *"ghcr.io/peonist-ai/halogen@sha256:cafe"* ]]
    # The unit's restart policy is real because the script becomes
    # `podman wait`: the container's exit code is the service's exit.
    [[ "$output" == *"PODMAN wait frank-large-halogen"* ]]
}

@test "the shipped env pins the fleet's window and shape" {
    # 200700 is the per-conversation hard ceiling (a longer prompt is a
    # 400 naming the limit, never a silent truncation), and the harness
    # compaction math must fit prompt + budget under it: OMP compacts at
    # 70% = 140490, and 140490 + 57344 = 197834 <= 200700.
    grep -qE '^HALOGEN_SLOT_CTX=200700$' "$ENV_FILE"
    ctx=200700
    [ $(( ctx * 70 / 100 + 57344 )) -le "$ctx" ]
}

@test "no HIP-sensitive bare names leak through the env file" {
    # HIP reads bare env vars out of a `set -a` source; a tunable named
    # HOST or TEMP used to send the JIT cache to path 1.0 and segfault
    # every load (kept from the llama.cpp era — the container gets only
    # the explicit -e list, but the host env still reaches podman itself).
    ! grep -qE '^(HOST|TEMP|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
}

@test "the deployer gates exactly the artifacts the serve script opens" {
    # Serve and deploy must name the SAME image digest, checkpoint and
    # tokenizer: the 2026-10-01 crash-loop was exactly these twins
    # disagreeing mid-cutover.
    img="$(sed -nE 's/^FRANK_LARGE_IMAGE=(.*)$/\1/p' "$ENV_FILE")"
    grep -qF "LLM_LARGE_IMAGE=\"\${LLM_LARGE_IMAGE:-$img}\"" "$DEPLOYER"
    dir="$(sed -nE 's/^FRANK_LARGE_MODEL_DIR=(.*)$/\1/p' "$ENV_FILE")"
    ckpt="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT=(.*)$/\1/p' "$ENV_FILE")"
    tok="$(sed -nE 's/^FRANK_LARGE_TOKENIZER=(.*)$/\1/p' "$ENV_FILE")"
    grep -qF "LLM_LARGE_CKPT=\"\${LLM_LARGE_CKPT:-$dir/$ckpt}\"" "$DEPLOYER"
    grep -qF "LLM_LARGE_TOKENIZER=\"\${LLM_LARGE_TOKENIZER:-$dir/$tok}\"" "$DEPLOYER"
    bytes="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")"
    grep -qF "LLM_LARGE_CKPT_BYTES=\"\${LLM_LARGE_CKPT_BYTES:-$bytes}\"" "$DEPLOYER"
}

@test "the deployer points a missing artifact at the halogen fetch script" {
    grep -qE 'download-frank-large-halogen\.sh' "$DEPLOYER"
    grep -qE 'podman pull' "$DEPLOYER"
}

@test "the stray sweep cannot take frank-small or the deployer down" {
    # frank-small also listens on this host (7771, llama-server) and the
    # deployer's own command line contains "frank-large"; only the FULL
    # container name matches a stray manual halogen run and nothing else.
    grep -q "pkill -f 'frank-large-halogen'" "$DEPLOYER"
    ! grep -qE "pkill -f 'llama-server'" "$DEPLOYER"
    ! grep -qE "pkill -f 'frank-large'" "$DEPLOYER"
}

@test "the deployed unit keeps the fleet's restart and boot-persistence shape" {
    grep -qE '^Restart=(always|on-failure)$' "$UNIT"
    grep -qE '^WantedBy=' "$UNIT"
    grep -qE '^ExecStart=.*/serve-frank-large\.sh$' "$UNIT"
    grep -qE '^User=dank$' "$UNIT"
    grep -qE '^LimitMEMLOCK=infinity$' "$UNIT"
    # The service's lifetime is the container's; stopping the service
    # must stop the container by its real name.
    grep -qE '^ExecStop=.*podman stop .* frank-large-halogen$' "$UNIT"
}

@test "the unit runs the halogen runtime and no retired path" {
    grep -q 'halogen' "$UNIT"
    # Every retired payload/build stays on the host, so a stale path in
    # a unit still resolves — to a runtime nobody has measured in weeks.
    ! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT"
    ! grep -qE 'kingjones-ROCmFPX' "$UNIT"
}

@test "the rollback door stays qualified: the ROCmFPX build pin survives" {
    # Rolling back this swap means rebuilding the fork the old payload
    # needs; the pin (real commit, gfx1151, no webui fetch) is what makes
    # that door trustworthy months later.
    grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD"
    grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
    grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
}