serverAppConfigs/tests/frank_large_unit.bats

#!/usr/bin/env bats
# Seam: frank-large serves kingjones777's Qwen3.8-27B Q8_0_ROCMFPX_AGENT (a
# ftype-115 ROCmFPX file) with the repo's MTP draft head on the Strix Halo
# iGPU, via the ROCmFPX fork of llama.cpp — one session at the native
# 262,144 window.
#
# This seam exists because the format is the constraint: GGML_TYPE_Q8_0_ROCMFPX
# (tensor type 103) is understood by the ROCmFPX fork and by nothing else on
# this host, so the runtime, the payload and the deployer's gate have to agree
# on paths that no other endpoint uses. On 2026-10-01 a serve-script gate and
# its deployer twin disagreed during a cutover and the endpoint crash-looped;
# both sides are pinned here.
#
# Everything is tested from what an operator or a supervisor can observe — the
# command line, the exit code, the deployer's error text — never from anything
# inside llama.cpp.
setup() {
    load helpers

    SERVE="$REPO_ROOT/scripts/serve-frank-large.sh"
    UNIT="$REPO_ROOT/scripts/frank-large.service"
    ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
    DEPLOYER="$REPO_ROOT/scripts/deploy-remote.sh"
    BUILD="$REPO_ROOT/scripts/build-rocmfpx.sh"
    RUNTIME_STUB="/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server"
    MODEL_STUB="/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf"
    DRAFT_STUB="/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf"
    MMPROJ_STUB="/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf"

    TEST_TMP="$(mktemp -d)"
    SRC="$TEST_TMP/src"
    mkdir -p "$SRC"
    git -C "$SRC" init -q
    git -C "$SRC" -c user.email=t -c user.name=t commit -q --allow-empty -m x
}

teardown() {
    rm -rf "$TEST_TMP"
}

# make_stub — an executable that prints its whole argv on one line, so
# "what command line would the supervisor run" is observable output.
make_stub() {
    cat > "$TEST_TMP/bin" <<'STUB'
#!/bin/sh
printf '%s\n' "$*"
STUB
    chmod +x "$TEST_TMP/bin"
}

@test "the env file defines the deployment values" {
    grep -qE '^LLAMA_BIN=/' "$ENV_FILE"
    grep -qE '^MODEL_FILE=/' "$ENV_FILE"
    grep -qE '^DRAFT_FILE=/' "$ENV_FILE"
    grep -qE '^ALIAS=[a-z0-9-]+$' "$ENV_FILE"
    grep -qE '^LISTEN_PORT=[0-9]+$' "$ENV_FILE"
    grep -qE '^CTX_SIZE=[0-9]+$' "$ENV_FILE"
}

@test "the serve script fails when the runtime is missing" {
    # Only the model, the head and the projector exist; the fork was never
    # built. llama.cpp is not reached, so this fails fast and says what to
    # run instead of failing at bind time.
    touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
    run env "LLAMA_BIN=$TEST_TMP/nope/llama-server" \
        "MODEL_FILE=$TEST_TMP/model.gguf" "DRAFT_FILE=$TEST_TMP/draft.gguf" \
        "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
    [ "$status" -ne 0 ]
    [[ "$output" == *"build it with scripts/build-rocmfpx.sh"* ]]
}

@test "the serve script refuses to start without the model file" {
    make_stub
    touch "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
    run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/absent.gguf" \
        "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
    [ "$status" -ne 0 ]
    [[ "$output" == *"is not readable"* ]]
    [[ "$output" == *"download-frank-large-kingjones.sh"* ]]
}

@test "the serve script refuses to start without the MTP draft head" {
    make_stub
    touch "$TEST_TMP/model.gguf" "$TEST_TMP/mmproj.gguf"
    run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
        "DRAFT_FILE=$TEST_TMP/absent.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
    [ "$status" -ne 0 ]
    [[ "$output" == *"MTP draft head"* ]]
}

@test "the serve script refuses a missing mmproj unless vision is opted out" {
    make_stub
    touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf"
    run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
        "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/absent.gguf" "$SERVE"
    [ "$status" -ne 0 ]
    [[ "$output" == *"set MMPROJ_FILE='' for text-only"* ]]
    # The documented opt-out must actually start, without a --mmproj argument.
    run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
        "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=" "$SERVE"
    [ "$status" -eq 0 ]
    [[ "$output" != *"--mmproj"* ]]
}

@test "the serve script fails when LLAMA_BIN is empty" {
    touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
    run env LLAMA_BIN= "MODEL_FILE=$TEST_TMP/model.gguf" \
        "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
    [ "$status" -ne 0 ]
    [[ "$output" == *"LLAMA_BIN is empty"* ]]
}

@test "the shipped env points at the ROCmFPX runtime and the AGENT payload" {
    grep -qE "^LLAMA_BIN=$RUNTIME_STUB$" "$ENV_FILE"
    grep -qE "^MODEL_FILE=$MODEL_STUB$" "$ENV_FILE"
    grep -qE "^DRAFT_FILE=$DRAFT_STUB$" "$ENV_FILE"
    grep -qE "^MMPROJ_FILE=$MMPROJ_STUB$" "$ENV_FILE"
    # HIP reads bare HOST/TEMP/GPU_MAX_HEAP_SIZE out of the environment that
    # `set -a` creates; a tunable named HOST leaks into the runtime.
    ! grep -qE '^(HOST|TEMP|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
}

@test "the argv contract keeps the fleet's port and alias" {
    make_stub
    touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
    run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
        "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" \
        ALIAS=frank-large BIND_ADDR=127.0.0.1 LISTEN_PORT=7770 "$SERVE"
    [ "$status" -eq 0 ]
    # The caller's values must survive the env file, or a single-run operator
    # override silently does nothing.
    [[ "$output" == *"--host 127.0.0.1"* ]]
    [[ "$output" == *"--port 7770"* ]]
    [[ "$output" == *"--alias frank-large"* ]]
    # Vision is wired: the projector is the whole vision tower.
    [[ "$output" == *"--mmproj $TEST_TMP/mmproj.gguf"* ]]
}

@test "the shipped argv is one full native window on one session" {
    grep -qE '^CTX_SIZE=262144$' "$ENV_FILE"
    grep -qE '^PARALLEL=1$' "$ENV_FILE"
    # 262144 IS the model's trained window (general.context_length in the
    # GGUF), so 1 slot gets the whole thing — no truncation, no yarn scaling.
    #
    # Budget, from the GGUF header: 26.28 GiB of weights + 1.57 GiB draft head
    # + 0.87 GiB projector + 16 full-attention layers x 4 kv heads x 256 dim x
    # 2 (K+V) x 2 B = 64 KiB/token of f16 KV (~16 GiB at 262,144) + ~0.14 GiB
    # of DeltaNet state + a couple GiB of compute buffers, against the iGPU's
    # ~114 GiB GTT aperture. Change CTX_SIZE and this arithmetic, not a boot
    # log, is what tells you it no longer fits.
    ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
    kv_gib=$((ctx * 65536 / 1073741824))
    total_gib=$((27 + 2 + 1 + kv_gib + 3))
    [ "$total_gib" -le 114 ]
}

@test "the MTP draft contract matches the model card's measured optimum" {
    make_stub
    touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
    run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
        "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
    [ "$status" -eq 0 ]
    # The head is the model's own nextn layer, loaded as a separate draft
    # model because this checkpoint ships it standalone (no blk.<n>.nextn.*
    # tensors in the target file).
    [[ "$output" == *"--spec-type draft-mtp"* ]]
    [[ "$output" == *"--model-draft $TEST_TMP/draft.gguf"* ]]
    # n-max 4 is the measured optimum; n-max 8 measured WORSE (acceptance
    # fell to 0.757 and the extra verification cost outran the savings), so
    # the flag is a measurement, not a wish.
    [[ "$output" == *"--spec-draft-n-max 4"* ]]
    [[ "$output" == *"--spec-draft-n-min 0"* ]]
    [[ "$output" == *"--spec-draft-p-min 0.0"* ]]
    # The draft runs on the iGPU beside the target: no device-to-device hop
    # on a link that measures single-digit GB/s.
    [[ "$output" == *"--spec-draft-device ROCm0"* ]]
    [[ "$output" == *"--spec-draft-ngl 99"* ]]
}

@test "KV and placement stay unquantized with flash attention on" {
    make_stub
    touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
    run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
        "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
    [ "$status" -eq 0 ]
    [[ "$output" == *"--cache-type-k f16"* ]]
    [[ "$output" == *"--cache-type-v f16"* ]]
    # ROCmFPx quantized KV has no kernel on this branch, and the endpoint has
    # the aperture to keep KV unquantized anyway.
    [[ "$output" == *"--flash-attn on"* ]]
    # --fit's placement heuristics do not know that 311 tensors of type 103
    # belong on the iGPU, so the whole stack is placed explicitly instead.
    [[ "$output" == *"--fit off"* ]]
    [[ "$output" == *"--n-gpu-layers 999"* ]]
}

@test "the client contract wires thinking and template kwargs" {
    make_stub
    touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
    run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
        "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
    [ "$status" -eq 0 ]
    # Without --jinja, llama.cpp silently ignores the harness's
    # chat_template_kwargs, so thinking control disappears from the API.
    [[ "$output" == *"--jinja"* ]]
    # The harness reads `reasoning_content` for this endpoint, which is what
    # the deepseek emission format produces (vLLM's `reasoning` is not it).
    [[ "$output" == *"--reasoning-format deepseek"* ]]
    # The template needs an unbounded thinking budget.
    [[ "$output" == *"--reasoning-budget -1"* ]]
}

@test "the deployer gates exactly the paths the serve script opens" {
    # A gate that checks a different file than the engine opens is the
    # 2026-10-01 crash-loop: deploy passed, the unit restart-looped. Read the
    # engine's own paths out of the env file and require the deployer's
    # defaults to name the same ones.
    for pair in "LLAMA_BIN:LLM_LARGE_RUNTIME" \
                "MODEL_FILE:LLM_LARGE_MODEL" \
                "DRAFT_FILE:LLM_LARGE_DRAFT" \
                "MMPROJ_FILE:LLM_LARGE_MMPROJ"; do
        env_key="${pair%%:*}"
        dep_key="${pair##*:}"
        want="$(sed -nE "s|^${env_key}=(.*)\$|\1|p" "$ENV_FILE")"
        [ -n "$want" ]
        got="$(grep -oE "${dep_key}:-[^}]+" "$DEPLOYER" | head -1 | sed "s|^${dep_key}:-||")"
        [ "$got" = "$want" ]
    done
}

@test "the deployer points a missing runtime at the fork build script" {
    grep -qE 'build it: bash scripts/build-rocmfpx\.sh' "$DEPLOYER"
    grep -qE 'fetch it: bash scripts/download-frank-large-kingjones\.sh' "$DEPLOYER"
}

@test "the stray-process sweep cannot take frank-small down with frank-large" {
    # frank-small serves a different GGUF over llama-server on the SAME host.
    # A sweep keyed on the binary name (`pkill -f llama-server`) would kill
    # the 3090 endpoint too, so it is keyed on frank-large's own payload file.
    grep -q "pkill -f 'Qwen3.8-27B-Q8_0_ROCMFPX_AGENT'" "$DEPLOYER"
    ! grep -qE "pkill -f 'llama-server'" "$DEPLOYER"
}

@test "the build script refuses a source tree that is not the pinned commit" {
    # The pin is load-bearing: the older builds of this fork line segfault in
    # ggml_compute_forward_set_rows on this exact payload. A runtime that
    # differs from what was qualified must be heard about at deploy time.
    wrong="$(git -C "$SRC" rev-parse HEAD)"
    run env ROCMFPX_SRC_DIR="$SRC" ROCMFPX_COMMIT="ffff$wrong" \
        "$BUILD" --verify
    [ "$status" -ne 0 ]
    [[ "$output" == *"ROCMFPX_COMMIT"*mismatch* ]]
}

@test "the build script pins a real commit and the gfx1151 target" {
    grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD"
    grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
    grep -qE 'DGPU_TARGETS=' "$BUILD"
    # The webui asset fetch half-failed on this box and aborted the link at
    # 98%; the build must not depend on it.
    grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
}

@test "the deployed unit keeps the fleet's restart and boot-persistence shape" {
    grep -qE '^Restart=(always|on-failure)$' "$UNIT"
    grep -qE '^WantedBy=' "$UNIT"
    grep -qE '^ExecStart=.*/serve-frank-large\.sh$' "$UNIT"
    grep -qE '^User=dank$' "$UNIT"
    # The payload is 26 GiB of mmap'd weights plus pinned staging buffers; an
    # RLIMIT smaller than the pinned set fails the load in a way that looks
    # like an engine crash.
    grep -qE '^LimitMEMLOCK=infinity$' "$UNIT"
}

@test "the unit points at no retired frank-large runtime path" {
    # The host keeps every retired payload and build on disk, so a stale path
    # in a unit still resolves — to a runtime nobody has measured in weeks.
    ! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT"
    ! grep -q 'halogen' "$UNIT"
}