#!/usr/bin/env bats # Seam: frank-large serves peonist-ai/halogen-qwen3.8-27b (a .hgn halogen # checkpoint) on the Strix Halo iGPU through the halogen CONTAINER # (ghcr.io/peonist-ai/halogen, digest-pinned), driven by the root unit's # serve script — one conversation of 200,700 tokens with the DFlash2 # drafter ON. 2026-10-05 cutover from the ROCmFPX llama.cpp shape. # # This seam exists because the artifacts are a triangle that must agree on # exact strings the engine reads: the image DIGEST, the checkpoint file # name AND byte count inside the /models mount, and the tokenizer's # flat-file shape (a HF cache snapshot's symlinks dangle inside the # container). A serve gate and a deployer gate that disagree is the # 2026-10-01 crash-loop; both sides are pinned here, plus the one engine # law the fleet must never "optimize" away: HALOGEN_KV_SLOTS=1 is what # keeps speculative decoding on. # # Everything is tested from what an operator or supervisor can observe — # the command line podman would run, the exit code, the deployer's error # text — never from anything inside the engine. setup() { ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)" SERVE="$ROOT/scripts/serve-frank-large.sh" ENV_FILE="$ROOT/scripts/frank-large.env" UNIT="$ROOT/scripts/frank-large.service" DEPLOYER="$ROOT/scripts/deploy-remote.sh" BUILD="$ROOT/scripts/build-rocmfpx.sh" TEST_TMP="$(mktemp -d)" touch "$TEST_TMP/empty.env" } teardown() { rm -rf "$TEST_TMP" } # podman_stub — a fake podman on PATH that prints the argv of `run` and # `wait` and answers `images` with whatever digests the test names, so # "what would the supervisor run" is observable output and the digest # gate is testable without a 3.5 GB image. podman_stub() { mkdir -p "$TEST_TMP/bin" cat > "$TEST_TMP/bin/podman" <<'STUB' #!/usr/bin/env bash case "$1" in images) printf '%s\n' "${STUB_DIGESTS:-}" ;; rm) exit 0 ;; run|wait) echo "PODMAN $*"; exit "${STUB_WAIT_RC:-0}" ;; *) exit 0 ;; esac STUB chmod +x "$TEST_TMP/bin/podman" } # good_env — the minimum env the serve script demands, with the digest # stub agrees on and zero-byte artifacts passing the size gate. good_env() { : > "$TEST_TMP/empty.env" touch "$TEST_TMP/qwen.hgn" mkdir -p "$TEST_TMP/tokenizer" && touch "$TEST_TMP/tokenizer/tok.json" GOOD_ENV=( "FRANK_LARGE_ENV_FILE=$TEST_TMP/empty.env" "FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen@sha256:cafe" "FRANK_LARGE_CONTAINER=frank-large-halogen" "FRANK_LARGE_LEGACY_CONTAINERS=frank-large" "FRANK_LARGE_DEV_KFD=$TEST_TMP/qwen.hgn" "FRANK_LARGE_DEV_DRI=$TEST_TMP" "FRANK_LARGE_LOG=$TEST_TMP/serve.log" "FRANK_LARGE_PUBLISH=7770:8731" "FRANK_LARGE_MODEL_DIR=$TEST_TMP" "FRANK_LARGE_CHECKPOINT=qwen.hgn" "FRANK_LARGE_CHECKPOINT_BYTES=0" "FRANK_LARGE_TOKENIZER_FILES=tok.json" "HALOGEN_MODEL_ID=frank-large" "HALOGEN_SLOT_CTX=200700" "HALOGEN_KV_SLOTS=1" "HALOGEN_DRAFTER=2" "STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen@sha256:cafe" ) } @test "the env file defines the halogen deployment values" { grep -qE '^FRANK_LARGE_IMAGE=ghcr\.io/peonist-ai/halogen@sha256:[0-9a-f]{64}$' "$ENV_FILE" grep -qE '^FRANK_LARGE_CONTAINER=frank-large-halogen$' "$ENV_FILE" grep -qE '^FRANK_LARGE_PUBLISH=7770:8731$' "$ENV_FILE" grep -qE '^FRANK_LARGE_CHECKPOINT=.*\.hgn$' "$ENV_FILE" grep -qE '^FRANK_LARGE_CHECKPOINT_BYTES=[0-9]+$' "$ENV_FILE" grep -qE '^FRANK_LARGE_CHECKPOINT_SHA256=[0-9a-f]{64}$' "$ENV_FILE" grep -qE '^FRANK_LARGE_TOKENIZER_FILES="tokenizer\.json( [a-z_.]+)+"$' "$ENV_FILE" grep -qE '^HALOGEN_MODEL_ID=frank-large$' "$ENV_FILE" grep -qE '^HALOGEN_SLOT_CTX=[0-9]+$' "$ENV_FILE" } @test "the drafter stays on: slots must stay 1" { # halogen makes batching and speculation mutually exclusive — with # more than one slot the drafter is DISABLED and per-stream decode # falls to serial speed (the sibling engine measured 41 -> 14 t/s # when slots went up). frank-large was a one-session endpoint before # this swap too (PARALLEL=1), so slots=1 preserves the contract; # raising it is a product decision, never a default change. grep -qE '^HALOGEN_KV_SLOTS=1$' "$ENV_FILE" grep -qE '^HALOGEN_DRAFTER=2$' "$ENV_FILE" # The cap/timeout pair is coupled (FLAGS.md table): a request at the # cap must not outlast the queue timeout, or everyone queued behind # it eats a 503. grep -qE '^HALOGEN_MAX_TOKENS_CAP=[0-9]+$' "$ENV_FILE" grep -qE '^HALOGEN_QUEUE_TIMEOUT=[0-9]+$' "$ENV_FILE" } @test "the serve script refuses when there is no GPU to serve on" { podman_stub good_env run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ FRANK_LARGE_DEV_KFD="$TEST_TMP/absent-node" "$SERVE" [ "$status" -ne 0 ] [[ "$output" == *"no gfx1151 to serve on"* ]] } @test "the KV budget fits unified memory with margin" { # Pool arithmetic is the engine's documented product: slots x # slot_ctx x 64 KiB. Against it: 33.4 GiB of mapped weights, the # engine working set + auto prompt cache (~25 GiB of headroom # allowed), host baseline and frank-small co-tenancy. The box has # 121 GiB unified; the 2026-09-16 halogen era ran the same engine # class here on a LARGER pool with less co-tenancy slack. ctx="$(sed -nE 's/^HALOGEN_SLOT_CTX=([0-9]+)$/\1/p' "$ENV_FILE")" slots="$(sed -nE 's/^HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" pool_gib=$(( slots * ctx * 65536 / 1073741824 )) total_gib=$(( 34 + pool_gib + 25 + 15 )) [ "$total_gib" -le 121 ] } @test "the serve script refuses when the image is not in storage" { podman_stub good_env run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" STUB_DIGESTS="" "$SERVE" [ "$status" -ne 0 ] [[ "$output" == *"not in podman storage"* ]] [[ "$output" == *"podman pull"* ]] } @test "the serve script refuses a missing checkpoint and names the fetch script" { podman_stub good_env run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ FRANK_LARGE_CHECKPOINT=absent.hgn "$SERVE" [ "$status" -ne 0 ] [[ "$output" == *"checkpoint"*"missing"* ]] [[ "$output" == *"download-frank-large-halogen.sh"* ]] } @test "the serve script refuses a checkpoint of the wrong byte count" { # A truncated curl exit-0 artifact must not boot; the model card's # byte count is the contract (fleet fetch law). podman_stub good_env run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ FRANK_LARGE_CHECKPOINT_BYTES=42 "$SERVE" [ "$status" -ne 0 ] [[ "$output" == *"expected 42"* ]] } @test "the serve script refuses a symlinked tokenizer file" { # A HuggingFace cache snapshot's entries are symlinks into blobs/ and # dangle inside the container mount — even when the directory itself # is real. podman_stub good_env rm "$TEST_TMP/tokenizer/tok.json" ln -sf /nowhere/x "$TEST_TMP/tokenizer/tok.json" run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "$SERVE" [ "$status" -ne 0 ] [[ "$output" == *"missing or a symlink"* ]] } @test "the render names the container, the port map and the engine shape" { podman_stub good_env run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "$SERVE" [ "$status" -eq 0 ] # `podman run` output is redirected to the log by design (the unit # journal must not absorb engine chatter); the stub's printed argv # lands there, so the render assertions read the same log. output="$output $(cat "$TEST_TMP/serve.log")" [[ "$output" == *"--name frank-large-halogen"* ]] [[ "$output" == *"--publish 7770:8731"* ]] [[ "$output" == *"-e HALOGEN_MODEL_ID=frank-large"* ]] [[ "$output" == *"-e HALOGEN_SLOT_CTX=200700"* ]] [[ "$output" == *"-e HALOGEN_KV_SLOTS=1"* ]] [[ "$output" == *"-e HALOGEN_DRAFTER=2"* ]] [[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]] [[ "$output" == *"--device $TEST_TMP/qwen.hgn"* ]] [[ "$output" == *"$TEST_TMP/tokenizer:/tokenizer:ro"* ]] [[ "$output" == *"ghcr.io/peonist-ai/halogen@sha256:cafe"* ]] # The unit's restart policy is real because the script becomes # `podman wait`: the container's exit code is the service's exit. [[ "$output" == *"PODMAN wait frank-large-halogen"* ]] } @test "the shipped env pins the fleet's window and shape" { # 200700 is the per-conversation hard ceiling (a longer prompt is a # 400 naming the limit, never a silent truncation), and the harness # compaction math must fit prompt + budget under it: OMP compacts at # 70% = 140490, and 140490 + 57344 = 197834 <= 200700. grep -qE '^HALOGEN_SLOT_CTX=200700$' "$ENV_FILE" ctx=200700 [ $(( ctx * 70 / 100 + 57344 )) -le "$ctx" ] } @test "no HIP-sensitive bare names leak through the env file" { # HIP reads bare env vars out of a `set -a` source; a tunable named # HOST or TEMP used to send the JIT cache to path 1.0 and segfault # every load (kept from the llama.cpp era — the container gets only # the explicit -e list, but the host env still reaches podman itself). ! grep -qE '^(HOST|TEMP|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE" } @test "the deployer gates exactly the artifacts the serve script opens" { # Serve and deploy must name the SAME image digest, checkpoint and # tokenizer: the 2026-10-01 crash-loop was exactly these twins # disagreeing mid-cutover. img="$(sed -nE 's/^FRANK_LARGE_IMAGE=(.*)$/\1/p' "$ENV_FILE")" grep -qF "LLM_LARGE_IMAGE=\"\${LLM_LARGE_IMAGE:-$img}\"" "$DEPLOYER" dir="$(sed -nE 's/^FRANK_LARGE_MODEL_DIR=(.*)$/\1/p' "$ENV_FILE")" ckpt="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT=(.*)$/\1/p' "$ENV_FILE")" tokf="$(sed -nE 's/^FRANK_LARGE_TOKENIZER_FILES="(.*)"$/\1/p' "$ENV_FILE")" grep -qF "LLM_LARGE_TOK_DIR=\"\${LLM_LARGE_TOK_DIR:-$dir/tokenizer}\"" "$DEPLOYER" grep -qF "LLM_LARGE_TOK_FILES=\"\${LLM_LARGE_TOK_FILES:-$tokf}\"" "$DEPLOYER" bytes="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")" grep -qF "LLM_LARGE_CKPT_BYTES=\"\${LLM_LARGE_CKPT_BYTES:-$bytes}\"" "$DEPLOYER" } @test "the deployer points a missing artifact at the halogen fetch script" { grep -qE 'download-frank-large-halogen\.sh' "$DEPLOYER" grep -qE 'podman pull' "$DEPLOYER" } @test "the stray sweep cannot take frank-small or the deployer down" { # frank-small also listens on this host (7771, llama-server) and the # deployer's own command line contains "frank-large"; only the FULL # container name matches a stray manual halogen run and nothing else. grep -q "pkill -f 'frank-large-halogen'" "$DEPLOYER" ! grep -qE "pkill -f 'llama-server'" "$DEPLOYER" ! grep -qE "pkill -f 'frank-large'" "$DEPLOYER" } @test "the deployed unit keeps the fleet's restart and boot-persistence shape" { grep -qE '^Restart=(always|on-failure)$' "$UNIT" grep -qE '^WantedBy=' "$UNIT" grep -qE '^ExecStart=.*/serve-frank-large\.sh$' "$UNIT" grep -qE '^User=dank$' "$UNIT" grep -qE '^LimitMEMLOCK=infinity$' "$UNIT" # The service's lifetime is the container's; stopping the service # must stop the container by its real name. grep -qE '^ExecStop=.*podman stop .* frank-large-halogen$' "$UNIT" } @test "the unit runs the halogen runtime and no retired path" { grep -q 'halogen' "$UNIT" # Every retired payload/build stays on the host, so a stale path in # a unit still resolves — to a runtime nobody has measured in weeks. ! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT" ! grep -qE 'kingjones-ROCmFPX' "$UNIT" } @test "the rollback door stays qualified: the ROCmFPX build pin survives" { # Rolling back this swap means rebuilding the fork the old payload # needs; the pin (real commit, gfx1151, no webui fetch) is what makes # that door trustworthy months later. grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD" grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD" grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD" }