Files
serverAppConfigs/tests/frank_large_unit.bats
T
Frank 76aaa2543b frank-large: raise ctx to 262144 to close the harness 400-zone
The harness clamps max_tokens to min(45056, contextWindow - est - 4096),
its chars/4 est under-counts the rendered prompt by ~3 % (measured:
152,938 -> 157,388 and 158,906 -> 165,301), and compaction fires only at
85 % -- so with window == engine ceiling 200192, every request in the
est ~146k-184k zone was a guaranteed 400. HALOGEN_CTX=262144 (native) +
KV pool 524288; harness window 217088 = ceiling - 45k absorbs the
estimate error (worst-case request ~219k <= 262144). models.yml updated
out-of-repo: contextWindow 217088, maxTokens 45056.
2026-10-08 00:40:12 -07:00

373 lines
17 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bats
# Seam: frank-large serves the Quat3rnion ABLITERATED Swift1.5 finetune of
# peonist's Qwen3.8-Flash-Next v2 checkpoint (a .hgn flash-server payload)
# on the Strix Halo iGPU through the halogen-flash-server CONTAINER (0.16.2,
# digest-pinned), driven by the root unit's serve script — two 262,144-token
# slots with the MTP head always on (HALOGEN_SPEC_ADAPT=0) and the vision
# tower mounted. 2026-10-06 cutover from the halogen 27B-dense shape.
#
# This seam exists because the artifacts are a triangle that must agree on
# exact strings the engine reads: the image DIGEST, the checkpoint/n-gram/
# vision file names AND byte counts inside the /models mount, and the
# tokenizer's flat-file shape (a HF cache snapshot's symlinks dangle inside
# the container). A serve gate and a deployer gate that disagree is the
# 2026-10-01 crash-loop; both sides are pinned here, plus the one engine
# law the card is emphatic about: HALOGEN_SPEC_ADAPT=0 — the default
# adaptive policy silently disables the MTP head per request and HALVES
# decode.
#
# Everything is tested from what an operator or supervisor can observe —
# the command line podman would run, the exit code, the deployer's error
# text — never from anything inside the engine.
setup() {
ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
SERVE="$ROOT/scripts/serve-frank-large.sh"
ENV_FILE="$ROOT/scripts/frank-large.env"
UNIT="$ROOT/scripts/frank-large.service"
DEPLOYER="$ROOT/scripts/deploy-remote.sh"
BUILD="$ROOT/scripts/build-rocmfpx.sh"
TEST_TMP="$(mktemp -d)"
touch "$TEST_TMP/empty.env"
}
teardown() {
rm -rf "$TEST_TMP"
}
# podman_stub — a fake podman on PATH that prints the argv of `run` and
# `wait` and answers `images` with whatever digests the test names, so
# "what would the supervisor run" is observable output and the digest
# gate is testable without a 3.6 GB image.
podman_stub() {
mkdir -p "$TEST_TMP/bin"
cat > "$TEST_TMP/bin/podman" <<'STUB'
#!/usr/bin/env bash
case "$1" in
images) printf '%s\n' "${STUB_DIGESTS:-}" ;;
rm) exit 0 ;;
run|wait) echo "PODMAN $*"; exit "${STUB_WAIT_RC:-0}" ;;
*) exit 0 ;;
esac
STUB
chmod +x "$TEST_TMP/bin/podman"
}
# good_env — the minimum env the serve script demands, with the digest
# stub agrees on and zero-byte artifacts passing the size gates.
good_env() {
: > "$TEST_TMP/empty.env"
touch "$TEST_TMP/qwen.hgn" "$TEST_TMP/ngram.hgn" "$TEST_TMP/vision.hgn"
mkdir -p "$TEST_TMP/tokenizer" && touch "$TEST_TMP/tokenizer/tok.json"
# A host with plenty of RAM: the pin-gate preflight passes by default so
# the artifact gates below are the thing under test.
printf 'MemTotal: 127108112 kB\nMemAvailable: 100000000 kB\n' > "$TEST_TMP/meminfo"
GOOD_ENV=(
"FRANK_LARGE_ENV_FILE=$TEST_TMP/empty.env"
"FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"
"FRANK_LARGE_CONTAINER=frank-large-halogen"
"FRANK_LARGE_LEGACY_CONTAINERS=frank-large"
"FRANK_LARGE_DEV_KFD=$TEST_TMP/qwen.hgn"
"FRANK_LARGE_DEV_DRI=$TEST_TMP"
"FRANK_LARGE_LOG=$TEST_TMP/serve.log"
"FRANK_LARGE_PUBLISH=7770:8731"
"FRANK_LARGE_MODEL_DIR=$TEST_TMP"
"FRANK_LARGE_CHECKPOINT=qwen.hgn"
"FRANK_LARGE_CHECKPOINT_BYTES=0"
"FRANK_LARGE_NGRAM_TABLE=ngram.hgn"
"FRANK_LARGE_NGRAM_BYTES=0"
"FRANK_LARGE_VISION_TOWER=vision.hgn"
"FRANK_LARGE_VISION_BYTES=0"
"FRANK_LARGE_TOKENIZER_FILES=tok.json"
"HALOGEN_MODEL_ID=frank-large"
"HALOGEN_CTX=262144"
"HALOGEN_KV_POOL_POSITIONS=524288"
"HALOGEN_KV_SLOTS=2"
"HALOGEN_SPEC_ADAPT=0"
"HALOGEN_MAX_TOK=8192"
"FRANK_LARGE_MEMINFO=$TEST_TMP/meminfo"
"STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"
)
}
@test "the env file defines the flash-server deployment values" {
grep -qE '^FRANK_LARGE_IMAGE=ghcr\.io/peonist-ai/halogen-flash-server@sha256:[0-9a-f]{64}$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_CONTAINER=frank-large-halogen$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_PUBLISH=7770:8731$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_CHECKPOINT=.*\.hgn$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_CHECKPOINT_BYTES=[0-9]+$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_CHECKPOINT_SHA256=[0-9a-f]{64}$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_NGRAM_TABLE=.*\.hgn$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_NGRAM_BYTES=[0-9]+$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_NGRAM_SHA256=[0-9a-f]{64}$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_VISION_TOWER=.*\.hgn$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_VISION_BYTES=[0-9]+$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_VISION_SHA256=[0-9a-f]{64}$' "$ENV_FILE"
grep -qE '^FRANK_LARGE_TOKENIZER_FILES="tokenizer\.json( [a-z_.]+)+"$' "$ENV_FILE"
grep -qE '^HALOGEN_MODEL_ID=frank-large$' "$ENV_FILE"
grep -qE '^HALOGEN_CTX=[0-9]+$' "$ENV_FILE"
}
@test "the head stays on: adaptive spec must stay 0 and the window shape pinned" {
# The card is emphatic: the default adaptive policy turns the MTP head
# off for whole requests after low acceptance and HALVES decode. The
# two-slot law does NOT carry over from the 27B engine — on this line
# batching and speculation coexist (the card's quickstart ships 2
# slots WITH the head), so 2 is concurrent capacity, not a tax.
grep -qE '^HALOGEN_SPEC_ADAPT=0$' "$ENV_FILE"
grep -qE '^HALOGEN_KV_SLOTS=2$' "$ENV_FILE"
# The cap/timeout pair is coupled (FLAGS.md table): a request at the
# cap must not outlast the queue timeout, or everyone queued behind
# it eats a 503. The per-request default must stay under the cap.
grep -qE '^HALOGEN_MAX_TOK=[0-9]+$' "$ENV_FILE"
grep -qE '^HALOGEN_MAX_TOKENS_CAP=[0-9]+$' "$ENV_FILE"
grep -qE '^HALOGEN_QUEUE_TIMEOUT=[0-9]+$' "$ENV_FILE"
mt="$(sed -nE 's/^HALOGEN_MAX_TOK=([0-9]+)$/\1/p' "$ENV_FILE")"
cap="$(sed -nE 's/^HALOGEN_MAX_TOKENS_CAP=([0-9]+)$/\1/p' "$ENV_FILE")"
[ "$mt" -le "$cap" ]
}
@test "the serve script refuses when there is no GPU to serve on" {
podman_stub
good_env
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_LARGE_DEV_KFD="$TEST_TMP/absent-node" "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"no gfx1151 to serve on"* ]]
}
@test "the pool product and memory ledger fit unified memory with margin" {
# Pool arithmetic is the engine's documented product: the explicit
# HALOGEN_KV_POOL_POSITIONS must equal slots x ctx, and KV costs
# ~28 KiB/position (FLAGS.md). Against it: ~62 GiB resident weights,
# the arena scratch 3.8 + 8.4*(MAX_TOK/16384) GiB, the vision tower
# ~2 GiB pool charge, the 20 GiB host reserve and the host baseline
# + co-tenancy (frank-secondary, ComfyUI, Caddy/observability) ~15. The
# box has 121 GiB unified. The n-gram table rides as mmap'd page
# cache beside the checkpoint and is not a pinned charge (the card's
# own quickstart arithmetic does not bill it).
ctx="$(sed -nE 's/^HALOGEN_CTX=([0-9]+)$/\1/p' "$ENV_FILE")"
slots="$(sed -nE 's/^HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")"
pool="$(sed -nE 's/^HALOGEN_KV_POOL_POSITIONS=([0-9]+)$/\1/p' "$ENV_FILE")"
mt="$(sed -nE 's/^HALOGEN_MAX_TOK=([0-9]+)$/\1/p' "$ENV_FILE")"
reserve="$(sed -nE 's/^HALOGEN_HOST_RESERVE_GIB=([0-9]+)$/\1/p' "$ENV_FILE")"
[ "$pool" -eq $(( slots * ctx )) ]
kv_gib=$(( pool * 28 * 1024 / 1073741824 ))
arena_tenths=$(( 38 + 84 * mt / 16384 ))
total_gib=$(( 62 + kv_gib + arena_tenths / 10 + 2 + reserve + 15 ))
[ "$total_gib" -le 121 ]
}
@test "the serve script refuses when the image is not in storage" {
podman_stub
good_env
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" STUB_DIGESTS="" "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"not in podman storage"* ]]
[[ "$output" == *"podman pull"* ]]
}
@test "the serve script refuses a missing checkpoint and names the fetch script" {
podman_stub
good_env
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_LARGE_CHECKPOINT=absent.hgn "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"checkpoint"*"missing"* ]]
[[ "$output" == *"download-frank-large-flashnext.sh"* ]]
}
@test "the serve script refuses a checkpoint of the wrong byte count" {
# A truncated curl exit-0 artifact must not boot; the Hub's byte
# count is the contract (fleet fetch law).
podman_stub
good_env
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_LARGE_CHECKPOINT_BYTES=42 "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"expected 42"* ]]
}
@test "the serve script refuses a missing or wrong-size n-gram table" {
# The lookup table is REQUIRED: the engine reads it beside the
# checkpoint (the card names it explicitly), and prompt-lookup
# drafting dies without it.
podman_stub
good_env
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_LARGE_NGRAM_TABLE=absent.hgn "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"n-gram-table"*"missing"* ]]
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_LARGE_NGRAM_BYTES=42 "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"expected 42"* ]]
}
@test "the serve script refuses a missing or wrong-size vision tower" {
# Optional on the engine, ENABLED here (image input is a fleet
# contract) — a silent absence would degrade the endpoint to
# text-only with no error anywhere.
podman_stub
good_env
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_LARGE_VISION_TOWER=absent.hgn "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"vision-tower"*"missing"* ]]
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_LARGE_VISION_BYTES=42 "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"expected 42"* ]]
}
@test "the serve script refuses a symlinked tokenizer file" {
# A HuggingFace cache snapshot's entries are symlinks into blobs/ and
# dangle inside the container mount — even when the directory itself
# is real.
podman_stub
good_env
rm "$TEST_TMP/tokenizer/tok.json"
ln -sf /nowhere/x "$TEST_TMP/tokenizer/tok.json"
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"missing or a symlink"* ]]
}
@test "the serve script refuses to start when host RAM is below the pin gate" {
# The engine's pin gate: MemAvailable must clear the checkpoint plus a
# 16 GiB floor, or the unit crash-loops (each boot dying in ~5 s) until
# the RAM clears. The 2026-10-06 cutover hit this — 8 boots, 74.77 GiB
# vs 78.11 needed. RAM_WAIT=0 so the preflight refuses instead of
# waiting; the checkpoint is zero bytes (good_env) so the floor is the
# whole gate.
podman_stub
good_env
printf 'MemTotal: 127108112 kB\nMemAvailable: 10000000 kB\n' > "$TEST_TMP/meminfo-short"
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_LARGE_MEMINFO="$TEST_TMP/meminfo-short" FRANK_LARGE_RAM_WAIT=0 "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"pin gate"* ]]
[[ "$output" == *"refusing to start"* ]]
}
@test "the render names the container, the port map and the engine shape" {
podman_stub
good_env
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "$SERVE"
[ "$status" -eq 0 ]
# `podman run` output is redirected to the log by design (the unit
# journal must not absorb engine chatter); the stub's printed argv
# lands there, so the render assertions read the same log.
output="$output
$(cat "$TEST_TMP/serve.log")"
[[ "$output" == *"--name frank-large-halogen"* ]]
[[ "$output" == *"--publish 7770:8731"* ]]
[[ "$output" == *"-e HALOGEN_MODEL_ID=frank-large"* ]]
[[ "$output" == *"-e HALOGEN_CTX=262144"* ]]
[[ "$output" == *"-e HALOGEN_KV_POOL_POSITIONS=524288"* ]]
[[ "$output" == *"-e HALOGEN_KV_SLOTS=2"* ]]
[[ "$output" == *"-e HALOGEN_SPEC_ADAPT=0"* ]]
[[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]]
[[ "$output" == *"-e HALOGEN_NGRAM_TABLE=/models/ngram.hgn"* ]]
[[ "$output" == *"-e HALOGEN_VISION_TOWER=/models/vision.hgn"* ]]
[[ "$output" == *"--device $TEST_TMP/qwen.hgn"* ]]
[[ "$output" == *"$TEST_TMP/tokenizer:/tokenizer:ro"* ]]
[[ "$output" == *"ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"* ]]
# Foreground run, never -d: the engine ledger must land in the log
# file (`podman run -d` captured only the container ID and the
# runbook's ledger grep read a one-line hex file, found 2026-10-06).
! grep -qE '(^|[[:space:]])-d([[:space:]]|$)' "$TEST_TMP/serve.log"
! grep -q 'PODMAN wait' "$TEST_TMP/serve.log"
}
@test "the shipped env keeps the harness fit law inside the ceiling" {
# 262144 (the checkpoint's native window) is the per-slot hard ceiling
# (a longer prompt is a 400 naming the limit, never a silent
# truncation). The harness clamps max_tokens = min(45056, 217088 − est
# − 4096), compacts at 85 % of its declared 217088 window, and its
# chars/4 est UNDER-counts the rendered prompt by ~3 % on code/agent
# traffic (measured 4,450 tokens at est 152,938 on 2026-10-07 — with
# window == ceiling, every request over ~146k est was a guaranteed
# 400 before compaction could fire at 85 %). The 45k gap between the
# harness window and the engine ceiling must absorb that error at
# both monotone maxima of est + budget (est ≤ 217088 − 4096):
# full budget at the compaction threshold: 1.03*184524 + 45056
# clamped budget at the clamp's zero point: 1.03*212992 + 0
# and the compaction request that rides on frank-secondary (its
# 200,704-token slot) must fit: input ≈ threshold − keepRecent(20k).
grep -qE '^HALOGEN_CTX=262144$' "$ENV_FILE"
ctx=262144
hw=217088; mt=45056
thr=$(( hw * 85 / 100 ))
[ $(( thr * 103 / 100 + mt )) -le "$ctx" ]
zero=$(( hw - 4096 ))
[ $(( zero * 103 / 100 )) -le "$ctx" ]
[ $(( (thr - 20000) * 103 / 100 )) -le 200704 ]
}
@test "no HIP-sensitive bare names leak through the env file" {
# HIP reads bare env vars out of a `set -a` source; a tunable named
# HOST or TEMP used to send the JIT cache to path 1.0 and segfault
# every load (kept from the llama.cpp era — the container gets only
# the explicit -e list, but the host env still reaches podman itself).
! grep -qE '^(HOST|TEMP|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
}
@test "the deployer gates exactly the artifacts the serve script opens" {
# Serve and deploy must name the SAME image digest, checkpoint and
# tokenizer: the 2026-10-01 crash-loop was exactly these twins
# disagreeing mid-cutover.
img="$(sed -nE 's/^FRANK_LARGE_IMAGE=(.*)$/\1/p' "$ENV_FILE")"
grep -qF "LLM_LARGE_IMAGE=\"\${LLM_LARGE_IMAGE:-$img}\"" "$DEPLOYER"
dir="$(sed -nE 's/^FRANK_LARGE_MODEL_DIR=(.*)$/\1/p' "$ENV_FILE")"
ckpt="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT=(.*)$/\1/p' "$ENV_FILE")"
tokf="$(sed -nE 's/^FRANK_LARGE_TOKENIZER_FILES=\"(.*)\"$/\1/p' "$ENV_FILE")"
grep -qF "LLM_LARGE_TOK_DIR=\"\${LLM_LARGE_TOK_DIR:-$dir/tokenizer}\"" "$DEPLOYER"
grep -qF "LLM_LARGE_TOK_FILES=\"\${LLM_LARGE_TOK_FILES:-$tokf}\"" "$DEPLOYER"
bytes="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")"
grep -qF "LLM_LARGE_CKPT_BYTES=\"\${LLM_LARGE_CKPT_BYTES:-$bytes}\"" "$DEPLOYER"
}
@test "the deployer points a missing artifact at the flashnext fetch script" {
grep -qE 'download-frank-large-flashnext\.sh' "$DEPLOYER"
grep -qE 'podman pull' "$DEPLOYER"
}
@test "the stray sweep cannot take frank-secondary or the deployer down" {
# frank-secondary also listens on this host (7771, llama-server) and the
# deployer's own command line contains "frank-large"; only the FULL
# container name matches a stray manual flash-server run and nothing
# else.
grep -q "pkill -f 'frank-large-halogen'" "$DEPLOYER"
! grep -qE "pkill -f 'llama-server'" "$DEPLOYER"
! grep -qE "pkill -f 'frank-large'" "$DEPLOYER"
}
@test "the deployed unit keeps the fleet's restart and boot-persistence shape" {
grep -qE '^Restart=(always|on-failure)$' "$UNIT"
grep -qE '^WantedBy=' "$UNIT"
grep -qE '^ExecStart=.*/serve-frank-large\.sh$' "$UNIT"
grep -qE '^User=dank$' "$UNIT"
grep -qE '^LimitMEMLOCK=infinity$' "$UNIT"
# The service's lifetime is the container's; stopping the service
# must stop the container by its real name.
grep -qE '^ExecStop=.*podman stop .* frank-large-halogen$' "$UNIT"
}
@test "the unit runs the flash-server runtime and no retired path" {
grep -q 'halogen-flash-server' "$UNIT"
# Every retired payload/build stays on the host, so a stale path in
# a unit still resolves — to a runtime nobody has measured in weeks.
! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT"
! grep -qE 'kingjones-ROCmFPX' "$UNIT"
}
@test "the rollback door stays qualified: the ROCmFPX build pin survives" {
# Rolling back to the pre-halogen shape means rebuilding the fork the
# old payload needs; the pin (real commit, gfx1151, no webui fetch)
# is what makes that door trustworthy months later.
grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD"
grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
}