Frank
fcf94562d1
The armed boot reached the engine and died in its own preflight:
halogen npu: /dev/accel/accel0 is here, but this process cannot open it
(crw-rw---- nobody:nogroup).
The node is root:render 0660 and a rootless container maps exactly one uid/gid,
so `render` is unmapped inside it and `--group-add keep-groups` — the vendor's
advice, and the ComfyUI pattern for /dev/kfd — cannot grant a group the
namespace does not map. Measured 2026-10-19 from a `dank` shell already in
`render`: without the flag denied, with the flag denied, with the udev mode
0666 the open succeeds (OPEN_OK). Same wall 99-comfyui-gpu-access.rules already
solved for the GPU compute nodes, so the accel node gets its counterpart:
dankclaw/udev/98-halogen-npu-access.rules — KERNEL=="accel[0-9]*",
SUBSYSTEM=="accel", MODE="0666"
The deployer installs the rule, reloads rules, re-triggers the node and only
then checks the mode; the serve script refuses any node that is not
other-writable, naming the rule, so this cannot return as a crash loop (189
restarts while chat was down is the failure mode being closed).
Both checks read the permission bit directly: `[ -o FILE ]` is not
"other-writable" — it means "exists" and `-o` doubles as the OR operator inside
test, which is why the first cut of the gate refused everything.
Dragoon seam: 33 -> 35 tests. The fixture node is 0666 like the installed one,
a 0660 node is refused with the rule named and nothing rendered, and the rule
file's shape plus the deployer's install/trigger are asserted.
664 lines
31 KiB
Bash
Executable File
664 lines
31 KiB
Bash
Executable File
#!/usr/bin/env bats
|
||
# Seam: frank-dragoon serves Quat3rnion's NATIVE v2 conversion (2026-10-11)
|
||
# of the abliterated Qwen3.8-Flash-Next (a .hgn flash-server payload) on the
|
||
# Strix Halo iGPU through the halogen-flash-server CONTAINER (0.16.2,
|
||
# digest-pinned), driven by the root unit's serve script — 2 slots over ONE
|
||
# 200,704-position KV pool with the MTP head always on (HALOGEN_SPEC_ADAPT=0)
|
||
# and the vision tower mounted. 2026-10-17 cutover from the Swift1.5-finetune
|
||
# shape, which itself cut over the halogen 27B-dense shape on 2026-10-06.
|
||
#
|
||
# This seam exists because the artifacts are a triangle that must agree on
|
||
# exact strings the engine reads: the image DIGEST, the checkpoint/n-gram/
|
||
# vision file names AND byte counts inside the /models mount, and the
|
||
# tokenizer's flat-file shape (a HF cache snapshot's symlinks dangle inside
|
||
# the container). A serve gate and a deployer gate that disagree is the
|
||
# 2026-10-01 crash-loop; both sides are pinned here, plus the one engine
|
||
# law the card is emphatic about: HALOGEN_SPEC_ADAPT=0 — the default
|
||
# adaptive policy silently disables the MTP head per request and HALVES
|
||
# decode.
|
||
#
|
||
# Everything is tested from what an operator or supervisor can observe —
|
||
# the command line podman would run, the exit code, the deployer's error
|
||
# text — never from anything inside the engine.
|
||
|
||
setup() {
|
||
ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
||
SERVE="$ROOT/scripts/serve-frank-dragoon.sh"
|
||
ENV_FILE="$ROOT/scripts/frank-dragoon.env"
|
||
UNIT="$ROOT/scripts/frank-dragoon.service"
|
||
DEPLOYER="$ROOT/scripts/deploy-remote.sh"
|
||
HOST_UNIT="$ROOT/dankclaw/host/halogen-fabric-clock"
|
||
HOST_SERVICE="$ROOT/dankclaw/host/halogen-fabric-clock.service"
|
||
NPU_FETCH="$ROOT/scripts/download-frank-dragoon-npu.sh"
|
||
NPU_RULE="$ROOT/dankclaw/udev/98-halogen-npu-access.rules"
|
||
BUILD="$ROOT/scripts/build-rocmfpx.sh"
|
||
TEST_TMP="$(mktemp -d)"
|
||
touch "$TEST_TMP/empty.env"
|
||
}
|
||
|
||
teardown() {
|
||
rm -rf "$TEST_TMP"
|
||
}
|
||
|
||
# podman_stub — a fake podman on PATH that prints the argv of `run` and
|
||
# `wait` and answers `images` with whatever digests the test names, so
|
||
# "what would the supervisor run" is observable output and the digest
|
||
# gate is testable without a 3.6 GB image.
|
||
podman_stub() {
|
||
mkdir -p "$TEST_TMP/bin"
|
||
cat > "$TEST_TMP/bin/podman" <<'STUB'
|
||
#!/usr/bin/env bash
|
||
case "$1" in
|
||
images) printf '%s\n' "${STUB_DIGESTS:-}" ;;
|
||
rm) exit 0 ;;
|
||
run|wait) echo "PODMAN $*"; exit "${STUB_WAIT_RC:-0}" ;;
|
||
*) exit 0 ;;
|
||
esac
|
||
STUB
|
||
chmod +x "$TEST_TMP/bin/podman"
|
||
}
|
||
|
||
# good_env — the minimum env the serve script demands, with the digest
|
||
# stub agrees on and zero-byte artifacts passing the size gates.
|
||
good_env() {
|
||
: > "$TEST_TMP/empty.env"
|
||
touch "$TEST_TMP/qwen.hgn" "$TEST_TMP/ngram.hgn" "$TEST_TMP/vision.hgn"
|
||
mkdir -p "$TEST_TMP/tokenizer" && touch "$TEST_TMP/tokenizer/tok.json"
|
||
# A host with plenty of RAM: the pin-gate preflight passes by default so
|
||
# the artifact gates below are the thing under test.
|
||
printf 'MemTotal: 127108112 kB\nMemAvailable: 100000000 kB\n' > "$TEST_TMP/meminfo"
|
||
GOOD_ENV=(
|
||
"FRANK_DRAGOON_ENV_FILE=$TEST_TMP/empty.env"
|
||
"FRANK_DRAGOON_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"
|
||
"FRANK_DRAGOON_CONTAINER=frank-dragoon-halogen"
|
||
"FRANK_DRAGOON_LEGACY_CONTAINERS=frank-dragoon"
|
||
"FRANK_DRAGOON_DEV_KFD=$TEST_TMP/qwen.hgn"
|
||
"FRANK_DRAGOON_DEV_DRI=$TEST_TMP"
|
||
"FRANK_DRAGOON_LOG=$TEST_TMP/serve.log"
|
||
"FRANK_DRAGOON_PUBLISH=7770:8731"
|
||
"FRANK_DRAGOON_MODEL_DIR=$TEST_TMP"
|
||
"FRANK_DRAGOON_CHECKPOINT=qwen.hgn"
|
||
"FRANK_DRAGOON_CHECKPOINT_BYTES=0"
|
||
"FRANK_DRAGOON_NGRAM_TABLE=ngram.hgn"
|
||
"FRANK_DRAGOON_NGRAM_BYTES=0"
|
||
"FRANK_DRAGOON_VISION_TOWER=vision.hgn"
|
||
"FRANK_DRAGOON_VISION_BYTES=0"
|
||
"FRANK_DRAGOON_TOKENIZER_FILES=tok.json"
|
||
"HALOGEN_MODEL_ID=frank-dragoon"
|
||
"HALOGEN_CTX=200704"
|
||
"HALOGEN_KV_POOL_POSITIONS=401408"
|
||
"HALOGEN_KV_SLOTS=2"
|
||
"HALOGEN_SPEC_ADAPT=0"
|
||
"HALOGEN_MAX_TOK=8192"
|
||
"FRANK_DRAGOON_MEMINFO=$TEST_TMP/meminfo"
|
||
"STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"
|
||
)
|
||
}
|
||
|
||
@test "the env file defines the flash-server deployment values" {
|
||
grep -qE '^FRANK_DRAGOON_IMAGE=ghcr\.io/peonist-ai/halogen-flash-server@sha256:[0-9a-f]{64}$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_CONTAINER=frank-dragoon-halogen$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_PUBLISH=7770:8731$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_CHECKPOINT=.*\.hgn$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_CHECKPOINT_BYTES=[0-9]+$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_CHECKPOINT_SHA256=[0-9a-f]{64}$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_NGRAM_TABLE=.*\.hgn$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_NGRAM_BYTES=[0-9]+$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_NGRAM_SHA256=[0-9a-f]{64}$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_VISION_TOWER=.*\.hgn$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_VISION_BYTES=[0-9]+$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_VISION_SHA256=[0-9a-f]{64}$' "$ENV_FILE"
|
||
grep -qE '^FRANK_DRAGOON_TOKENIZER_FILES="tokenizer\.json( [a-z_.]+)+"$' "$ENV_FILE"
|
||
grep -qE '^HALOGEN_MODEL_ID=frank-dragoon$' "$ENV_FILE"
|
||
grep -qE '^HALOGEN_CTX=[0-9]+$' "$ENV_FILE"
|
||
}
|
||
|
||
@test "the head stays on: adaptive spec must stay 0 and the window shape pinned" {
|
||
# The card is emphatic: the default adaptive policy turns the MTP head
|
||
# off for whole requests after low acceptance and HALVES decode. The
|
||
# two-slot law does NOT carry over from the 27B engine — on this line
|
||
# batching and speculation coexist (the card's quickstart ships 2
|
||
# slots WITH the head), so 2 is concurrent capacity, not a tax.
|
||
grep -qE '^HALOGEN_SPEC_ADAPT=0$' "$ENV_FILE"
|
||
grep -qE '^HALOGEN_KV_SLOTS=2$' "$ENV_FILE"
|
||
# The cap/timeout pair is coupled (FLAGS.md table): a request at the
|
||
# cap must not outlast the queue timeout, or everyone queued behind
|
||
# it eats a 503. The per-request default must stay under the cap.
|
||
grep -qE '^HALOGEN_MAX_TOK=[0-9]+$' "$ENV_FILE"
|
||
grep -qE '^HALOGEN_MAX_TOKENS_CAP=[0-9]+$' "$ENV_FILE"
|
||
grep -qE '^HALOGEN_QUEUE_TIMEOUT=[0-9]+$' "$ENV_FILE"
|
||
mt="$(sed -nE 's/^HALOGEN_MAX_TOK=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
cap="$(sed -nE 's/^HALOGEN_MAX_TOKENS_CAP=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
[ "$mt" -le "$cap" ]
|
||
}
|
||
|
||
@test "the serve script refuses when there is no GPU to serve on" {
|
||
podman_stub
|
||
good_env
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||
FRANK_DRAGOON_DEV_KFD="$TEST_TMP/absent-node" "$SERVE"
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"no gfx1151 to serve on"* ]]
|
||
}
|
||
|
||
@test "the pool and memory ledger fit unified memory with margin" {
|
||
# Pool arithmetic is the engine's documented shape: the explicit
|
||
# HALOGEN_KV_POOL_POSITIONS must back at least one full window, and KV
|
||
# costs
|
||
# ~28 KiB/position (FLAGS.md). Against it: ~62 GiB resident weights,
|
||
# the arena scratch 3.8 + 8.4*(MAX_TOK/16384) GiB, the vision tower
|
||
# ~2 GiB pool charge, the 20 GiB host reserve and the host baseline
|
||
# + co-tenancy (frank-zealot, ComfyUI, Caddy/observability) ~15. The
|
||
# box has 121 GiB unified. The n-gram table rides as mmap'd page
|
||
# cache beside the checkpoint and is not a pinned charge (the card's
|
||
# own quickstart arithmetic does not bill it).
|
||
ctx="$(sed -nE 's/^HALOGEN_CTX=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
slots="$(sed -nE 's/^HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
pool="$(sed -nE 's/^HALOGEN_KV_POOL_POSITIONS=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
mt="$(sed -nE 's/^HALOGEN_MAX_TOK=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
reserve="$(sed -nE 's/^HALOGEN_HOST_RESERVE_GIB=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
# The pool is ONE budget behind the slots (boot ledger form: "2 slots over
|
||
# ONE <pool>-position KV pool, each request up to HALOGEN_CTX"), NOT
|
||
# a per-slot private pool. Since 2026-10-18 it is sized pool = slots * ctx
|
||
# (401,408 = 2 x 200,704), so two MAXIMUM-length conversations can be
|
||
# resident at once — frank-zealot parity on total in-flight context. What
|
||
# still binds: the pool must back ONE full-length request (below that the
|
||
# advertised per-request ceiling would be unreachable, hence pool >= ctx),
|
||
# it must not pretend to be more than slots x ctx (pool <= slots * ctx), and
|
||
# the ledger sum below must stay inside unified memory.
|
||
[ "$pool" -ge "$ctx" ]
|
||
[ "$pool" -le $(( slots * ctx )) ]
|
||
kv_gib=$(( pool * 28 * 1024 / 1073741824 ))
|
||
arena_tenths=$(( 38 + 84 * mt / 16384 ))
|
||
total_gib=$(( 62 + kv_gib + arena_tenths / 10 + 2 + reserve + 15 ))
|
||
# The NPU lane bills extra pinned host RAM per named model (the vendor's
|
||
# budget note: ~2 GiB for an embedding/decider/rerank-sized program, more
|
||
# for FLUX-class ones), and the lane is named in the shipped env, so the
|
||
# ledger must still close on a 121 GiB unified-memory host.
|
||
npu_models=0
|
||
npu="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE")"
|
||
[ -n "$npu" ] && npu_models="$(echo "${npu//,/ }" | wc -w)"
|
||
total_gib=$(( total_gib + 2 * npu_models ))
|
||
[ "$total_gib" -le 121 ]
|
||
}
|
||
|
||
@test "the serve script refuses when the image is not in storage" {
|
||
podman_stub
|
||
good_env
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" STUB_DIGESTS="" "$SERVE"
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"not in podman storage"* ]]
|
||
[[ "$output" == *"podman pull"* ]]
|
||
}
|
||
|
||
@test "the serve script refuses a missing checkpoint and names the fetch script" {
|
||
podman_stub
|
||
good_env
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||
FRANK_DRAGOON_CHECKPOINT=absent.hgn "$SERVE"
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"checkpoint"*"missing"* ]]
|
||
[[ "$output" == *"download-frank-dragoon-v2abl.sh"* ]]
|
||
}
|
||
|
||
@test "the serve script refuses a checkpoint of the wrong byte count" {
|
||
# A truncated curl exit-0 artifact must not boot; the Hub's byte
|
||
# count is the contract (fleet fetch law).
|
||
podman_stub
|
||
good_env
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||
FRANK_DRAGOON_CHECKPOINT_BYTES=42 "$SERVE"
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"expected 42"* ]]
|
||
}
|
||
|
||
@test "the serve script refuses a missing or wrong-size n-gram table" {
|
||
# The lookup table is REQUIRED: the engine reads it beside the
|
||
# checkpoint (the card names it explicitly), and prompt-lookup
|
||
# drafting dies without it.
|
||
podman_stub
|
||
good_env
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||
FRANK_DRAGOON_NGRAM_TABLE=absent.hgn "$SERVE"
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"n-gram-table"*"missing"* ]]
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||
FRANK_DRAGOON_NGRAM_BYTES=42 "$SERVE"
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"expected 42"* ]]
|
||
}
|
||
|
||
@test "the serve script refuses a missing or wrong-size vision tower" {
|
||
# Optional on the engine, ENABLED here (image input is a fleet
|
||
# contract) — a silent absence would degrade the endpoint to
|
||
# text-only with no error anywhere.
|
||
podman_stub
|
||
good_env
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||
FRANK_DRAGOON_VISION_TOWER=absent.hgn "$SERVE"
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"vision-tower"*"missing"* ]]
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||
FRANK_DRAGOON_VISION_BYTES=42 "$SERVE"
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"expected 42"* ]]
|
||
}
|
||
|
||
@test "the serve script refuses a symlinked tokenizer file" {
|
||
# A HuggingFace cache snapshot's entries are symlinks into blobs/ and
|
||
# dangle inside the container mount — even when the directory itself
|
||
# is real.
|
||
podman_stub
|
||
good_env
|
||
rm "$TEST_TMP/tokenizer/tok.json"
|
||
ln -sf /nowhere/x "$TEST_TMP/tokenizer/tok.json"
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "$SERVE"
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"missing or a symlink"* ]]
|
||
}
|
||
|
||
@test "the serve script refuses to start when host RAM is below the pin gate" {
|
||
# The engine's pin gate: MemAvailable must clear the checkpoint plus a
|
||
# 16 GiB floor, or the unit crash-loops (each boot dying in ~5 s) until
|
||
# the RAM clears. The 2026-10-06 cutover hit this — 8 boots, 74.77 GiB
|
||
# vs 78.11 needed. RAM_WAIT=0 so the preflight refuses instead of
|
||
# waiting; the checkpoint is zero bytes (good_env) so the floor is the
|
||
# whole gate.
|
||
podman_stub
|
||
good_env
|
||
printf 'MemTotal: 127108112 kB\nMemAvailable: 10000000 kB\n' > "$TEST_TMP/meminfo-short"
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||
FRANK_DRAGOON_MEMINFO="$TEST_TMP/meminfo-short" FRANK_DRAGOON_RAM_WAIT=0 "$SERVE"
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"pin gate"* ]]
|
||
[[ "$output" == *"refusing to start"* ]]
|
||
}
|
||
|
||
@test "the render names the container, the port map and the engine shape" {
|
||
podman_stub
|
||
good_env
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "$SERVE"
|
||
[ "$status" -eq 0 ]
|
||
# `podman run` output is redirected to the log by design (the unit
|
||
# journal must not absorb engine chatter); the stub's printed argv
|
||
# lands there, so the render assertions read the same log.
|
||
output="$output
|
||
$(cat "$TEST_TMP/serve.log")"
|
||
[[ "$output" == *"--name frank-dragoon-halogen"* ]]
|
||
[[ "$output" == *"--publish 7770:8731"* ]]
|
||
[[ "$output" == *"-e HALOGEN_MODEL_ID=frank-dragoon"* ]]
|
||
[[ "$output" == *"-e HALOGEN_CTX=200704"* ]]
|
||
[[ "$output" == *"-e HALOGEN_KV_POOL_POSITIONS=401408"* ]]
|
||
[[ "$output" == *"-e HALOGEN_KV_SLOTS=2"* ]]
|
||
[[ "$output" == *"-e HALOGEN_SPEC_ADAPT=0"* ]]
|
||
[[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]]
|
||
[[ "$output" == *"-e HALOGEN_NGRAM_TABLE=/models/ngram.hgn"* ]]
|
||
[[ "$output" == *"-e HALOGEN_VISION_TOWER=/models/vision.hgn"* ]]
|
||
[[ "$output" == *"--device $TEST_TMP/qwen.hgn"* ]]
|
||
[[ "$output" == *"$TEST_TMP/tokenizer:/tokenizer:ro"* ]]
|
||
[[ "$output" == *"ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"* ]]
|
||
# Foreground run, never -d: the engine ledger must land in the log
|
||
# file (`podman run -d` captured only the container ID and the
|
||
# runbook's ledger grep read a one-line hex file, found 2026-10-06).
|
||
! grep -qE '(^|[[:space:]])-d([[:space:]]|$)' "$TEST_TMP/serve.log"
|
||
! grep -q 'PODMAN wait' "$TEST_TMP/serve.log"
|
||
}
|
||
|
||
@test "the shipped env keeps the harness fit law inside the ceiling" {
|
||
# HALOGEN_CTX is the hard per-request ceiling (a longer prompt is a 400
|
||
# naming the limit, never a silent truncation). The law is INVARIANT to
|
||
# the value: the harness window must sit ~45k UNDER it, because the
|
||
# harness clamps max_tokens = min(45056, window − est − 4096), compacts
|
||
# at 85 % of its declared window, and its chars/4 est UNDER-counts the
|
||
# rendered prompt by ~3 % on code/agent traffic (measured 4,450 tokens at
|
||
# est 152,938 on 2026-10-07 — with window == ceiling, every request over
|
||
# ~65 % of the window was a guaranteed 400 before compaction could fire).
|
||
# The shipped pair is ctx 200704 / window 155648 (lowered from
|
||
# 262144/217088 on 2026-10-17); the gap must absorb the error at BOTH
|
||
# monotone maxima of est + budget (est ≤ window − 4096):
|
||
# full budget at the compaction threshold: 1.03*132300 + 45056
|
||
# clamped budget at the clamp's zero point: 1.03*151552 + 0
|
||
# and the compaction request that rides on frank-zealot (its
|
||
# 200,704-token slot) must fit: input ≈ threshold − keepRecent(20k).
|
||
grep -qE '^HALOGEN_CTX=200704$' "$ENV_FILE"
|
||
ctx=200704
|
||
hw=155648; mt=45056
|
||
thr=$(( hw * 85 / 100 ))
|
||
[ $(( thr * 103 / 100 + mt )) -le "$ctx" ]
|
||
zero=$(( hw - 4096 ))
|
||
[ $(( zero * 103 / 100 )) -le "$ctx" ]
|
||
[ $(( (thr - 20000) * 103 / 100 )) -le 200704 ]
|
||
}
|
||
|
||
@test "no HIP-sensitive bare names leak through the env file" {
|
||
# HIP reads bare env vars out of a `set -a` source; a tunable named
|
||
# HOST or TEMP used to send the JIT cache to path 1.0 and segfault
|
||
# every load (kept from the llama.cpp era — the container gets only
|
||
# the explicit -e list, but the host env still reaches podman itself).
|
||
! grep -qE '^(HOST|TEMP|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
|
||
}
|
||
|
||
@test "the deployer gates exactly the artifacts the serve script opens" {
|
||
# Serve and deploy must name the SAME image digest, checkpoint and
|
||
# tokenizer: the 2026-10-01 crash-loop was exactly these twins
|
||
# disagreeing mid-cutover.
|
||
img="$(sed -nE 's/^FRANK_DRAGOON_IMAGE=(.*)$/\1/p' "$ENV_FILE")"
|
||
grep -qF "LLM_DRAGOON_IMAGE=\"\${LLM_DRAGOON_IMAGE:-$img}\"" "$DEPLOYER"
|
||
dir="$(sed -nE 's/^FRANK_DRAGOON_MODEL_DIR=(.*)$/\1/p' "$ENV_FILE")"
|
||
ckpt="$(sed -nE 's/^FRANK_DRAGOON_CHECKPOINT=(.*)$/\1/p' "$ENV_FILE")"
|
||
tokf="$(sed -nE 's/^FRANK_DRAGOON_TOKENIZER_FILES=\"(.*)\"$/\1/p' "$ENV_FILE")"
|
||
grep -qF "LLM_DRAGOON_TOK_DIR=\"\${LLM_DRAGOON_TOK_DIR:-$dir/tokenizer}\"" "$DEPLOYER"
|
||
grep -qF "LLM_DRAGOON_TOK_FILES=\"\${LLM_DRAGOON_TOK_FILES:-$tokf}\"" "$DEPLOYER"
|
||
bytes="$(sed -nE 's/^FRANK_DRAGOON_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
grep -qF "LLM_DRAGOON_CKPT_BYTES=\"\${LLM_DRAGOON_CKPT_BYTES:-$bytes}\"" "$DEPLOYER"
|
||
# The NPU twins: the lane switches on the same value on both sides. A blank
|
||
# env line with a named deployer default (or the reverse) is the 2026-10-01
|
||
# disagreement again — one side gates a lane the other does not build.
|
||
npu_dir="$(sed -nE 's/^FRANK_DRAGOON_NPU_DIR=(.*)$/\1/p' "$ENV_FILE")"
|
||
npu_models="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE")"
|
||
grep -qF "LLM_DRAGOON_NPU_DIR=\"\${LLM_DRAGOON_NPU_DIR:-$npu_dir}\"" "$DEPLOYER"
|
||
grep -qF "LLM_DRAGOON_NPU_MODELS=\"\${LLM_DRAGOON_NPU_MODELS:-$npu_models}\"" "$DEPLOYER"
|
||
# The deployer's own copy of the payload dir is what the lane's containment
|
||
# gate compares against — a stale twin would wave a bad staging path through.
|
||
grep -qF "LLM_DRAGOON_MODEL_DIR=\"\${LLM_DRAGOON_MODEL_DIR:-$dir}\"" "$DEPLOYER"
|
||
net="$(sed -nE 's/^FRANK_DRAGOON_NETWORK=(.*)$/\1/p' "$ENV_FILE")"
|
||
grep -qF "LLM_DRAGOON_NETWORK=\"\${LLM_DRAGOON_NETWORK:-$net}\"" "$DEPLOYER"
|
||
}
|
||
|
||
@test "the deployer points a missing artifact at the v2abl fetch script" {
|
||
grep -qE 'download-frank-dragoon-v2abl\.sh' "$DEPLOYER"
|
||
grep -qE 'podman pull' "$DEPLOYER"
|
||
}
|
||
|
||
@test "the stray sweep cannot take frank-zealot or the deployer down" {
|
||
# frank-zealot also listens on this host (7771, llama-server) and the
|
||
# deployer's own command line contains "frank-dragoon"; only the FULL
|
||
# container name matches a stray manual flash-server run and nothing
|
||
# else.
|
||
grep -q "pkill -f 'frank-dragoon-halogen'" "$DEPLOYER"
|
||
! grep -qE "pkill -f 'llama-server'" "$DEPLOYER"
|
||
! grep -qE "pkill -f 'frank-dragoon'" "$DEPLOYER"
|
||
}
|
||
|
||
@test "the deployed unit keeps the fleet's restart and boot-persistence shape" {
|
||
grep -qE '^Restart=(always|on-failure)$' "$UNIT"
|
||
grep -qE '^WantedBy=' "$UNIT"
|
||
grep -qE '^ExecStart=.*/serve-frank-dragoon\.sh$' "$UNIT"
|
||
grep -qE '^User=dank$' "$UNIT"
|
||
grep -qE '^LimitMEMLOCK=infinity$' "$UNIT"
|
||
# The service's lifetime is the container's; stopping the service
|
||
# must stop the container by its real name.
|
||
grep -qE '^ExecStop=.*podman stop .* frank-dragoon-halogen$' "$UNIT"
|
||
}
|
||
|
||
@test "the unit runs the flash-server runtime and no retired path" {
|
||
grep -q 'halogen-flash-server' "$UNIT"
|
||
# Every retired payload/build stays on the host, so a stale path in
|
||
# a unit still resolves — to a runtime nobody has measured in weeks.
|
||
! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT"
|
||
! grep -qE 'kingjones-ROCmFPX' "$UNIT"
|
||
}
|
||
|
||
@test "the rollback door stays qualified: the ROCmFPX build pin survives" {
|
||
# Rolling back to the pre-halogen shape means rebuilding the fork the
|
||
# old payload needs; the pin (real commit, gfx1151, no webui fetch)
|
||
# is what makes that door trustworthy months later.
|
||
grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD"
|
||
grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
|
||
grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
|
||
}
|
||
|
||
# --- the NPU lane (XDNA2) ----------------------------------------------------
|
||
# Opt-in lane added 2026-10-19: halogen runs the vendor's precompiled AIE
|
||
# programs for the models HALOGEN_NPU_MODELS names on the NPU inside the SAME
|
||
# container (they answer /v1/embeddings on 7770). Two things make this seam
|
||
# worth pinning: the lane must be genuinely OFF when the variable is blank (the
|
||
# rollback is one line, so the render must show nothing lane-specific), and
|
||
# every host precondition the engine needs — the accel node, the distro XRT's
|
||
# three libraries, the staged model files, the held GPU fabric clock — must be
|
||
# refused BY NAME instead of surfacing as an engine crash-loop.
|
||
|
||
npu_fixture() {
|
||
# 0666 like the installed node: a rootless container's namespace cannot open
|
||
# the stock root:render 0660 mode, and the serve gate checks exactly that.
|
||
touch "$TEST_TMP/accel0"
|
||
chmod 666 "$TEST_TMP/accel0"
|
||
mkdir -p "$TEST_TMP/xrt"
|
||
for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||
touch "$TEST_TMP/xrt/$l"
|
||
done
|
||
mkdir -p "$TEST_TMP/npu/qwen3-embedding-0.6b/devices"
|
||
mkdir -p "$TEST_TMP/sys/class/drm/card1/device"
|
||
printf '0: 400Mhz \n7: 2000Mhz *\n' > "$TEST_TMP/sys/class/drm/card1/device/pp_dpm_fclk"
|
||
printf 'manual\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level"
|
||
NPU_ENV=(
|
||
"HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"
|
||
"FRANK_DRAGOON_NPU_DIR=$TEST_TMP/npu"
|
||
"FRANK_DRAGOON_DEV_ACCEL=$TEST_TMP/accel0"
|
||
"FRANK_DRAGOON_XRT_DIR=$TEST_TMP/xrt"
|
||
"FRANK_DRAGOON_SYSFS=$TEST_TMP/sys"
|
||
)
|
||
}
|
||
|
||
# run_with_log <args...> — run the serve script and fold its own stderr plus
|
||
# the podman stub's argv (which the script redirects into the log) into
|
||
# $output, so both refusal text and render assertions read one variable.
|
||
run_with_log() {
|
||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "${NPU_ENV[@]}" "$@" "$SERVE"
|
||
if [ -f "$TEST_TMP/serve.log" ]; then
|
||
output="$output
|
||
$(cat "$TEST_TMP/serve.log")"
|
||
fi
|
||
}
|
||
|
||
@test "the lane is off when HALOGEN_NPU_MODELS is blank: no device, no mounts, no engine env" {
|
||
# The rollback is blanking one env line, so "off" has to be observable in
|
||
# the command podman would run — not just implied by an absent log line.
|
||
podman_stub
|
||
good_env
|
||
run_with_log HALOGEN_NPU_MODELS=
|
||
[ "$status" -eq 0 ]
|
||
[[ "$output" == *"--publish 7770:8731"* ]]
|
||
! grep -q '/models/npu' "$TEST_TMP/serve.log"
|
||
! grep -q 'HALOGEN_NPU_MODELS' "$TEST_TMP/serve.log"
|
||
! grep -q 'xilinx' "$TEST_TMP/serve.log"
|
||
! grep -q 'accel0' "$TEST_TMP/serve.log"
|
||
}
|
||
|
||
@test "the lane arms the accel node, the staged model dir and each XRT library twice" {
|
||
# Each library mounts at the image's XRT path AND at its own path: the
|
||
# distro libraries find each other where they were installed, and an
|
||
# /opt/xilinx/xrt of symlinks into the system dir dangles inside the
|
||
# container (the vendor calls that out by name).
|
||
podman_stub
|
||
good_env
|
||
npu_fixture
|
||
run_with_log
|
||
[ "$status" -eq 0 ]
|
||
[[ "$output" == *"--device $TEST_TMP/accel0"* ]]
|
||
# The lane's files ride the payload mount: /models is a read-only bind and
|
||
# runc cannot mkdir a nested /models/npu mountpoint through it (measured
|
||
# 2026-10-19), so the staging dir lives INSIDE the payload dir and gets no
|
||
# -v of its own.
|
||
[[ "$output" == *"-v $TEST_TMP:/models:ro"* ]]
|
||
[[ "$output" != *":/models/npu"* ]]
|
||
[[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]]
|
||
for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||
[[ "$output" == *"-v $TEST_TMP/xrt/$l:/opt/xilinx/xrt/lib/$l:ro"* ]]
|
||
[[ "$output" == *"-v $TEST_TMP/xrt/$l:$TEST_TMP/xrt/$l:ro"* ]]
|
||
done
|
||
# The Flash payload still rides the same container: the lane is additive.
|
||
[[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]]
|
||
}
|
||
|
||
@test "the lane refuses a staging dir outside the payload dir and says why" {
|
||
# A staging dir outside the payload dir needs a nested bind the read-only
|
||
# /models mount cannot host: runc dies at container init, which is AFTER the
|
||
# deployer has stopped the old unit — so the shape is refused here, before
|
||
# anything is rendered.
|
||
podman_stub
|
||
good_env
|
||
npu_fixture
|
||
run_with_log FRANK_DRAGOON_NPU_DIR=/elsewhere/npu
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"/elsewhere/npu"*"outside the payload dir"* ]]
|
||
[[ "$output" != *"--device"* ]]
|
||
}
|
||
|
||
@test "the lane refuses a model that is not staged and names the fetch script" {
|
||
# The engine cannot fetch these at boot here (/models is read-only and
|
||
# HALOGEN_DOWNLOAD stays unset), so an un-staged name must stop the unit
|
||
# before podman runs, with the command that fixes it.
|
||
podman_stub
|
||
good_env
|
||
npu_fixture
|
||
rm -rf "$TEST_TMP/npu/qwen3-embedding-0.6b"
|
||
run_with_log
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"qwen3-embedding-0.6b"*"not staged"* ]]
|
||
[[ "$output" == *"download-frank-dragoon-npu.sh"* ]]
|
||
}
|
||
|
||
@test "the lane refuses without the accel node and says what to check" {
|
||
podman_stub
|
||
good_env
|
||
npu_fixture
|
||
rm "$TEST_TMP/accel0"
|
||
run_with_log
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"$TEST_TMP/accel0 missing"* ]]
|
||
[[ "$output" == *"iommu=pt"* ]]
|
||
}
|
||
|
||
@test "the lane refuses an NPU node the container namespace could not open, and names the rule" {
|
||
# Stock Debian mode is root:render 0660. The engine's own first words on such
|
||
# a node are "this process cannot open it (crw-rw---- nobody:nogroup)" and no
|
||
# flag fixes a mode an unmapped gid cannot cross (keep-groups measured dead
|
||
# 2026-10-19), so the unit refuses instead of crash-looping 189 times.
|
||
podman_stub
|
||
good_env
|
||
npu_fixture
|
||
chmod 660 "$TEST_TMP/accel0"
|
||
run_with_log
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"$TEST_TMP/accel0 is 660"* ]]
|
||
[[ "$output" == *"98-halogen-npu-access.rules"* ]]
|
||
[[ "$output" != *"--device"* ]]
|
||
}
|
||
|
||
@test "the NPU access rule is the accel counterpart of the GPU compute rule" {
|
||
[ -f "$NPU_RULE" ]
|
||
grep -qE '^KERNEL=="accel\[0-9\]\*", SUBSYSTEM=="accel", MODE="0666"$' "$NPU_RULE"
|
||
# The deployer installs the rule and re-triggers the node BEFORE it trusts
|
||
# the mode, so a machine that never had it self-heals instead of refusing.
|
||
grep -qE '98-halogen-npu-access\.rules' "$DEPLOYER"
|
||
grep -qE 'udevadm control --reload-rules' "$DEPLOYER"
|
||
grep -qE 'udevadm trigger --sysname-match=' "$DEPLOYER"
|
||
}
|
||
|
||
@test "the lane refuses when an XRT NPU library is missing" {
|
||
podman_stub
|
||
good_env
|
||
npu_fixture
|
||
rm "$TEST_TMP/xrt/libxrt_driver_xdna.so.2"
|
||
run_with_log
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"libxrt_driver_xdna.so.2 missing"* ]]
|
||
}
|
||
|
||
@test "the lane refuses while the fabric clock is moving" {
|
||
# The engine itself refuses to start in this state (GPU+NPU co-run hangs
|
||
# Strix Halo while FCLK changes speed) but cannot see host sysfs from
|
||
# inside the container — the proof has to happen on the host side.
|
||
podman_stub
|
||
good_env
|
||
npu_fixture
|
||
printf 'auto\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level"
|
||
run_with_log
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"not held"* ]]
|
||
[[ "$output" == *"sudo halogen-fabric-clock hold"* ]]
|
||
}
|
||
|
||
@test "the lane refuses when no fabric-clock control can be read" {
|
||
# A host whose sysfs exposes nothing cannot PROVE the hold, and a silent
|
||
# pass here is a boot that dies in the engine instead.
|
||
podman_stub
|
||
good_env
|
||
npu_fixture
|
||
rm -rf "$TEST_TMP/sys/class/drm"
|
||
run_with_log
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"cannot prove"* ]]
|
||
}
|
||
|
||
@test "the lane also passes when the clock is held at the high level" {
|
||
# The vendor's status accepts `high` as well as `manual` + the top fclk
|
||
# level; refusing a correctly-held host would block a deploy for nothing.
|
||
podman_stub
|
||
good_env
|
||
npu_fixture
|
||
printf 'high\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level"
|
||
run_with_log
|
||
[ "$status" -eq 0 ]
|
||
[[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]]
|
||
}
|
||
|
||
@test "the deployer gates the same lane artifacts and installs the fabric-clock unit" {
|
||
grep -qE 'LLM_DRAGOON_NPU_MODELS' "$DEPLOYER"
|
||
grep -qE 'install -m 0755 dankclaw/host/halogen-fabric-clock' "$DEPLOYER"
|
||
grep -qE 'install -m 0644 dankclaw/host/halogen-fabric-clock\.service' "$DEPLOYER"
|
||
grep -qE 'halogen-fabric-clock status' "$DEPLOYER"
|
||
grep -qE 'download-frank-dragoon-npu\.sh' "$DEPLOYER"
|
||
# The lane's own health gate is a real embedding, not a 200 on /health.
|
||
grep -qE '/v1/embeddings' "$DEPLOYER"
|
||
# Parity with the serve gate: a staging dir outside the payload dir is
|
||
# refused before the restart, not left to runc.
|
||
grep -qE 'outside the payload dir' "$DEPLOYER"
|
||
}
|
||
|
||
@test "the fabric-clock host unit is the vendor shape and boots on its own" {
|
||
[ -x "$HOST_UNIT" ]
|
||
grep -qE '^ (hold\)|release\)|status\))' "$HOST_UNIT"
|
||
grep -qE 'pp_dpm_fclk' "$HOST_UNIT"
|
||
grep -qE '^Type=oneshot$' "$HOST_SERVICE"
|
||
grep -qE '^WantedBy=.*multi-user\.target' "$HOST_SERVICE"
|
||
# A resume hands the clock back to the driver, so the unit must re-run then.
|
||
grep -qE '^WantedBy=.*suspend\.target' "$HOST_SERVICE"
|
||
grep -qE '^ExecStart=.*/usr/local/sbin/halogen-fabric-clock hold$' "$HOST_SERVICE"
|
||
}
|
||
|
||
@test "the NPU fetch script pins the vendor catalog by size and sha256" {
|
||
# XDNA2 runs only AIE-compiled programs, so the vendor's per-model repo is
|
||
# the only source; the engine re-checks size+sha256 against its own catalog
|
||
# at boot, so this manifest has to agree with it or the boot refuses.
|
||
[ -x "$NPU_FETCH" ]
|
||
grep -qE 'curl -fL' "$NPU_FETCH"
|
||
grep -qE 'qwen3-embedding-0\.6b\.hnpw 813032248 [0-9a-f]{64}' "$NPU_FETCH"
|
||
grep -qE 'sha256sum' "$NPU_FETCH"
|
||
# curl -f alone is not enough on Hugging Face: the resolve URL redirects
|
||
# and curl drops Range across it (the sibling 62 GiB fetch, 2026-10-18).
|
||
grep -qE 'url_effective' "$NPU_FETCH"
|
||
}
|
||
|
||
@test "the container network is podman's default when unnamed and the app bridge when named" {
|
||
# The app-to-endpoint leg hangs on this: pasta is a private netns nothing
|
||
# else can join (podman refuses a second network, the published port is
|
||
# refused from app containers, and host.containers.internal is link-local,
|
||
# which open-notebook refuses to store), so reaching the NPU endpoints by
|
||
# name means the container lives on dankclaw_default. Both arms are
|
||
# observable in the command podman would run.
|
||
podman_stub
|
||
good_env
|
||
run_with_log FRANK_DRAGOON_NETWORK=
|
||
[ "$status" -eq 0 ]
|
||
! grep -q -- '--network' "$TEST_TMP/serve.log"
|
||
run_with_log FRANK_DRAGOON_NETWORK=dankclaw_default
|
||
[ "$status" -eq 0 ]
|
||
[[ "$output" == *"--network dankclaw_default"* ]]
|
||
# The deployer refuses a network that does not exist rather than starting a
|
||
# unit that restart-loops on it.
|
||
grep -qE 'podman network exists' "$DEPLOYER"
|
||
grep -qE '^FRANK_DRAGOON_NETWORK=dankclaw_default$' "$ENV_FILE"
|
||
}
|