Frank
a3ff55efd8
The calibration mode kept a private step (8 seats at MNBT 65,536) that it called "free throughput". The step's activation arena is billed to the card, so the wide step cost the container its own expert slab and KV: at 65,536 the arena is 10.65 GiB and the claim falls to 16.85 GiB (weights 14.24 GiB in VRAM, 56.24 GiB stranded on disk, KV 271 MiB for 8 seats), and the boot dies with "startup stopped: out of memory" (measured 2026-10-29, while the kldref boot at MNBT 1,024 — arena 504 MiB, claimable 30.87 GiB — had just recorded for 16 hours without fault). Every mode now takes its seat count from FRANK_ARCHON_MAX_SEQS, the file that owns it, exactly as PR #77 established for the card split, and its step from FRANK_QWEN38_MNBT. The per-mode literals in kldref and score follow, so one number defines the geometry for all three. The test that pinned the old exemption is re-pinned to the invariant that replaced it. tests/run.sh: 576/576 green.
347 lines
16 KiB
Bash
Executable File
347 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the BF16 REFERENCE container of the Qwen3.8-Flash-Next five-bit
|
|
# campaign (feat/qwen38-w5nl64) — scripts/build-qwen38-bf16-rad.sh and
|
|
# scripts/run-qwen38-bf16.sh.
|
|
#
|
|
# The reference is what every later stage is judged or derived from (the KLD
|
|
# record scores the arms, the Grams derive the protected experts), so the
|
|
# deliverable here is the set of refusals: a half checkpoint, a second convert,
|
|
# a volume that cannot hold the convert, an engine-refused plan, and any run —
|
|
# including a dry one — that would take the GPUs from frank-archon. Each test
|
|
# drives the script's own exit status and printed argv. Offline: no engine, no
|
|
# GPU, no container, no systemctl, no network. Path knobs aim at
|
|
# $BATS_TEST_TMPDIR.
|
|
#
|
|
# The third mode, score, carries the campaign's comparability law: its engine
|
|
# argv must equal kldref's except for --model and the --kld-* pair, or the KL
|
|
# numbers of the FP8-native anchor, the served payload and the arms are not one
|
|
# measurement series. The seam proves that by diffing the two dry runs.
|
|
|
|
# The 1.8.1 law asserted in two places below: --vram-kv-mib was REMOVED at
|
|
# 1.3.0, so naming it is a boot refusal — the reference serve must carry the
|
|
# budget-resolver knobs (--host-pool-mib / --gpu-headroom-mib) instead.
|
|
|
|
load helpers
|
|
|
|
BUILD="$REPO_ROOT/scripts/build-qwen38-bf16-rad.sh"
|
|
RUNNER="$REPO_ROOT/scripts/run-qwen38-bf16.sh"
|
|
|
|
# The reference shape, spelled exactly as run-qwen38-bf16.sh must pass it: this
|
|
# is the contract the recorded reference is measured at, so a silent edit here
|
|
# is a silent edit to every KLD number in the campaign.
|
|
SHAPE="--tp 2 --placement expert_tiered --weights-disk-tier --host-pool-mib 49152 --gpu-headroom-mib 96 --deterministic --num-speculative-tokens 0 --kv-cache-dtype fp8 --max-model-len 32768 --ngram-placement ram --override-chat-template /models/tpl.jinja"
|
|
|
|
# oneline — both dry runs print one shell-quoted argument a line; joining them
|
|
# lets an option and its value be asserted as the pair they are.
|
|
oneline() { local s="${output//$'\n'/ }"; printf '%s' "$s"; }
|
|
|
|
# engine_argv — the arguments AFTER the image line of a dry run, with --model
|
|
# and every --kld-* flag and its value removed: what is left is the measuring
|
|
# shape, and it must be identical between kldref and score.
|
|
engine_argv() {
|
|
local seen=0 skip=0 a
|
|
while IFS= read -r a; do
|
|
if [ "$seen" = 0 ]; then
|
|
[[ "$a" == *"radiance@sha256:"* ]] && seen=1
|
|
continue
|
|
fi
|
|
case "$a" in
|
|
--model|--kld-record|--kld-corpus|--kld-ref|--kld-out) skip=1; continue ;;
|
|
esac
|
|
if [ "$skip" = 1 ]; then skip=0; continue; fi
|
|
printf '%s\n' "$a"
|
|
done <<< "$1"
|
|
}
|
|
|
|
# flag_value FLAG — the value a dry run printed for FLAG (the argv is one
|
|
# argument a line, so the value is the line after it). Lets a test ask what
|
|
# number the script chose instead of re-declaring the literal it guards.
|
|
flag_value() {
|
|
local want="$1" prev="" a
|
|
while IFS= read -r a; do
|
|
if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi
|
|
prev="$a"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
@test "the build dry run prints the rad-convert argv for the checkpoint and the bf16 output" {
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
run env FRANK_QWEN38_MODELS="$M" bash "$BUILD" --dry-run
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"rad-convert"* ]]
|
|
# the argv is the container's view: the models dir rides the /models mount
|
|
[[ "$output" == *"$M:/models"* ]]
|
|
flat="$(oneline)"
|
|
[[ "$flat" == *"/models/Qwen3.8-Flash-Next -o /models/rad/qwen38-bf16.rad"* ]]
|
|
# a reference is the checkpoint re-headed, not re-encoded, and it declares a
|
|
# window the long KLD corpus can be read inside of
|
|
[[ "$output" != *"--recipe"* ]]
|
|
[[ "$output" != *"--reuse"* ]]
|
|
[[ "$flat" == *"--max-ctx 32768"* ]]
|
|
# and a dry run looked at nothing: the fixture dirs were never created
|
|
[ ! -e "$M" ]
|
|
}
|
|
|
|
@test "the build refuses a checkpoint that is not on disk and names the fetch command" {
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
run env FRANK_QWEN38_MODELS="$M" bash "$BUILD"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"$M/Qwen3.8-Flash-Next"* ]]
|
|
[[ "$output" == *"scripts/download-qwen38-bf16.sh --fetch"* ]]
|
|
[ ! -e "$M/rad/qwen38-bf16.rad" ]
|
|
}
|
|
|
|
@test "the build refuses an existing reference container and names --force as the way past it" {
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
mkdir -p "$M/Qwen3.8-Flash-Next" "$M/rad"
|
|
echo "not a container" > "$M/rad/qwen38-bf16.rad"
|
|
run env FRANK_QWEN38_MODELS="$M" QWEN38_MIN_FREE_MIB=1 \
|
|
QWEN38_TRASH="$BATS_TEST_TMPDIR/trash" bash "$BUILD"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"$M/rad/qwen38-bf16.rad"* ]]
|
|
[[ "$output" == *"--force"* ]]
|
|
# refusing retires nothing: the trash law means the file survives a refusal
|
|
[ -f "$M/rad/qwen38-bf16.rad" ]
|
|
[ ! -e "$BATS_TEST_TMPDIR/trash" ]
|
|
}
|
|
|
|
@test "the build refuses when the output filesystem cannot hold a second 335 GiB" {
|
|
# The campaign's peak-disk law: 343,323 MiB of container written next to
|
|
# 343,323 MiB of checkpoint. The wanted figure is set above whatever this
|
|
# box really has, which is what the gate reads.
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
mkdir -p "$M/Qwen3.8-Flash-Next" "$M/rad"
|
|
run env FRANK_QWEN38_MODELS="$M" QWEN38_MIN_FREE_MIB=999999999 bash "$BUILD"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"REFUSED"* ]]
|
|
[[ "$output" =~ [0-9]+" MiB free" ]]
|
|
[[ "$output" == *"999999999 MiB wanted"* ]]
|
|
# the disk gate is the cheap one: it refuses long before the Hub sweep runs
|
|
[[ "$output" != *"VERIFY_OK"* ]]
|
|
[ ! -e "$M/rad/qwen38-bf16.rad" ]
|
|
}
|
|
|
|
@test "the reference serve rejects an unknown mode and names every valid mode" {
|
|
# `probe` is the swift15 era's third mode; it is retired and must not boot
|
|
# a container here either.
|
|
run bash "$RUNNER" probe
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"probe"* ]]
|
|
[[ "$output" == *"kldref"* ]]
|
|
[[ "$output" == *"calib"* ]]
|
|
[[ "$output" == *"score"* ]]
|
|
[[ "$output" != *"podman"* ]]
|
|
run bash "$RUNNER"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"kldref"* ]]
|
|
[[ "$output" == *"calib"* ]]
|
|
[[ "$output" == *"score"* ]]
|
|
# score without its two positionals is the same usage refusal
|
|
run bash "$RUNNER" score
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"score RAD OUT"* ]]
|
|
}
|
|
|
|
@test "the kldref dry run carries the record flags at the reference shape and no --vram-kv-mib" {
|
|
run bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
flat="$(oneline)"
|
|
[[ "$flat" == *"--kld-record /data/kld-ref"* ]]
|
|
[[ "$flat" == *"--kld-corpus /data/kld.jsonl"* ]]
|
|
# the corpus arrives read-only, the record dir writable
|
|
[[ "$flat" == *":/data/kld.jsonl:ro"* ]]
|
|
[[ "$flat" == *":/data/kld-ref "* ]]
|
|
[[ "$flat" == *"$SHAPE"* ]]
|
|
# 1.8.1: the flag does not exist — naming it is a boot refusal
|
|
[[ "$output" != *"--vram-kv-mib"* ]]
|
|
}
|
|
|
|
@test "the calib dry run opens the tap at 12M tokens and serves on 7770" {
|
|
run bash "$RUNNER" calib --dry-run
|
|
[ "$status" -eq 0 ]
|
|
flat="$(oneline)"
|
|
[[ "$flat" == *"-e RADIANCE_CALIB_TOKENS=12000000"* ]]
|
|
[[ "$flat" == *"-e RADIANCE_CALIB_DIR=/data/calib"* ]]
|
|
[[ "$flat" == *"--port 7770"* ]]
|
|
[[ "$flat" == *":/data/calib "* ]]
|
|
# calib serves; it does not record
|
|
[[ "$output" != *"--kld-record"* ]]
|
|
}
|
|
|
|
@test "a dry run prints argv only: no drain, no container, nothing written" {
|
|
L="$BATS_TEST_TMPDIR/logs"
|
|
run env FRANK_QWEN38_LOG_DIR="$L" \
|
|
bash "$RUNNER" kldref --dry-run --image example.invalid/radiance@sha256:0000000000000000000000000000000000000000000000000000000000000000
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" != *"systemctl"* ]]
|
|
[[ "$output" != *"frank-archon stopped"* ]]
|
|
[[ "$output" != *"podman rm"* ]]
|
|
# not vacuous: the podman argv WAS printed, and --image replaced the pin
|
|
[[ "$output" == *"podman"* ]]
|
|
[[ "$output" == *"example.invalid/radiance@sha256:000"* ]]
|
|
[[ "$output" != *"stilldeadcode/radiance"* ]]
|
|
[ ! -d "$L" ]
|
|
}
|
|
|
|
@test "the score dry run is the kldref argv with only --model and the --kld-* pair changed" {
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
C="$BATS_TEST_TMPDIR/campaign"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \
|
|
bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
ref_argv="$(engine_argv "$output")"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \
|
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run
|
|
[ "$status" -eq 0 ]
|
|
score_argv="$(engine_argv "$output")"
|
|
# The whole value of the campaign's three KL numbers is that they rode this
|
|
# one shape: same pin, --tp 2, fp8 KV, step size, --deterministic, no spec.
|
|
if [[ "$score_argv" != "$ref_argv" ]]; then
|
|
echo "--- kldref engine argv ---"; echo "$ref_argv"
|
|
echo "--- score engine argv ---"; echo "$score_argv"
|
|
return 1
|
|
fi
|
|
flat="$(oneline)"
|
|
[[ "$flat" == *"--model /models/rad/arm-w5nl64.rad"* ]]
|
|
[[ "$flat" == *"--kld-ref /data/kld-ref --kld-out /data/out/arm1.json"* ]]
|
|
# the zero point mounts read-only; the report needs a writable bind because
|
|
# --kld-out F also writes F.rows beside it
|
|
[[ "$flat" == *":/data/kld-ref:ro "* ]]
|
|
[[ "$flat" == *":/data/out "* ]]
|
|
[[ "$output" != *"--kld-record"* ]]
|
|
[[ "$output" != *"--kld-corpus"* ]]
|
|
[[ "$output" != *"--vram-kv-mib"* ]]
|
|
}
|
|
|
|
@test "score refuses a missing reference, a missing container and an existing report" {
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
C="$BATS_TEST_TMPDIR/campaign"
|
|
L="$BATS_TEST_TMPDIR/logs"
|
|
mkdir -p "$M/rad"
|
|
: > "$M/rad/arm-w5nl64.rad"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \
|
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"$C/kld-ref"* ]]
|
|
[[ "$output" == *"run-qwen38-bf16.sh kldref"* ]]
|
|
# refused before the log dir, so before anything that could reach systemctl
|
|
[ ! -d "$L" ]
|
|
|
|
mkdir -p "$C/kld-ref"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \
|
|
bash "$RUNNER" score "$M/rad/gone.rad" "$C/scores/arm1.json"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"$M/rad/gone.rad"* ]]
|
|
[ ! -d "$L" ]
|
|
|
|
mkdir -p "$C/scores"
|
|
: > "$C/scores/arm1.json"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \
|
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"$C/scores/arm1.json"* ]]
|
|
[[ "$output" == *"trash"* ]]
|
|
# the trash law: a refusal keeps the old report, it does not overwrite it
|
|
[ -f "$C/scores/arm1.json" ]
|
|
[ ! -d "$L" ]
|
|
}
|
|
|
|
@test "every mode rides ONE step size and ONE seat count, legal on the file tier" {
|
|
# The gate's claim is one shape; that shape's ceiling is set by the tightest
|
|
# payload it boots, and the served iq4r and every arm are file-tier
|
|
# containers, which stream only at <=2048. A reference recorded above that is
|
|
# a KL between two shapes, not two weights. Calib used to exempt itself from
|
|
# the shared step; the exemption cost the container its own cache, so the
|
|
# test now holds every mode to the same two numbers.
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
C="$BATS_TEST_TMPDIR/campaign"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
ref_step="$(flag_value --max-num-batched-tokens <<< "$output")"
|
|
ref_seats="$(flag_value --max-num-seqs <<< "$output")"
|
|
[ -n "$ref_step" ]
|
|
[ "$ref_step" -le 2048 ]
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \
|
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run
|
|
[ "$status" -eq 0 ]
|
|
score_step="$(flag_value --max-num-batched-tokens <<< "$output")"
|
|
[ "$ref_step" = "$score_step" ]
|
|
# one knob moves both, so the operator can lower it at 2 a.m. without an edit
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \
|
|
bash "$RUNNER" kldref --dry-run
|
|
[ "$(flag_value --max-num-batched-tokens <<< "$output")" = "1024" ]
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \
|
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run
|
|
[ "$(flag_value --max-num-batched-tokens <<< "$output")" = "1024" ]
|
|
# calibration rides the shared step too. Its old exemption (8 seats at
|
|
# MNBT 65,536, "free throughput") is what made the container refuse to
|
|
# boot: at that step the engine charges a 10.65 GiB activation arena to
|
|
# the card, the experts fall to 10.98 GiB and the KV to 271 MiB —
|
|
# "0.0 full-length sessions", "startup stopped: out of memory", measured
|
|
# 2026-10-29 while kldref on the shared step ran 16 hours without fault.
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \
|
|
bash "$RUNNER" calib --dry-run
|
|
[ "$(flag_value --max-num-batched-tokens <<< "$output")" = "1024" ]
|
|
[ "$(flag_value --max-num-seqs <<< "$output")" = "$ref_seats" ]
|
|
# a step size that is not a token count is refused, never handed to the engine
|
|
run env FRANK_QWEN38_MNBT=broad bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"FRANK_QWEN38_MNBT"* ]]
|
|
[[ "$output" == *"must be a token count"* ]]
|
|
}
|
|
|
|
@test "kldref refuses a corpus its window cannot hold, before the service is touched" {
|
|
M="$BATS_TEST_TMPDIR/models" R="$M/rad"
|
|
mkdir -p "$M/tpl" "$R" "$BATS_TEST_TMPDIR/out"
|
|
: > "$M/tpl/qwen38-froggeric-v22.5.jinja"
|
|
: > "$R/qwen38-bf16.rad"
|
|
CORPUS="$BATS_TEST_TMPDIR/long.jsonl"
|
|
{
|
|
printf '{"prompt":"short one"}\n'
|
|
printf '{"prompt":"%s"}\n' "$(head -c 120000 /dev/zero | tr '\0' 'x')"
|
|
printf '{"prompt":"short three"}\n'
|
|
} > "$CORPUS"
|
|
# 32,768 tokens x 3 chars is the pessimistic limit; the 120,002-char line
|
|
# that opens record 2 is well past it, so the campaign says no up front.
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
|
bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"REFUSED"* ]]
|
|
[[ "$output" == *"record 2"* ]]
|
|
[[ "$output" == *"120013 chars"* ]]
|
|
[[ "$output" == *"past the 98304 a 32768-token window"* ]]
|
|
[[ "$output" == *"FRANK_QWEN38_MAX_MODEL_LEN"* ]]
|
|
# the same corpus is fine once the window is raised past it, and the raised
|
|
# window is what the boot is asked for
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
|
FRANK_QWEN38_MAX_MODEL_LEN=65536 FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" \
|
|
bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"corpus fits"* ]]
|
|
flat="$(oneline)"
|
|
[[ "$flat" == *"--max-model-len 65536"* ]]
|
|
# the shipped corpus of three short records fits the shipped window
|
|
SHORT="$BATS_TEST_TMPDIR/short.jsonl"
|
|
printf '{"prompt":"tiny"}\n{"prompt":"also tiny"}\n' > "$SHORT"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$SHORT" \
|
|
FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"corpus fits: longest record 22 chars (number 2)"* ]]
|
|
# a real kldref run refuses at the same place: no log dir, no drain, and the
|
|
# service would still be up when the script gave up
|
|
L="$BATS_TEST_TMPDIR/logs-real"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
|
FRANK_QWEN38_LOG_DIR="$L" bash "$RUNNER" kldref
|
|
[ "$status" -eq 2 ]
|
|
[ ! -d "$L" ]
|
|
# and it is kldref that reads the corpus: score and calib never consult it
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
|
FRANK_QWEN38_KLD_REF_DIR="$BATS_TEST_TMPDIR/ref" \
|
|
FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" \
|
|
bash "$RUNNER" score "$R/qwen38-bf16.rad" "$BATS_TEST_TMPDIR/out/arm.json" --dry-run
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" != *"corpus fits"* ]]
|
|
[[ "$output" != *"REFUSED"* ]]
|
|
}
|