Frank
96c5a0625c
run-qwen38-bf16.sh carried its own --host-pool-mib 12288 / --gpu-headroom-mib 768,
copied from the w4-era swift15 rig. The engine derives the whole per-card split from
those two numbers, and the KV cache's funding is `elastic - expert_floor` where the
expert floor is a function of the pinned host pool — so the pool buys cache as well
as residency. The first kldref boot got a 270 MiB / 7,436-position pool and the
recorder refused all 197 corpus documents ("document 0 is 10554 tokens and the KV
pool holds 7436"), exiting 1.
The rig now sources FRANK_ARCHON_HOST_POOL_MIB / FRANK_ARCHON_GPU_HEADROOM_MIB from
scripts/frank-archon.env, the file that owns them (49152 / 96), with `:?` guards, and
the header records the law instead of the claim that the pool never touches logits.
tests/qwen38_reference.bats pins the shape verbatim, so its SHAPE contract moves with
it; the suite is green (558 tests).
338 lines
15 KiB
Bash
Executable File
338 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the BF16 REFERENCE container of the Qwen3.8-Flash-Next five-bit
|
|
# campaign (feat/qwen38-w5nl64) — scripts/build-qwen38-bf16-rad.sh and
|
|
# scripts/run-qwen38-bf16.sh.
|
|
#
|
|
# The reference is what every later stage is judged or derived from (the KLD
|
|
# record scores the arms, the Grams derive the protected experts), so the
|
|
# deliverable here is the set of refusals: a half checkpoint, a second convert,
|
|
# a volume that cannot hold the convert, an engine-refused plan, and any run —
|
|
# including a dry one — that would take the GPUs from frank-archon. Each test
|
|
# drives the script's own exit status and printed argv. Offline: no engine, no
|
|
# GPU, no container, no systemctl, no network. Path knobs aim at
|
|
# $BATS_TEST_TMPDIR.
|
|
#
|
|
# The third mode, score, carries the campaign's comparability law: its engine
|
|
# argv must equal kldref's except for --model and the --kld-* pair, or the KL
|
|
# numbers of the FP8-native anchor, the served payload and the arms are not one
|
|
# measurement series. The seam proves that by diffing the two dry runs.
|
|
|
|
# The 1.8.1 law asserted in two places below: --vram-kv-mib was REMOVED at
|
|
# 1.3.0, so naming it is a boot refusal — the reference serve must carry the
|
|
# budget-resolver knobs (--host-pool-mib / --gpu-headroom-mib) instead.
|
|
|
|
load helpers
|
|
|
|
BUILD="$REPO_ROOT/scripts/build-qwen38-bf16-rad.sh"
|
|
RUNNER="$REPO_ROOT/scripts/run-qwen38-bf16.sh"
|
|
|
|
# The reference shape, spelled exactly as run-qwen38-bf16.sh must pass it: this
|
|
# is the contract the recorded reference is measured at, so a silent edit here
|
|
# is a silent edit to every KLD number in the campaign.
|
|
SHAPE="--tp 2 --placement expert_tiered --weights-disk-tier --host-pool-mib 49152 --gpu-headroom-mib 96 --deterministic --num-speculative-tokens 0 --kv-cache-dtype fp8 --max-model-len 32768 --ngram-placement ram --override-chat-template /models/tpl.jinja"
|
|
|
|
# oneline — both dry runs print one shell-quoted argument a line; joining them
|
|
# lets an option and its value be asserted as the pair they are.
|
|
oneline() { local s="${output//$'\n'/ }"; printf '%s' "$s"; }
|
|
|
|
# engine_argv — the arguments AFTER the image line of a dry run, with --model
|
|
# and every --kld-* flag and its value removed: what is left is the measuring
|
|
# shape, and it must be identical between kldref and score.
|
|
engine_argv() {
|
|
local seen=0 skip=0 a
|
|
while IFS= read -r a; do
|
|
if [ "$seen" = 0 ]; then
|
|
[[ "$a" == *"radiance@sha256:"* ]] && seen=1
|
|
continue
|
|
fi
|
|
case "$a" in
|
|
--model|--kld-record|--kld-corpus|--kld-ref|--kld-out) skip=1; continue ;;
|
|
esac
|
|
if [ "$skip" = 1 ]; then skip=0; continue; fi
|
|
printf '%s\n' "$a"
|
|
done <<< "$1"
|
|
}
|
|
|
|
# flag_value FLAG — the value a dry run printed for FLAG (the argv is one
|
|
# argument a line, so the value is the line after it). Lets a test ask what
|
|
# number the script chose instead of re-declaring the literal it guards.
|
|
flag_value() {
|
|
local want="$1" prev="" a
|
|
while IFS= read -r a; do
|
|
if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi
|
|
prev="$a"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
@test "the build dry run prints the rad-convert argv for the checkpoint and the bf16 output" {
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
run env FRANK_QWEN38_MODELS="$M" bash "$BUILD" --dry-run
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"rad-convert"* ]]
|
|
# the argv is the container's view: the models dir rides the /models mount
|
|
[[ "$output" == *"$M:/models"* ]]
|
|
flat="$(oneline)"
|
|
[[ "$flat" == *"/models/Qwen3.8-Flash-Next -o /models/rad/qwen38-bf16.rad"* ]]
|
|
# a reference is the checkpoint re-headed, not re-encoded, and it declares a
|
|
# window the long KLD corpus can be read inside of
|
|
[[ "$output" != *"--recipe"* ]]
|
|
[[ "$output" != *"--reuse"* ]]
|
|
[[ "$flat" == *"--max-ctx 32768"* ]]
|
|
# and a dry run looked at nothing: the fixture dirs were never created
|
|
[ ! -e "$M" ]
|
|
}
|
|
|
|
@test "the build refuses a checkpoint that is not on disk and names the fetch command" {
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
run env FRANK_QWEN38_MODELS="$M" bash "$BUILD"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"$M/Qwen3.8-Flash-Next"* ]]
|
|
[[ "$output" == *"scripts/download-qwen38-bf16.sh --fetch"* ]]
|
|
[ ! -e "$M/rad/qwen38-bf16.rad" ]
|
|
}
|
|
|
|
@test "the build refuses an existing reference container and names --force as the way past it" {
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
mkdir -p "$M/Qwen3.8-Flash-Next" "$M/rad"
|
|
echo "not a container" > "$M/rad/qwen38-bf16.rad"
|
|
run env FRANK_QWEN38_MODELS="$M" QWEN38_MIN_FREE_MIB=1 \
|
|
QWEN38_TRASH="$BATS_TEST_TMPDIR/trash" bash "$BUILD"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"$M/rad/qwen38-bf16.rad"* ]]
|
|
[[ "$output" == *"--force"* ]]
|
|
# refusing retires nothing: the trash law means the file survives a refusal
|
|
[ -f "$M/rad/qwen38-bf16.rad" ]
|
|
[ ! -e "$BATS_TEST_TMPDIR/trash" ]
|
|
}
|
|
|
|
@test "the build refuses when the output filesystem cannot hold a second 335 GiB" {
|
|
# The campaign's peak-disk law: 343,323 MiB of container written next to
|
|
# 343,323 MiB of checkpoint. The wanted figure is set above whatever this
|
|
# box really has, which is what the gate reads.
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
mkdir -p "$M/Qwen3.8-Flash-Next" "$M/rad"
|
|
run env FRANK_QWEN38_MODELS="$M" QWEN38_MIN_FREE_MIB=999999999 bash "$BUILD"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"REFUSED"* ]]
|
|
[[ "$output" =~ [0-9]+" MiB free" ]]
|
|
[[ "$output" == *"999999999 MiB wanted"* ]]
|
|
# the disk gate is the cheap one: it refuses long before the Hub sweep runs
|
|
[[ "$output" != *"VERIFY_OK"* ]]
|
|
[ ! -e "$M/rad/qwen38-bf16.rad" ]
|
|
}
|
|
|
|
@test "the reference serve rejects an unknown mode and names every valid mode" {
|
|
# `probe` is the swift15 era's third mode; it is retired and must not boot
|
|
# a container here either.
|
|
run bash "$RUNNER" probe
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"probe"* ]]
|
|
[[ "$output" == *"kldref"* ]]
|
|
[[ "$output" == *"calib"* ]]
|
|
[[ "$output" == *"score"* ]]
|
|
[[ "$output" != *"podman"* ]]
|
|
run bash "$RUNNER"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"kldref"* ]]
|
|
[[ "$output" == *"calib"* ]]
|
|
[[ "$output" == *"score"* ]]
|
|
# score without its two positionals is the same usage refusal
|
|
run bash "$RUNNER" score
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"score RAD OUT"* ]]
|
|
}
|
|
|
|
@test "the kldref dry run carries the record flags at the reference shape and no --vram-kv-mib" {
|
|
run bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
flat="$(oneline)"
|
|
[[ "$flat" == *"--kld-record /data/kld-ref"* ]]
|
|
[[ "$flat" == *"--kld-corpus /data/kld.jsonl"* ]]
|
|
# the corpus arrives read-only, the record dir writable
|
|
[[ "$flat" == *":/data/kld.jsonl:ro"* ]]
|
|
[[ "$flat" == *":/data/kld-ref "* ]]
|
|
[[ "$flat" == *"$SHAPE"* ]]
|
|
# 1.8.1: the flag does not exist — naming it is a boot refusal
|
|
[[ "$output" != *"--vram-kv-mib"* ]]
|
|
}
|
|
|
|
@test "the calib dry run opens the tap at 12M tokens and serves on 7770" {
|
|
run bash "$RUNNER" calib --dry-run
|
|
[ "$status" -eq 0 ]
|
|
flat="$(oneline)"
|
|
[[ "$flat" == *"-e RADIANCE_CALIB_TOKENS=12000000"* ]]
|
|
[[ "$flat" == *"-e RADIANCE_CALIB_DIR=/data/calib"* ]]
|
|
[[ "$flat" == *"--port 7770"* ]]
|
|
[[ "$flat" == *":/data/calib "* ]]
|
|
# calib serves; it does not record
|
|
[[ "$output" != *"--kld-record"* ]]
|
|
}
|
|
|
|
@test "a dry run prints argv only: no drain, no container, nothing written" {
|
|
L="$BATS_TEST_TMPDIR/logs"
|
|
run env FRANK_QWEN38_LOG_DIR="$L" \
|
|
bash "$RUNNER" kldref --dry-run --image example.invalid/radiance@sha256:0000000000000000000000000000000000000000000000000000000000000000
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" != *"systemctl"* ]]
|
|
[[ "$output" != *"frank-archon stopped"* ]]
|
|
[[ "$output" != *"podman rm"* ]]
|
|
# not vacuous: the podman argv WAS printed, and --image replaced the pin
|
|
[[ "$output" == *"podman"* ]]
|
|
[[ "$output" == *"example.invalid/radiance@sha256:000"* ]]
|
|
[[ "$output" != *"stilldeadcode/radiance"* ]]
|
|
[ ! -d "$L" ]
|
|
}
|
|
|
|
@test "the score dry run is the kldref argv with only --model and the --kld-* pair changed" {
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
C="$BATS_TEST_TMPDIR/campaign"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \
|
|
bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
ref_argv="$(engine_argv "$output")"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \
|
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run
|
|
[ "$status" -eq 0 ]
|
|
score_argv="$(engine_argv "$output")"
|
|
# The whole value of the campaign's three KL numbers is that they rode this
|
|
# one shape: same pin, --tp 2, fp8 KV, step size, --deterministic, no spec.
|
|
if [[ "$score_argv" != "$ref_argv" ]]; then
|
|
echo "--- kldref engine argv ---"; echo "$ref_argv"
|
|
echo "--- score engine argv ---"; echo "$score_argv"
|
|
return 1
|
|
fi
|
|
flat="$(oneline)"
|
|
[[ "$flat" == *"--model /models/rad/arm-w5nl64.rad"* ]]
|
|
[[ "$flat" == *"--kld-ref /data/kld-ref --kld-out /data/out/arm1.json"* ]]
|
|
# the zero point mounts read-only; the report needs a writable bind because
|
|
# --kld-out F also writes F.rows beside it
|
|
[[ "$flat" == *":/data/kld-ref:ro "* ]]
|
|
[[ "$flat" == *":/data/out "* ]]
|
|
[[ "$output" != *"--kld-record"* ]]
|
|
[[ "$output" != *"--kld-corpus"* ]]
|
|
[[ "$output" != *"--vram-kv-mib"* ]]
|
|
}
|
|
|
|
@test "score refuses a missing reference, a missing container and an existing report" {
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
C="$BATS_TEST_TMPDIR/campaign"
|
|
L="$BATS_TEST_TMPDIR/logs"
|
|
mkdir -p "$M/rad"
|
|
: > "$M/rad/arm-w5nl64.rad"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \
|
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"$C/kld-ref"* ]]
|
|
[[ "$output" == *"run-qwen38-bf16.sh kldref"* ]]
|
|
# refused before the log dir, so before anything that could reach systemctl
|
|
[ ! -d "$L" ]
|
|
|
|
mkdir -p "$C/kld-ref"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \
|
|
bash "$RUNNER" score "$M/rad/gone.rad" "$C/scores/arm1.json"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"$M/rad/gone.rad"* ]]
|
|
[ ! -d "$L" ]
|
|
|
|
mkdir -p "$C/scores"
|
|
: > "$C/scores/arm1.json"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_LOG_DIR="$L" \
|
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json"
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"$C/scores/arm1.json"* ]]
|
|
[[ "$output" == *"trash"* ]]
|
|
# the trash law: a refusal keeps the old report, it does not overwrite it
|
|
[ -f "$C/scores/arm1.json" ]
|
|
[ ! -d "$L" ]
|
|
}
|
|
|
|
@test "the reference and a score pass ride ONE step size, legal on the file tier" {
|
|
# The gate's claim is one shape; that shape's ceiling is set by the tightest
|
|
# payload it boots, and the served iq4r and every arm are file-tier
|
|
# containers, which stream only at <=2048. A reference recorded above that is
|
|
# a KL between two shapes, not two weights.
|
|
M="$BATS_TEST_TMPDIR/models"
|
|
C="$BATS_TEST_TMPDIR/campaign"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
ref_step="$(flag_value --max-num-batched-tokens <<< "$output")"
|
|
[ -n "$ref_step" ]
|
|
[ "$ref_step" -le 2048 ]
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" \
|
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run
|
|
[ "$status" -eq 0 ]
|
|
score_step="$(flag_value --max-num-batched-tokens <<< "$output")"
|
|
[ "$ref_step" = "$score_step" ]
|
|
# one knob moves both, so the operator can lower it at 2 a.m. without an edit
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \
|
|
bash "$RUNNER" kldref --dry-run
|
|
[ "$(flag_value --max-num-batched-tokens <<< "$output")" = "1024" ]
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \
|
|
bash "$RUNNER" score "$M/rad/arm-w5nl64.rad" "$C/scores/arm1.json" --dry-run
|
|
[ "$(flag_value --max-num-batched-tokens <<< "$output")" = "1024" ]
|
|
# calibration is exempt: grams are not logits, so the tap keeps the wide step
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_CAMPAIGN="$C" FRANK_QWEN38_MNBT=1024 \
|
|
bash "$RUNNER" calib --dry-run
|
|
[ "$(flag_value --max-num-batched-tokens <<< "$output")" = "65536" ]
|
|
# a step size that is not a token count is refused, never handed to the engine
|
|
run env FRANK_QWEN38_MNBT=broad bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"FRANK_QWEN38_MNBT"* ]]
|
|
[[ "$output" == *"must be a token count"* ]]
|
|
}
|
|
|
|
@test "kldref refuses a corpus its window cannot hold, before the service is touched" {
|
|
M="$BATS_TEST_TMPDIR/models" R="$M/rad"
|
|
mkdir -p "$M/tpl" "$R" "$BATS_TEST_TMPDIR/out"
|
|
: > "$M/tpl/qwen38-froggeric-v22.5.jinja"
|
|
: > "$R/qwen38-bf16.rad"
|
|
CORPUS="$BATS_TEST_TMPDIR/long.jsonl"
|
|
{
|
|
printf '{"prompt":"short one"}\n'
|
|
printf '{"prompt":"%s"}\n' "$(head -c 120000 /dev/zero | tr '\0' 'x')"
|
|
printf '{"prompt":"short three"}\n'
|
|
} > "$CORPUS"
|
|
# 32,768 tokens x 3 chars is the pessimistic limit; the 120,002-char line
|
|
# that opens record 2 is well past it, so the campaign says no up front.
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
|
bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 2 ]
|
|
[[ "$output" == *"REFUSED"* ]]
|
|
[[ "$output" == *"record 2"* ]]
|
|
[[ "$output" == *"120013 chars"* ]]
|
|
[[ "$output" == *"past the 98304 a 32768-token window"* ]]
|
|
[[ "$output" == *"FRANK_QWEN38_MAX_MODEL_LEN"* ]]
|
|
# the same corpus is fine once the window is raised past it, and the raised
|
|
# window is what the boot is asked for
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
|
FRANK_QWEN38_MAX_MODEL_LEN=65536 FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" \
|
|
bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"corpus fits"* ]]
|
|
flat="$(oneline)"
|
|
[[ "$flat" == *"--max-model-len 65536"* ]]
|
|
# the shipped corpus of three short records fits the shipped window
|
|
SHORT="$BATS_TEST_TMPDIR/short.jsonl"
|
|
printf '{"prompt":"tiny"}\n{"prompt":"also tiny"}\n' > "$SHORT"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$SHORT" \
|
|
FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" bash "$RUNNER" kldref --dry-run
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"corpus fits: longest record 22 chars (number 2)"* ]]
|
|
# a real kldref run refuses at the same place: no log dir, no drain, and the
|
|
# service would still be up when the script gave up
|
|
L="$BATS_TEST_TMPDIR/logs-real"
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
|
FRANK_QWEN38_LOG_DIR="$L" bash "$RUNNER" kldref
|
|
[ "$status" -eq 2 ]
|
|
[ ! -d "$L" ]
|
|
# and it is kldref that reads the corpus: score and calib never consult it
|
|
run env FRANK_QWEN38_MODELS="$M" FRANK_QWEN38_KLD_CORPUS="$CORPUS" \
|
|
FRANK_QWEN38_KLD_REF_DIR="$BATS_TEST_TMPDIR/ref" \
|
|
FRANK_QWEN38_LOG_DIR="$BATS_TEST_TMPDIR/logs" \
|
|
bash "$RUNNER" score "$R/qwen38-bf16.rad" "$BATS_TEST_TMPDIR/out/arm.json" --dry-run
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" != *"corpus fits"* ]]
|
|
[[ "$output" != *"REFUSED"* ]]
|
|
}
|