diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index a982279..d952d8e 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -4,9 +4,9 @@ # went back to the halogen engine on the iGPU. This seam pins what the flags # alone do not say: boot persistence for a NATIVE runtime (no generated quadlet # is involved), the repo-relative paths the unit executes, the separate client -# port, the MTP contract (the head lives in the weights file, and no second copy -# of the model is loaded for it), the vision handoff from frank-large, and the -# KV budget that has to fit a 24 GiB card beside 12 GiB of weights. +# port, the MTP contract (the head is a separate sidecar file loaded as the +# draft model, chained behind ngram-mod), the vision handoff from frank-large, +# and the KV budget: q8_0 quantized KV whose context --fit sizes to the card. load helpers @@ -45,41 +45,42 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" grep -q -- '--metrics' "$SERVE" } -@test "frank-small's MTP head comes from the weights file, not a second copy" { - grep -q '^SPEC_TYPE=draft-mtp$' "$ENV_FILE" +@test "frank-small drafts with the MTP sidecar chained behind ngram-mod" { + grep -q '^SPEC_TYPE=draft-mtp,ngram-mod$' "$ENV_FILE" grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE" - # The -mtp build is the one carrying blk.64.nextn.*; the plain IQ3_S has no - # MTP tensors at all, so MTP is a property of the file, not of the flags. - grep -q '^MODEL_FILE=.*-mtp\.gguf$' "$ENV_FILE" - # A separate draft model would load a second 12 GiB copy of a 12 GiB model. - # Only the effective configuration counts: the script's own comment explains - # why there is no draft path, and a textual grep would match that prose. - ! grep -vE '^[[:space:]]*#' "$SERVE" | grep -q -- '--model-draft' + # The Gemma4 MTP head is a separate assistant file (gemma4-assistant, 4 + # nextn layers); it loads as --model-draft and shares the target's last + # layers' KV. ngram-mod is draftless, so the chain is legal. + grep -q '^DRAFT_FILE=mtp-gemma-4-12B-it\.gguf$' "$ENV_FILE" + grep -q -- '--model-draft "$MODEL_DIR/$DRAFT_FILE"' "$SERVE" } @test "frank-small carries the fleet's image path, and only it" { - grep -q '^MMPROJ_PATH=.*mmproj-Qwen3\.8-27B-BF16\.gguf$' "$ENV_FILE" + grep -q '^MMPROJ_PATH=.*mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16\.gguf$' "$ENV_FILE" grep -q -- '--mmproj "$MMPROJ_PATH"' "$SERVE" ! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE" } -@test "frank-small's KV fits the card beside the weights it loads" { - local ctx parallel parallel_slots cache_k cache_v - ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" +@test "frank-small quantizes KV to q8 and lets --fit size the context" { + local parallel cache_k cache_v ctx parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")" cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")" cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")" - [ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] + ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" + [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] [ "$parallel" -ge 1 ] - # qwen35 spends KV on its 16 full-attention layers only: 2 (K,V) x 4 heads x - # 256 dim x 2 B = 64 KiB per token at f16. 11.29 GiB of weights + 0.87 of - # mmproj + ~2 GiB of context/compute/MTP leave under 10 GiB of the card's - # 24 GiB for KV, and --parallel divides --ctx-size between the slots. - [ $((ctx * 64)) -le $((10 * 1024 * 1024)) ] - # f16 is the deployed type on purpose: a quantized KV needs the - # GGML_CUDA_FA_ALL_QUANTS build option, without which prefill on CUDA - # collapses about 12x (see the CUDA lesson in AGENTS.md). - [ "$cache_k" = "f16" ] && [ "$cache_v" = "f16" ] + # q8_0 KV is what fits the 24 GiB card beside 7.4 GiB of Q4_K_M weights + + # mmproj + sidecar: Gemma4's 16 kv heads x 512 dim cost 16 KiB per token + # per layer at q8, and the model's sliding-window pattern keeps the pool + # far below full-ctx x all-layers. The engine itself sizes the context: + # CTX_SIZE is unset, so the serve script passes --fit on. + [ -z "$ctx" ] + [ "$cache_k" = "q8_0" ] && [ "$cache_v" = "q8_0" ] + grep -q -- '--fit on' "$SERVE" + # Quantized KV on CUDA needs the GGML_CUDA_FA_ALL_QUANTS build option, + # without which prefill collapses about 12x (see the CUDA lesson in + # AGENTS.md). The build-cuda tree must carry it. + grep -q '^FLASH_ATTN=on$' "$ENV_FILE" } @test "frank-small runs on its own port beside frank-large's" { @@ -98,7 +99,7 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" grep -q 'relink_unit frank-small' "$REMOTE_DEPLOY" grep -q 'systemctl enable frank-small.service' "$REMOTE_DEPLOY" grep -q 'systemctl restart frank-small.service' "$REMOTE_DEPLOY" - grep -q 'Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp' "$REMOTE_DEPLOY" + grep -q 'Gemma4-12B-QAT' "$REMOTE_DEPLOY" grep -q 'refusing to start a unit that cannot load' "$REMOTE_DEPLOY" grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY" }