frank-small: two 128k slots, no spec, ubatch 512 (Frank's tight-fit order, 2026-10-06)
CTX 262144 / PARALLEL 2 = two 131,072 conversations; SPEC_TYPE=none strips both speculative flags in the serve script (ngram-mod skipped on purpose — on a qwen35 hybrid it disables recurrent-state rollback); ubatch 1024->512 claws back ~0.3 GiB. Budget: non-KV ~15.5 GiB + two q8_0 slots 8,704 MiB lands AT the card — the 2-3 GiB floor is knowingly violated by explicit order; nvidia-smi free at boot is the measured truth. Seam retargeted: shape (2 slots, 131072/slot), budget (two fit, three don't), spec-off (absence of both flags), unit Description.
This commit is contained in:
+36
-33
@@ -1,13 +1,15 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the frank-small llama.cpp root unit on dankclaw.
|
||||
# frank-small is the RTX 3090 endpoint; since 2026-10-05 evening it serves
|
||||
# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S-mtp (27B, MTP head in the
|
||||
# file) via a CUDA (sm_86) build of llama.cpp master: ONE 131,072-token
|
||||
# conversation at Frank's 128k cap, q8_0 K + q8_0 V (FP8 class), the
|
||||
# in-file MTP head drafting 3, and a vision projector. The card must keep
|
||||
# 2-3 GiB free. This seam runs the serve script against a stub engine and
|
||||
# asserts on the argv it actually execs, so what is pinned is the
|
||||
# command the operator gets.
|
||||
# frank-small is the RTX 3090 endpoint; since 2026-10-06 it serves
|
||||
# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S-mtp (27B; the -mtp file's
|
||||
# head is present but NOT drafted — speculation is off) via a CUDA (sm_86)
|
||||
# build of llama.cpp master: TWO 131,072-token conversations at Frank's
|
||||
# 128k cap (pool 262144 / parallel 2), q8_0 K + q8_0 V (FP8 class), and a
|
||||
# vision projector. Frank accepted the tight fit explicitly — the card may
|
||||
# land under the old 2-3 GiB floor; the seam pins the arithmetic, the boot
|
||||
# log carries the measured free. This seam runs the serve script against a
|
||||
# stub engine and asserts on the argv it actually execs, so what is pinned
|
||||
# is the command the operator gets.
|
||||
|
||||
load helpers
|
||||
|
||||
@@ -110,10 +112,9 @@ argv_value() {
|
||||
[ "$(argv_value --n-gpu-layers)" -gt 0 ]
|
||||
# llama.cpp divides --ctx-size across slots and a mamba hybrid cannot
|
||||
# share one cache, so the observable contract is the per-slot window.
|
||||
# 2026-10-05 evening (Frank): the 27B Swift-1.5 swap caps each
|
||||
# conversation at 131,072, and ONE slot is the maximum — a second
|
||||
# 131,072 slot does not fit the 2-3 GiB floor (KV budget test below).
|
||||
[ "$(argv_value --parallel)" -eq 1 ]
|
||||
# 2026-10-06 (Frank): drop MTP, drop ubatch, and take TWO conversations
|
||||
# at 131,072 — pool 262144, parallel 2, per-slot 131072 (the cap).
|
||||
[ "$(argv_value --parallel)" -eq 2 ]
|
||||
ctx="$(argv_value --ctx-size)"
|
||||
par="$(argv_value --parallel)"
|
||||
[ $((ctx % par)) -eq 0 ]
|
||||
@@ -134,7 +135,7 @@ argv_value() {
|
||||
[ "$(argv_value --flash-attn)" = "on" ]
|
||||
}
|
||||
|
||||
@test "the KV budget fits the 2-3 GiB floor and forbids a second 128k slot" {
|
||||
@test "the KV budget fits TWO 128k slots (Frank's tight-fit order) and forbids a third" {
|
||||
# Rate, from the Swift-1.5 IQ3_S-mtp GGUF header (read from the Hub
|
||||
# file, 2026-10-05): SIXTEEN full-attention layers (block_count 65 = 64
|
||||
# real layers + the blk.64 MTP head, full_attention_interval 4) x 4 kv
|
||||
@@ -142,18 +143,19 @@ argv_value() {
|
||||
# => 34 KiB/token at q8_0 (budgeted 37 with slack; f16 would be 64).
|
||||
# The non-KV half: IQ3_S-mtp weights 11,561 MiB (12,120,016,896 B) +
|
||||
# vision tower ~1.35 GiB (927,607,488 B F16 mmproj at the measured
|
||||
# ~1.45x cost law) + DeltaNet states + compute buffers at ubatch 1024 +
|
||||
# CUDA graphs + the MTP draft context => 17 GiB budgeted. Two asserts,
|
||||
# both about the output the operator sees: the pool must still fit with
|
||||
# 2 GiB spare, and a SECOND 131,072 slot must not fit — Frank's "as
|
||||
# many 128k contexts as fit" has the answer 1 on this card, and the
|
||||
# pool itself is capped by the 128k law, not by the card.
|
||||
# Arithmetic, not a boot log: change CTX_SIZE and this bites before the
|
||||
# deploy does.
|
||||
# ~1.45x cost law) + compute buffers at ubatch 512 + CUDA graphs + TWO
|
||||
# DeltaNet states (~150 MiB each) and NO draft context (speculation is
|
||||
# off) => 15 GiB budgeted. 2026-10-06 (Frank): two 131,072 slots are
|
||||
# the order — the budget lands AT the card (tight fit, accepted
|
||||
# explicitly; the old 2-3 GiB floor is knowingly violated) and a THIRD
|
||||
# slot must not fit. Arithmetic, not a boot log: change CTX_SIZE and
|
||||
# this bites before the deploy does; the boot's nvidia-smi free is the
|
||||
# measured truth.
|
||||
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
kv_gib=$((ctx * 37 / 1048576))
|
||||
[ $((kv_gib + 17 + 2)) -le 24 ]
|
||||
[ $((2 * kv_gib + 17)) -gt 24 ]
|
||||
third_slot_kv=$(( (ctx + 131072) * 37 / 1048576 ))
|
||||
[ $((kv_gib + 15)) -le 24 ]
|
||||
[ $((third_slot_kv + 15)) -gt 24 ]
|
||||
}
|
||||
|
||||
@test "the compute-buffer pair follows the env file and stays self-consistent" {
|
||||
@@ -183,18 +185,19 @@ argv_value() {
|
||||
argv_value --ubatch-size | grep -qE '^[0-9]+$'
|
||||
}
|
||||
|
||||
@test "the in-file MTP head ships as draft-mtp at depth 3" {
|
||||
# The -mtp GGUF carries the head (blk.64.nextn.*, nextn_predict_layers
|
||||
# =1): draft-mtp loads it against the main model's own weights — no
|
||||
# --model-draft and no second draft KV pool. Speculative flags move as
|
||||
# a set; both must appear in the argv the supervisor runs, and the env
|
||||
# must carry the same contract the argv executes.
|
||||
@test "speculation is off: no half spec pair reaches the argv" {
|
||||
# Frank, 2026-10-06: drop MTP, skip ngram-mod (on a qwen35 hybrid
|
||||
# ngram disables recurrent-state rollback — a rejected draft corrupts
|
||||
# the DeltaNet state). The flags move as a set, so the no-spec contract
|
||||
# is ABSENCE of both in the argv the supervisor runs, and the env must
|
||||
# carry the same contract. The -mtp file's head stays; it is not
|
||||
# drafted with. Reference numbers stay in frank-small.env for the day
|
||||
# MTP returns (44.8 % acceptance at the recipe temp => 58.9 t/s).
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --spec-type)" = "draft-mtp" ]
|
||||
[ "$(argv_value --spec-draft-n-max)" = "3" ]
|
||||
grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE"
|
||||
grep -qE '^SPEC_DRAFT_N_MAX=3$' "$ENV_FILE"
|
||||
! grep -qxF -- '--spec-type' <<<"$output"
|
||||
! grep -qxF -- '--spec-draft-n-max' <<<"$output"
|
||||
grep -qE '^SPEC_TYPE=none$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {
|
||||
|
||||
Reference in New Issue
Block a user