diff --git a/scripts/frank-secondary.env b/scripts/frank-secondary.env index 87f6f14..b26b438 100644 --- a/scripts/frank-secondary.env +++ b/scripts/frank-secondary.env @@ -57,23 +57,22 @@ ALIAS=frank-secondary BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 -# TWO conversations at Frank's 128k cap (2026-10-06): --ctx-size is the -# POOL llama.cpp divides across slots; 262144 / 2 = 131,072 per -# conversation. Frank accepted the tight fit explicitly (arithmetic above); -# the per-slot window is the product contract, so models.yml stays at -# 131,072. -CTX_SIZE=262144 +# TWO conversations at Frank's 112k cap (2026-10-06 evening): --ctx-size is +# the POOL llama.cpp divides across slots; 229376 / 2 = 114,688 per +# conversation. Lowering from 128k (131,072) frees ~1 GiB of KV, which +# funds the ubatch 512 -> 1024 increase (prefill throughput). The +# per-slot window is the product contract, so models.yml follows to +# 114,688. +CTX_SIZE=229376 PARALLEL=2 # Compute-buffer geometry. Ubatch is a prefill knob (decode never moves); -# on the 9B the 1024->2048 rung measured +4.6 % prefill for 616 MiB, and -# Frank's 2026-10-06 order spends the other direction: ubatch 512 claws -# back ~0.3 GiB toward the second slot at some prefill cost. The pair is a -# measurement, not a law: ubatch 1024 is the first A/B back if a future -# boot lands with headroom (loud failure at graph capture if it does not +# the 1024 rung was the first A/B back once the per-slot context dropped +# to 112k (2026-10-06 evening) and freed ~1 GiB of KV. The pair is a +# measurement, not a law (loud failure at graph capture if it does not # fit). BATCH_SIZE=4096 -UBATCH_SIZE=512 +UBATCH_SIZE=1024 # The KV contract for this swap (Frank, 2026-10-05): q8_0 K + q8_0 V — # llama.cpp's FP8 class. The flags stay explicit so the choice is visible in diff --git a/tests/frank_secondary_unit.bats b/tests/frank_secondary_unit.bats index b813345..26f242c 100755 --- a/tests/frank_secondary_unit.bats +++ b/tests/frank_secondary_unit.bats @@ -103,7 +103,7 @@ argv_value() { done } -@test "the pool is exactly the native 262144 window, so each of the two conversations is 131072" { +@test "the pool is 229376, so each of the two conversations is 114688" { serve_argv [ "$status" -eq 0 ] [ "$(argv_value --alias)" = "frank-secondary" ] @@ -113,13 +113,14 @@ argv_value() { [ "$(argv_value --n-gpu-layers)" -gt 0 ] # llama.cpp divides --ctx-size across slots and a mamba hybrid cannot # share one cache, so the observable contract is the per-slot window. - # 2026-10-06 (Frank): drop MTP, drop ubatch, and take TWO conversations - # at 131,072 — pool 262144, parallel 2, per-slot 131072 (the cap). + # 2026-10-06 evening (Frank): lower per-slot to 114,688 (112k) to fund + # the ubatch 512 -> 1024 increase — pool 229376, parallel 2, per-slot + # 114688 (the cap). [ "$(argv_value --parallel)" -eq 2 ] ctx="$(argv_value --ctx-size)" par="$(argv_value --parallel)" [ $((ctx % par)) -eq 0 ] - [ $((ctx / par)) -eq 131072 ] + [ $((ctx / par)) -eq 114688 ] } @test "the KV contract is q8_0 both halves (FP8 class), flash-attn on" { @@ -136,7 +137,7 @@ argv_value() { [ "$(argv_value --flash-attn)" = "on" ] } -@test "the KV budget fits two 131072 slots at q8_0 and forbids a third" { +@test "the KV budget fits two 114688 slots at q8_0 and forbids a third" { # Rate, from the RCO-IQ3_XXS GGUF header (same qwen35 architecture): # SIXTEEN full-attention layers (block_count 65 = 64 real layers + the # blk.64 MTP head, full_attention_interval 4) x 4 kv heads x 256 dim x @@ -145,15 +146,15 @@ argv_value() { # RCO-IQ3_XXS weights 10,148 MiB (10,442,827,392 B) — 1.56 GiB LIGHTER # than the IQ3_S-mtp predecessor — + vision tower ~1.4 GiB (931,145,888 # B BF16 mmproj at the measured ~1.45x cost law) + compute buffers at - # ubatch 512 + CUDA graphs + TWO DeltaNet states (~150 MiB each) + the - # MTP draft ctx => 14 GiB budgeted. Two 131,072 slots are the - # 2026-10-06 shape (the lighter quant funds the draft ctx plus the - # second slot); a THIRD slot must not fit. Arithmetic, not a boot log: + # ubatch 1024 + CUDA graphs + TWO DeltaNet states (~150 MiB each) + the + # MTP draft ctx => 14 GiB budgeted. Two 114,688 slots are the + # 2026-10-06 evening shape (the lighter per-slot funds the ubatch + # increase); a THIRD slot must not fit. Arithmetic, not a boot log: # change CTX_SIZE and this bites before the deploy does; the boot's # nvidia-smi free is the measured truth. ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" kv_gib=$((ctx * 37 / 1048576)) - third_slot_kv=$(( (ctx + 131072) * 37 / 1048576 )) + third_slot_kv=$(( (ctx + 114688) * 37 / 1048576 )) [ $((kv_gib + 14)) -le 24 ] [ $((third_slot_kv + 14)) -gt 24 ] }