From 7fe881de016f211a6a7713603c0c4d99565eb80b Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 2 Oct 2026 03:17:35 -0700 Subject: [PATCH] frank-small: pool 540672 (3 x 180224) so the card keeps ~2.7 GiB free Measured after the first 3-slot deploy: 3 x 196,608 booted (n_slots=3, n_ctx_slot=196608, multimodal loaded) but used 22,972/24,576 MiB - 1,153 MiB free, outside the 2-3 GiB band. The two measurements now recorded in frank-small.env price the vision tower at ~1.4 GiB (non-KV 7,390 -> 8,859 MiB), so the pool drops 49,152 tokens (16,384/slot, 1,176 MiB) to land ~2.7 GiB free. The bats budget gate is recalibrated to the measured 9 GiB non-KV cost, which now rejects the 3 x 196,608 shape instead of blessing it. --- scripts/frank-small.env | 40 ++++++++++++++++++++---------------- scripts/frank-small.service | 2 +- scripts/serve-frank-small.sh | 10 ++++----- tests/frank_small_unit.bats | 29 ++++++++++++++------------ 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/scripts/frank-small.env b/scripts/frank-small.env index 872504d..c30377d 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -5,20 +5,23 @@ # leave 2-3 GiB of the card free. Both new demands cost memory, so the per-slot # window shrinks — see the budget below. # -# Measured on this card, this runtime, 2 x 262,144 (2026-10-02, steady state): -# 20,190 MiB used of 24,576. KV at that pool was 524,288 tokens x 24.5 KiB = -# 12,800 MiB, so everything that is NOT KV — weights 6,557 + Gated DeltaNet -# state ~25/slot + compute buffers and CUDA graphs ~800 — is 7,390 MiB. +# Two measurements on this card, this runtime, price the terms: +# 2 x 262,144, no projector: 20,190 MiB used / 4,386 free +# KV = 524,288 tok x 24.5 KiB = 12,800 MiB => non-KV (weights 6,557 + +# DeltaNet state ~25/slot + compute buffers + CUDA graphs) = 7,390 MiB +# 3 x 196,608, projector on: 22,972 MiB used / 1,153 free +# KV = 14,113 MiB => non-KV WITH the vision tower = 8,859 MiB +# The tower costs ~1.4 GiB, not the ~1.0 its file size suggested, so 196,608 +# per slot left only 1.1 GiB free — outside the band. The pool therefore is +# 3 x 180,224 = 540,672 tokens: KV ~12,936 + non-KV 8,859 = ~21,800 MiB used, +# ~2.7 GiB free, the middle of what was asked for. # -# KV arithmetic (unchanged): attention KV exists only in the 8 full-attention -# layers (4 KV heads x 256 head_dim). K f16 = 16 KiB/token, V q8_0 = -# 8.5 KiB/token (34 B per 32-element block) => 24.5 KiB/token. Three slots at -# 196,608 = 589,824 tokens cost 14,113 MiB. Add the non-KV 7,390, a third -# DeltaNet state (~25) and the vision tower + its graph (~1,030: 876 MiB of -# mmproj weights plus buffer) => ~22.5 GiB used, ~2.0 GiB free — the bottom of -# the requested band. Step CTX_SIZE down by 12,288 (=4096/slot) if measurement -# says tighter, up if it says looser; the native 262,144 per slot is NOT -# reachable three times (3 x 262,144 x 24.5 KiB = 18.4 GiB of KV alone). +# KV arithmetic: attention KV exists only in the 8 full-attention layers +# (4 KV heads x 256 head_dim). K f16 = 16 KiB/token, V q8_0 = 8.5 KiB/token +# (34 B per 32-element block) => 24.5 KiB/token, so 12,288 tokens (=4096/slot) +# is one 293 MiB step — that is the ladder to walk if this moves again. The +# native 262,144 per slot is NOT reachable three times: 3 x 262,144 x +# 24.5 KiB = 18.4 GiB of KV before anything else is loaded. # # Why V stays q8_0 and K stays f16: full-precision V would add 8 KiB/token — # +4.5 GiB on this pool — which is what pushes the card past its 24 GiB. K is @@ -40,11 +43,12 @@ LISTEN_PORT=7771 # Three independent conversations. llama.cpp divides --ctx-size across # --parallel slots (n_ctx_slot = CTX_SIZE/PARALLEL), and a mamba-hybrid model # cannot use unified KV, so each slot gets its own cache and its own window. -# 196,608 per slot is BELOW the native 262,144: a single request longer than -# ~196k tokens now needs compaction instead of one shot (the 2026-10-01 -# 2 x 262,144 shape was chosen to hold a 216,269-token compaction request; -# three contexts is the trade Frank asked for). -CTX_SIZE=589824 +# 180,224 per slot is BELOW the native 262,144, and that is the cost of the +# third slot: a longer single prompt needs compaction instead of one shot. +# Consequence worth knowing: frank-primary's compaction trigger is 183,501 +# tokens, which no longer fits a frank-small slot — see the runbook's +# "three slots vs compaction" note. +CTX_SIZE=540672 PARALLEL=3 # The KV contract (Frank, 2026-10-01 night): unquantized K, q8_0 V. diff --git a/scripts/frank-small.service b/scripts/frank-small.service index 5834475..f8a242c 100644 --- a/scripts/frank-small.service +++ b/scripts/frank-small.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q5_K_M GGUF (qwen35 hybrid, 3x196608, f16 K + q8_0 V, mmproj vision) on the RTX 3090 (dankclaw, frank-small) +Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q5_K_M GGUF (qwen35 hybrid, 3x180224, f16 K + q8_0 V, mmproj vision) on the RTX 3090 (dankclaw, frank-small) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 637b232..300b55b 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -37,11 +37,11 @@ fi : "${ALIAS:=frank-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" -# ctx-size is the POOL llama.cpp divides across slots: 589824 / 3 gives -# 196608 per slot — three independent conversations, each with a 192k window -# (the native 262144 no longer fits three times on a 24 GiB card; see -# frank-small.env for the arithmetic that lands ~2.5 GiB of card free). -: "${CTX_SIZE:=589824}" +# ctx-size is the POOL llama.cpp divides across slots: 540672 / 3 gives +# 180224 per slot — three independent conversations (the native 262144 no +# longer fits three times on a 24 GiB card once the vision tower is loaded; +# frank-small.env carries the two measurements that set this number). +: "${CTX_SIZE:=540672}" : "${PARALLEL:=3}" : "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_V:=q8_0}" diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index b2c95fd..da65a8b 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -98,7 +98,7 @@ argv_value() { done } -@test "the shipped env serves the fleet shape: three 196608-token contexts" { +@test "the shipped env serves the fleet shape: three 180224-token contexts" { serve_argv [ "$status" -eq 0 ] [ "$(argv_value --alias)" = "frank-small" ] @@ -112,7 +112,7 @@ argv_value() { ctx="$(argv_value --ctx-size)" par="$(argv_value --parallel)" [ $((ctx % par)) -eq 0 ] - [ $((ctx / par)) -eq 196608 ] + [ $((ctx / par)) -eq 180224 ] [ $((ctx / par)) -le 262144 ] } @@ -128,20 +128,23 @@ argv_value() { @test "the KV budget leaves 2-3 GiB of the 3090 free" { # Rate: K f16 (16 KiB/token) + V q8_0 (~8.5, budgeted 25 with slack). - # The non-KV half is MEASURED on this card on 2026-10-02, not guessed: - # weights 6.4 + projector 0.85 + compute buffers/CUDA graphs ~0.8 => 8. - # Two asserts, both about output the operator sees: the pool must still - # fit with 2 GiB spare, and it must be large enough that 5 GiB spare is - # NOT reachable — an undersized pool is as wrong as an oversized one. - # Arithmetic, not a boot log: change CTX_SIZE and this bites before the - # deploy does. The same pool with f16 V (34 KiB/token) lands past the - # card, which is why the contract is asymmetric. + # The non-KV half is MEASURED on this card on 2026-10-02: with the + # projector loaded the engine reported 22,972 MiB used against 14,113 + # MiB of KV, so weights + vision tower + DeltaNet state + compute + # buffers + CUDA graphs = 8,859 MiB => 9 GiB. Two asserts, both about + # the output the operator sees: the pool must still fit with 2 GiB + # spare, and it must be large enough that 5 GiB spare is NOT reachable + # (an undersized pool is as wrong as an oversized one). Arithmetic, not + # a boot log: change CTX_SIZE and this bites before the deploy does — + # it rejects the 3 x 196,608 pool that measured only 1.1 GiB free. The + # same pool with f16 V (34 KiB/token) lands past the card, which is why + # the contract is asymmetric. ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" kv_gib=$((ctx * 25 / 1048576)) - [ $((kv_gib + 8 + 2)) -le 24 ] - [ $((kv_gib + 8 + 5)) -gt 24 ] + [ $((kv_gib + 9 + 2)) -le 24 ] + [ $((kv_gib + 9 + 5)) -gt 24 ] f16v_gib=$((ctx * 34 / 1048576)) - [ $((f16v_gib + 8)) -gt 24 ] + [ $((f16v_gib + 9)) -gt 24 ] } @test "no speculative draft ships by default" {