diff --git a/scripts/frank-small.env b/scripts/frank-small.env index 7c28b51..2fa7bd9 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -72,12 +72,32 @@ CTX_SIZE=401408 PARALLEL=2 # Compute-buffer geometry. This endpoint passed neither flag until 2026-10-03, -# so it ran on llama.cpp's own defaults (2048/512); the fleet now asks for a -# 1024 ubatch on this shape. Both are tunables because the pair is a -# measurement, not a law: on the 27B sibling 512 was the prefill peak and every -# step up cost GiB of compute buffer for no speed. -BATCH_SIZE=2048 -UBATCH_SIZE=1024 +# so it ran on llama.cpp's own defaults (2048/512); the pair is now 4096/2048. +# Both are tunables because the pair is a measurement, not a law — and the +# measurement is CARD-SPECIFIC. On the 2x R9700 layer-split sibling the ubatch +# curve peaks at 512 and every step up costs GiB of compute buffer for no +# speed (see frank-medium-batch-ubatch). On THIS CUDA 3090 the curve climbs +# monotonically to 2048: 710 -> 1188 t/s prefill from ubatch 1024 to 2048 on +# the 35B MoE sweep, 1944 -> 2423 t/s on the Ornith 8k sweep, with decode flat +# to under 1 t/s on every rung. Ubatch is a prefill knob; decode never moves. +# +# The logical batch is free — two independent sweeps agree that at a fixed +# ubatch, 2048/4096/8192 give identical prefill speed AND identical peak VRAM, +# because the compute buffers are sized by the ubatch alone. Doubling it costs +# nothing and buys nothing; it is here because it is the value the pair was +# measured at. +# +# THE RISK IS ALL IN THE UBATCH. The 1024 -> 2048 step cost ~1,244 MiB of peak +# graph VRAM on the 35B MoE (7,330 -> 8,574 MiB); this 9B hybrid has far +# smaller compute tensors, so the delta should be well under that, but the +# predecessor Q5_K_M shape recorded "ubatch 2048 failed the scratch boot" at +# ~3,571 MiB free. This shape ships with LESS free (2,815 MiB measured +# 2026-10-05). Compute buffers are allocated at graph capture during BOOT, so +# a failure is loud and fast, not a surprise at the first long prompt. MEASURE +# nvidia-smi free immediately after the deploy: under ~1.5 GiB, or a boot +# failure, step down to 4096/1536 (or 4096/1024, the last known-good rung). +BATCH_SIZE=4096 +UBATCH_SIZE=2048 # The KV contract (Frank, 2026-10-03): unquantized K AND V. --flash-attn stays # pinned on in the serve script — it is what made the q8_0-V path cheap and it diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 7e7f907..4d89c93 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -45,10 +45,13 @@ fi : "${CTX_SIZE:=401408}" : "${PARALLEL:=2}" # Compute-buffer geometry. Before 2026-10-03 this endpoint never passed either -# flag, so it ran on llama.cpp's own defaults (2048/512); they are tunables now -# because the 1024 ubatch is what this shape was booted with. -: "${BATCH_SIZE:=2048}" -: "${UBATCH_SIZE:=1024}" +# flag, so it ran on llama.cpp's own defaults (2048/512). The pair is 4096/2048 +# since 2026-10-05: on this CUDA card the prefill curve climbs all the way to +# ubatch 2048 (unlike the R9700 layer-split sibling, where 512 is the peak), +# and the logical batch is VRAM- and throughput-neutral. frank-small.env +# carries the sweep tables and the VRAM risk note. +: "${BATCH_SIZE:=4096}" +: "${UBATCH_SIZE:=2048}" : "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_V:=f16}" : "${NGL:=99}" diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 0b7817c..e9e3a9c 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -52,6 +52,8 @@ BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 CTX_SIZE=$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE") PARALLEL=$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE") +BATCH_SIZE=$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE") +UBATCH_SIZE=$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE") CACHE_TYPE_K=$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE") CACHE_TYPE_V=$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE") NGL=$(sed -nE 's/^NGL=([0-9]+)$/\1/p' "$ENV_FILE") @@ -150,14 +152,31 @@ argv_value() { [ $((native * 33 / 1048576 + 9)) -gt 24 ] } -@test "the compute-buffer pair is explicit" { - # This endpoint ran on llama.cpp's own defaults until 2026-10-03; the - # fleet now pins 2048/1024 so the number a benchmark attributes the - # prefill rate to is a repo value, not an engine default nobody wrote down. +@test "the compute-buffer pair follows the env file and stays self-consistent" { + # This endpoint ran on llama.cpp's own defaults until 2026-10-03, so the + # contract is: whatever rung the repo carries is the rung the engine is + # exec'd with — the number a benchmark attributes a prefill rate to must be + # a repo value, never an engine default nobody wrote down. Asserted against + # the env rather than a literal so a retune moves one file, not two. + local env_batch env_ubatch + env_batch="$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" + env_ubatch="$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" serve_argv [ "$status" -eq 0 ] - [ "$(argv_value --batch-size)" = "2048" ] - [ "$(argv_value --ubatch-size)" = "1024" ] + [ "$(argv_value --batch-size)" = "$env_batch" ] + [ "$(argv_value --ubatch-size)" = "$env_ubatch" ] + # llama.cpp refuses ubatch > batch, so a retune that inverts the pair has + # to fail here instead of at boot on a live endpoint. + [ "$env_batch" -ge "$env_ubatch" ] + + # And an env that names neither flag still ships an explicit pair: the + # serve script's fallbacks are written-down values, not blanks the engine + # fills in silently. + sed -i '/^BATCH_SIZE=/d;/^UBATCH_SIZE=/d' "$STAGE/test.env" + serve_argv + [ "$status" -eq 0 ] + argv_value --batch-size | grep -qE '^[0-9]+$' + argv_value --ubatch-size | grep -qE '^[0-9]+$' } @test "no speculative draft ships by default" {