From 24eaba1078d2bc55f18691efe8c7da2b82956b5c Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Thu, 17 Sep 2026 04:28:51 -0700
Subject: [PATCH] =?UTF-8?q?frank-small:=20batch=208192/1024=20=E2=80=94=20?=
 =?UTF-8?q?the=20measured=20max-length=20winner?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Swept on the 3090 at the deployed 4x262144 geometry, salted API probes:
- ubatch 512: pp@254k ~918 t/s, tg@250k 90-204 (noisy)
- ubatch 1024: pp@254k 962 t/s (+4.8%), tg@250k 257-302
- ubatch 2048: OOM — the MTP draft context needs ~3.3 GiB of pp compute
  buffer against 2.2 GiB headroom; do not try.
- draft depth 4 + poll 100 decoded WORSE at 250k (230-257) than 3/50:
  the longer verification round re-reads more long-ctx attention than the
  extra accepted token earns. Kept 3/50, now with the evidence attached.
Idle VRAM 23,300 MiB; all ubatch-1024 variants idled identically
(compute buffers reserved at max graph size), so 8192 buys the deepest
prefill window for free over 4096.
---
 scripts/frank-small.env | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/scripts/frank-small.env b/scripts/frank-small.env
index acdcc31..dd23504 100644
--- a/scripts/frank-small.env
+++ b/scripts/frank-small.env
@@ -75,24 +75,27 @@ PARALLEL=4
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=q8_0
 
-# Batch geometry: llama.cpp's own defaults (2048 logical / 512 physical).
-BATCH_SIZE=2048
-UBATCH_SIZE=512
+# Batch geometry: 8192 logical / 1024 physical. Measured on this card at
+# max length (254k-token prefill / 253k-ctx decode, 4 slots, q8 KV):
+# ubatch 512 -> pp 915 t/s, tg@250k 90-204; ubatch 1024 -> pp 962,
+# tg 257-302. Larger ubatches (2048) OOM the MTP draft context's pp
+# compute buffer (~3.3 GiB against 2.2 GiB headroom) and must not be
+# tried. Idle VRAM is 23,300 MiB at this geometry.
+BATCH_SIZE=8192
+UBATCH_SIZE=1024
 
 # CPU threads: this service runs entirely on the GPU (NGL=999), so threads
 # cover tokenization, sampling and the HTTP layer only. Deliberately modest
 # -- this host's CPU is shared with halogen's engine, which is CPU-hungry.
 NUM_THREADS=8
 THREAD_BATCH=8
-
-# Every layer on the 3090, with flash attention (required for q8 KV).
-NGL=999
-FLASH_ATTN=on
-
 # Speculative decoding: the MTP sidecar (draft context) chained with the
 # draftless ngram-mod matcher. Draft depth and busy-wait level are made
-# explicit here for tuning (both were the implicit build defaults: 3 and 50;
-# the Gemma4 sidecar carries 4 nextn layers).
+# explicit here (both were the implicit build defaults: 3 and 50).
+# Measured at 250k ctx: depth 4 + poll 100 DECODED WORSE (230-257 t/s)
+# than depth 3 + poll 50 (257-302 t/s) — the longer verification round
+# costs more attention re-read at long context than the extra accepted
+# token earns. Keep 3/50.
 SPEC_TYPE=draft-mtp,ngram-mod
 SPEC_DRAFT_N_MAX=3
 POLL_LEVEL=50