From e45f457ad498b47d220effebfac04e5006bb391e Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Tue, 11 Aug 2026 01:33:28 -0700
Subject: [PATCH] perf: frank-default-small batch 3072 / ubatch 1536 +
 ctx-checkpoints

Bigger logical/physical batches for prefill throughput. Enable
--ctx-checkpoints to park attention checkpoints in RAM, holding peak GPU
memory down so the larger ubatch fits the 24GB VRAM alongside 3x262144 KV
without OOM. Weight split unchanged (experts CPU, non-experts GPU).
---
 scripts/frank-default-small.env    | 6 ++++++
 scripts/serve-llama-frank-small.sh | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env
index 652000f..2257cb0 100644
--- a/scripts/frank-default-small.env
+++ b/scripts/frank-default-small.env
@@ -15,5 +15,11 @@ CACHE_TYPE_V=f16
 N_CPU_MOE=40
 # Every non-expert tensor (attention, norms, embedding, LM head) goes to GPU.
 N_GPU_LAYERS=all
+# Larger batches for prefill throughput. The bigger physical batch grows GPU
+# activation buffers; ctx-checkpoints parks attention checkpoints in RAM to
+# hold peak GPU memory down so the larger ubatch fits without OOM.
+BATCH_SIZE=3072
+UBATCH_SIZE=1536
+CTX_CHECKPOINTS=4
 THREADS=8
 THREADS_HTTP=8
\ No newline at end of file
diff --git a/scripts/serve-llama-frank-small.sh b/scripts/serve-llama-frank-small.sh
index 5fe8969..4dcbe9b 100755
--- a/scripts/serve-llama-frank-small.sh
+++ b/scripts/serve-llama-frank-small.sh
@@ -32,6 +32,9 @@ fi
 : "${CACHE_TYPE_V:=f16}"
 : "${N_CPU_MOE:=40}"     # all 40 layers: MoE expert weights stay on CPU
 : "${N_GPU_LAYERS:=all}" # every non-expert tensor goes to GPU
+: "${BATCH_SIZE:=2048}"
+: "${UBATCH_SIZE:=512}"
+: "${CTX_CHECKPOINTS:=0}"
 : "${THREADS:=8}"        # 7700X physical cores
 : "${THREADS_HTTP:=8}"
 
@@ -60,6 +63,9 @@ exec "$LLAMA_SERVER" \
     --parallel "$PARALLEL" \
     --n-cpu-moe "$N_CPU_MOE" \
     --gpu-layers "$N_GPU_LAYERS" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --ctx-checkpoints "$CTX_CHECKPOINTS" \
     --threads "$THREADS" \
     --threads-http "$THREADS_HTTP" \
     --alias "$ALIAS" \