From 62cc2b850a528b9f5a0d8a6f2558be2a5ff4e977 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Tue, 11 Aug 2026 17:11:04 -0700
Subject: [PATCH] =?UTF-8?q?perf:=20frank-default-large=20batch=20capped=20?=
 =?UTF-8?q?at=202048/2048=20=E2=80=94=203x200700=20ctx=20KV=20~50GB=20leav?=
 =?UTF-8?q?es=20no=20room=20for=20batch=2012288=20(OOM)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/frank-default-large.env    | 8 ++++++--
 scripts/serve-llama-frank-large.sh | 4 ++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env
index 59b936a..6de99fd 100644
--- a/scripts/frank-default-large.env
+++ b/scripts/frank-default-large.env
@@ -23,7 +23,11 @@ N_GPU_LAYERS=all
 # KV on GPU, quantized (K q8_0, V f16).
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=f16
-BATCH_SIZE=12288
-UBATCH_SIZE=4096
+# Batch sizing constrained by ROCm memory at 602100 ctx (3x200700):
+# KV cache alone ~50GB, model 29.77GB; only ~16GB left for compute graphs.
+# Batch 12288 (80.7GB buffer) and 6144 (40.9GB) both OOM; 2048 fits.
+# Tried per request (ubatch 4096 / batch 12288) -> OOM. See bench notes.
+BATCH_SIZE=2048
+UBATCH_SIZE=2048
 THREADS=8
 THREADS_HTTP=8
diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh
index 48a3bf2..f447653 100755
--- a/scripts/serve-llama-frank-large.sh
+++ b/scripts/serve-llama-frank-large.sh
@@ -35,8 +35,8 @@ fi
 : "${N_GPU_LAYERS:=all}"
 : "${CACHE_TYPE_K:=q8_0}"
 : "${CACHE_TYPE_V:=f16}"
-: "${BATCH_SIZE:=12288}"
-: "${UBATCH_SIZE:=4096}"
+: "${BATCH_SIZE:=2048}"
+: "${UBATCH_SIZE:=2048}"
 : "${THREADS:=8}"
 : "${THREADS_HTTP:=8}"