From 668083fe761e38ba5dd83c9b0ec943154e5fd6d3 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Tue, 11 Aug 2026 17:19:57 -0700
Subject: [PATCH] perf: frank-default-large ubatch 512 (compute graph scales
 with ubatch; 602100 ctx KV leaves ~16GB, ubatch 2048 OOM)

---
 scripts/frank-default-large.env    | 9 +++++----
 scripts/serve-llama-frank-large.sh | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env
index 6de99fd..c2c3b1c 100644
--- a/scripts/frank-default-large.env
+++ b/scripts/frank-default-large.env
@@ -24,10 +24,11 @@ N_GPU_LAYERS=all
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=f16
 # Batch sizing constrained by ROCm memory at 602100 ctx (3x200700):
-# KV cache alone ~50GB, model 29.77GB; only ~16GB left for compute graphs.
-# Batch 12288 (80.7GB buffer) and 6144 (40.9GB) both OOM; 2048 fits.
-# Tried per request (ubatch 4096 / batch 12288) -> OOM. See bench notes.
+# KV ~50GB + model 29.77GB leave only ~16GB for compute graphs. The compute
+# graph scales with UBATCH, so ubatch must stay ~512; ubatch 2048 OOMs even
+# at batch 2048. Proven-fits config: batch 2048 / ubatch 512.
+# User-specified ubatch 4096 / batch 12288 is not feasible here (see bench).
 BATCH_SIZE=2048
-UBATCH_SIZE=2048
+UBATCH_SIZE=512
 THREADS=8
 THREADS_HTTP=8
diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh
index f447653..a008861 100755
--- a/scripts/serve-llama-frank-large.sh
+++ b/scripts/serve-llama-frank-large.sh
@@ -36,7 +36,7 @@ fi
 : "${CACHE_TYPE_K:=q8_0}"
 : "${CACHE_TYPE_V:=f16}"
 : "${BATCH_SIZE:=2048}"
-: "${UBATCH_SIZE:=2048}"
+: "${UBATCH_SIZE:=512}"
 : "${THREADS:=8}"
 : "${THREADS_HTTP:=8}"