From 317610a5403285af3902f81238eb91bae825d565 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Thu, 13 Aug 2026 18:56:12 -0700
Subject: [PATCH] perf: raise frank-default-large to batch 8192 / ubatch 2048

Empirically verified on 2026-08-13 (caches dropped): batch 8192 / ubatch
2048 loads and serves at ctx 602100 with 61 GiB VRAM peak. The benchmark
doc's 'ubatch 2048 OOMs' results were confounded by page-cache pressure -
with the model's pages fully cached the KV/graph allocation stalls at
28 GiB VRAM. Drop caches (sync; echo 3 > /proc/sys/vm/drop_caches) before
starting a load to make restarts reliable.
---
 scripts/frank-default-large.env    | 15 ++++++++-------
 scripts/serve-llama-frank-large.sh |  4 ++--
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env
index 56b22fd..cb117d7 100644
--- a/scripts/frank-default-large.env
+++ b/scripts/frank-default-large.env
@@ -26,13 +26,14 @@ N_GPU_LAYERS=all
 # long-context decode to ~1 t/s at 200k ctx. q8_0 V halves V memory too.
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=q8_0
-# Batch sizing constrained by ROCm memory at 602100 ctx (3x200700):
-# KV ~50GB + model 29.77GB leave only ~16GB for compute graphs. The compute
-# graph scales with UBATCH, so ubatch must stay ~512; ubatch 2048 OOMs even
-# at batch 2048. Proven-fits config: batch 2048 / ubatch 512.
-# User-specified ubatch 4096 / batch 12288 is not feasible here (see bench).
-BATCH_SIZE=2048
-UBATCH_SIZE=512
+# Batch sizing: empirically verified 2026-08-13 with caches dropped.
+# batch 8192 / ubatch 2048 loads and serves (61 GiB VRAM peak, ctx 602100).
+# The earlier "ubatch 2048 OOMs" notes in frank-default-large-benchmark.md
+# were confounded by page-cache pressure: with the model's pages fully
+# cached, the KV/graph allocation stalls (VRAM stuck at 28 GiB).
+# Drop caches before starting a load: sync && echo 3 > /proc/sys/vm/drop_caches.
+BATCH_SIZE=8192
+UBATCH_SIZE=2048
 # Sampling settings — per HauhauCS Qwen3.6-27B model card.
 # Thinking-general: temp=1.0, top_p=0.95, top_k=20, min_p=0.
 # Clients may override per-request; these are the server-side defaults.
diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh
index b7645d0..ab459bc 100755
--- a/scripts/serve-llama-frank-large.sh
+++ b/scripts/serve-llama-frank-large.sh
@@ -35,8 +35,8 @@ fi
 : "${N_GPU_LAYERS:=all}"
 : "${CACHE_TYPE_K:=q8_0}"
 : "${CACHE_TYPE_V:=q8_0}"
-: "${BATCH_SIZE:=2048}"
-: "${UBATCH_SIZE:=512}"
+: "${BATCH_SIZE:=8192}"
+: "${UBATCH_SIZE:=2048}"
 : "${TEMPERATURE:=1.0}"
 : "${TOP_K:=20}"
 : "${TOP_P:=0.95}"