perf: frank-default-large batch capped at 2048/2048 — 3x200700 ctx KV ~50GB leaves no room for batch 12288 (OOM)

scripts/frank-default-large.env

@@ -23,7 +23,11 @@ N_GPU_LAYERS=all
 # KV on GPU, quantized (K q8_0, V f16).
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=f16
-BATCH_SIZE=12288
+# Batch sizing constrained by ROCm memory at 602100 ctx (3x200700):
-UBATCH_SIZE=4096
+# KV cache alone ~50GB, model 29.77GB; only ~16GB left for compute graphs.
+# Batch 12288 (80.7GB buffer) and 6144 (40.9GB) both OOM; 2048 fits.
+# Tried per request (ubatch 4096 / batch 12288) -> OOM. See bench notes.
+BATCH_SIZE=2048
+UBATCH_SIZE=2048
 THREADS=8
 THREADS_HTTP=8

scripts/serve-llama-frank-large.sh

@@ -35,8 +35,8 @@ fi
 : "${N_GPU_LAYERS:=all}"
 : "${CACHE_TYPE_K:=q8_0}"
 : "${CACHE_TYPE_V:=f16}"
-: "${BATCH_SIZE:=12288}"
+: "${BATCH_SIZE:=2048}"
-: "${UBATCH_SIZE:=4096}"
+: "${UBATCH_SIZE:=2048}"
 : "${THREADS:=8}"
 : "${THREADS_HTTP:=8}"