frank-default-large: batch=4096, ubatch=4096 (OOM fix)

2026-08-20 19:58:45 -07:00
parent e09733ee62
commit cd6202f4a0
2 changed files with 4 additions and 4 deletions
@@ -48,8 +48,8 @@ LISTEN_PORT=52625
 # budget. KV at 262144 x 64 KiB/cell is ~16 GiB.
 CTX_SIZE=262144
 PARALLEL=1
-BATCH_SIZE=8192
-UBATCH_SIZE=8192
+BATCH_SIZE=4096
+UBATCH_SIZE=4096
 CACHE_TYPE_K=f16
 CACHE_TYPE_V=f16
 CTX_CHECKPOINTS=16
@@ -39,8 +39,8 @@ fi
 : "${LISTEN_PORT:=52625}"
 : "${CTX_SIZE:=524288}"
 : "${PARALLEL:=2}"
-: "${BATCH_SIZE:=2048}"
-: "${UBATCH_SIZE:=2048}"
+: "${BATCH_SIZE:=4096}"
+: "${UBATCH_SIZE:=4096}"
 : "${CACHE_TYPE_K:=f16}"
 : "${CACHE_TYPE_V:=f16}"
 : "${CTX_CHECKPOINTS:=16}"