frank-default-small: batch=5120, ubatch=2560

2026-08-20 15:38:18 -07:00
parent 635e074af8
commit e777d3cb56
2 changed files with 5 additions and 5 deletions
@@ -43,9 +43,9 @@ VERBOSITY=4
 # Store SWA/context checkpoints in RAM to hold peak GPU memory down.
 CTX_CHECKPOINTS=4

-# Big prefill batches (ubatch 3072 is the top prefill tier).
-BATCH_SIZE=6144
-UBATCH_SIZE=3072
+# Big prefill batches (ubatch 2560, batch 5120).
+BATCH_SIZE=5120
+UBATCH_SIZE=2560

 # Eager RAM load (MoE experts on CPU; ~31 GB RAM).
 NO_MMAP=1
@@ -38,8 +38,8 @@ fi
 : "${VERBOSITY:=4}"
 : "${N_CPU_MOE:=36}"
 : "${N_GPU_LAYERS:=all}"
-: "${BATCH_SIZE:=6144}"
-: "${UBATCH_SIZE:=3072}"
+: "${BATCH_SIZE:=5120}"
+: "${UBATCH_SIZE:=2560}"
 : "${CTX_CHECKPOINTS:=4}"
 : "${NO_MMAP:=1}"
 : "${TEMPERATURE:=1.0}"