frank-default-small: ubatch=1024 (tight 24GB VRAM, 786k ctx pool)

2026-08-20 16:45:20 -07:00
parent 26b42d2d6e
commit 87dd4224b7
2 changed files with 3 additions and 3 deletions
@@ -43,9 +43,9 @@ VERBOSITY=4
 # Store SWA/context checkpoints in RAM to hold peak GPU memory down.
 CTX_CHECKPOINTS=4

-# Big prefill batches (ubatch 2048, batch 4096).
+# Big prefill batches (ubatch 1024, batch 4096).
 BATCH_SIZE=4096
-UBATCH_SIZE=2048
+UBATCH_SIZE=1024

 # Eager RAM load (MoE experts on CPU; ~31 GB RAM).
 NO_MMAP=1
@@ -39,7 +39,7 @@ fi
 : "${N_CPU_MOE:=36}"
 : "${N_GPU_LAYERS:=all}"
 : "${BATCH_SIZE:=4096}"
-: "${UBATCH_SIZE:=2048}"
+: "${UBATCH_SIZE:=1024}"
 : "${CTX_CHECKPOINTS:=4}"
 : "${NO_MMAP:=1}"
 : "${TEMPERATURE:=1.0}"