frank-default-small: ubatch=1536, ctx-checkpoints=2 (24GB VRAM safety)

2026-08-20 16:51:40 -07:00
parent 87dd4224b7
commit f8deaa798a
2 changed files with 4 additions and 4 deletions
@@ -41,11 +41,11 @@ CACHE_TYPE_V=f16

 VERBOSITY=4
 # Store SWA/context checkpoints in RAM to hold peak GPU memory down.
-CTX_CHECKPOINTS=4
+CTX_CHECKPOINTS=2

-# Big prefill batches (ubatch 1024, batch 4096).
+# Big prefill batches (ubatch 1536, batch 4096).
 BATCH_SIZE=4096
-UBATCH_SIZE=1024
+UBATCH_SIZE=1536

 # Eager RAM load (MoE experts on CPU; ~31 GB RAM).
 NO_MMAP=1
@@ -40,7 +40,7 @@ fi
 : "${N_GPU_LAYERS:=all}"
 : "${BATCH_SIZE:=4096}"
 : "${UBATCH_SIZE:=1024}"
-: "${CTX_CHECKPOINTS:=4}"
+: "${CTX_CHECKPOINTS:=2}"
 : "${NO_MMAP:=1}"
 : "${TEMPERATURE:=1.0}"
 : "${TOP_K:=20}"