diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 7d91383..31823c0 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -43,9 +43,9 @@ VERBOSITY=4 # Store SWA/context checkpoints in RAM to hold peak GPU memory down. CTX_CHECKPOINTS=4 -# Big prefill batches (ubatch 3072 is the top prefill tier). -BATCH_SIZE=6144 -UBATCH_SIZE=3072 +# Big prefill batches (ubatch 2560, batch 5120). +BATCH_SIZE=5120 +UBATCH_SIZE=2560 # Eager RAM load (MoE experts on CPU; ~31 GB RAM). NO_MMAP=1 diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 7e28332..c743792 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -38,8 +38,8 @@ fi : "${VERBOSITY:=4}" : "${N_CPU_MOE:=36}" : "${N_GPU_LAYERS:=all}" -: "${BATCH_SIZE:=6144}" -: "${UBATCH_SIZE:=3072}" +: "${BATCH_SIZE:=5120}" +: "${UBATCH_SIZE:=2560}" : "${CTX_CHECKPOINTS:=4}" : "${NO_MMAP:=1}" : "${TEMPERATURE:=1.0}"