diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 995ee6e..6e39873 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -43,9 +43,9 @@ VERBOSITY=4 # Store SWA/context checkpoints in RAM to hold peak GPU memory down. CTX_CHECKPOINTS=4 -# Big prefill batches (ubatch 2048, batch 4096). +# Big prefill batches (ubatch 1024, batch 4096). BATCH_SIZE=4096 -UBATCH_SIZE=2048 +UBATCH_SIZE=1024 # Eager RAM load (MoE experts on CPU; ~31 GB RAM). NO_MMAP=1 diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 85a0dc2..a8d11bc 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -39,7 +39,7 @@ fi : "${N_CPU_MOE:=36}" : "${N_GPU_LAYERS:=all}" : "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=2048}" +: "${UBATCH_SIZE:=1024}" : "${CTX_CHECKPOINTS:=4}" : "${NO_MMAP:=1}" : "${TEMPERATURE:=1.0}"