diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 6e39873..d9a5758 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -41,11 +41,11 @@ CACHE_TYPE_V=f16 VERBOSITY=4 # Store SWA/context checkpoints in RAM to hold peak GPU memory down. -CTX_CHECKPOINTS=4 +CTX_CHECKPOINTS=2 -# Big prefill batches (ubatch 1024, batch 4096). +# Big prefill batches (ubatch 1536, batch 4096). BATCH_SIZE=4096 -UBATCH_SIZE=1024 +UBATCH_SIZE=1536 # Eager RAM load (MoE experts on CPU; ~31 GB RAM). NO_MMAP=1 diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index a8d11bc..5d29f20 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -40,7 +40,7 @@ fi : "${N_GPU_LAYERS:=all}" : "${BATCH_SIZE:=4096}" : "${UBATCH_SIZE:=1024}" -: "${CTX_CHECKPOINTS:=4}" +: "${CTX_CHECKPOINTS:=2}" : "${NO_MMAP:=1}" : "${TEMPERATURE:=1.0}" : "${TOP_K:=20}"