frank-default-small: batch=4096, ubatch=2048

scripts/serve-frank-small.sh

@@ -38,8 +38,8 @@ fi
 : "${VERBOSITY:=4}"
 : "${N_CPU_MOE:=36}"
 : "${N_GPU_LAYERS:=all}"
-: "${BATCH_SIZE:=5120}"
-: "${UBATCH_SIZE:=2560}"
+: "${BATCH_SIZE:=4096}"
+: "${UBATCH_SIZE:=2048}"
 : "${CTX_CHECKPOINTS:=4}"
 : "${NO_MMAP:=1}"
 : "${TEMPERATURE:=1.0}"