diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 6785c01..074e0e4 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -48,8 +48,8 @@ LISTEN_PORT=52625 # budget. KV at 262144 x 64 KiB/cell is ~16 GiB. CTX_SIZE=262144 PARALLEL=1 -BATCH_SIZE=8192 -UBATCH_SIZE=8192 +BATCH_SIZE=4096 +UBATCH_SIZE=4096 CACHE_TYPE_K=f16 CACHE_TYPE_V=f16 CTX_CHECKPOINTS=16 diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 60570c6..540d449 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -39,8 +39,8 @@ fi : "${LISTEN_PORT:=52625}" : "${CTX_SIZE:=524288}" : "${PARALLEL:=2}" -: "${BATCH_SIZE:=2048}" -: "${UBATCH_SIZE:=2048}" +: "${BATCH_SIZE:=4096}" +: "${UBATCH_SIZE:=4096}" : "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_V:=f16}" : "${CTX_CHECKPOINTS:=16}"