diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 57636a1..175e400 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -48,8 +48,8 @@ LISTEN_PORT=52625 # budget. KV at 262144 x 64 KiB/cell is ~16 GiB. CTX_SIZE=262144 PARALLEL=1 -BATCH_SIZE=16384 -UBATCH_SIZE=8192 +BATCH_SIZE=8192 +UBATCH_SIZE=4096 CACHE_TYPE_K=f16 CACHE_TYPE_V=f16 CTX_CHECKPOINTS=16