dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-default-large: batch=4096, ubatch=4096 (OOM fix)

Browse Source
This commit is contained in:
Frank committed 2026-08-20 19:58:45 -07:00
1 parent e09733ee62
commit cd6202f4a0
2 files changed
+4 -4
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/frank-default-large.env
+2 -2
View File
@@ -48,8 +48,8 @@ LISTEN_PORT=52625
# budget. KV at 262144 x 64 KiB/cell is ~16 GiB. # budget. KV at 262144 x 64 KiB/cell is ~16 GiB.
CTX_SIZE=262144 CTX_SIZE=262144
PARALLEL=1 PARALLEL=1
BATCH_SIZE=8192 BATCH_SIZE=4096
UBATCH_SIZE=8192 UBATCH_SIZE=4096
CACHE_TYPE_K=f16 CACHE_TYPE_K=f16
CACHE_TYPE_V=f16 CACHE_TYPE_V=f16
CTX_CHECKPOINTS=16 CTX_CHECKPOINTS=16
scripts/serve-frank-large.sh
+2 -2
View File
@@ -39,8 +39,8 @@ fi
: "${LISTEN_PORT:=52625}" : "${LISTEN_PORT:=52625}"
: "${CTX_SIZE:=524288}" : "${CTX_SIZE:=524288}"
: "${PARALLEL:=2}" : "${PARALLEL:=2}"
: "${BATCH_SIZE:=2048}" : "${BATCH_SIZE:=4096}"
: "${UBATCH_SIZE:=2048}" : "${UBATCH_SIZE:=4096}"
: "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_K:=f16}"
: "${CACHE_TYPE_V:=f16}" : "${CACHE_TYPE_V:=f16}"
: "${CTX_CHECKPOINTS:=16}" : "${CTX_CHECKPOINTS:=16}"