dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-default-small: ubatch=1536, ctx-checkpoints=2 (24GB VRAM safety)

Browse Source
This commit is contained in:
Frank committed 2026-08-20 16:51:40 -07:00
1 parent 87dd4224b7
commit f8deaa798a
2 files changed
+4 -4
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/frank-default-small.env
+3 -3
View File
@@ -41,11 +41,11 @@ CACHE_TYPE_V=f16
VERBOSITY=4 VERBOSITY=4
# Store SWA/context checkpoints in RAM to hold peak GPU memory down. # Store SWA/context checkpoints in RAM to hold peak GPU memory down.
CTX_CHECKPOINTS=4 CTX_CHECKPOINTS=2
# Big prefill batches (ubatch 1024, batch 4096). # Big prefill batches (ubatch 1536, batch 4096).
BATCH_SIZE=4096 BATCH_SIZE=4096
UBATCH_SIZE=1024 UBATCH_SIZE=1536
# Eager RAM load (MoE experts on CPU; ~31 GB RAM). # Eager RAM load (MoE experts on CPU; ~31 GB RAM).
NO_MMAP=1 NO_MMAP=1
scripts/serve-frank-small.sh
+1 -1
View File
@@ -40,7 +40,7 @@ fi
: "${N_GPU_LAYERS:=all}" : "${N_GPU_LAYERS:=all}"
: "${BATCH_SIZE:=4096}" : "${BATCH_SIZE:=4096}"
: "${UBATCH_SIZE:=1024}" : "${UBATCH_SIZE:=1024}"
: "${CTX_CHECKPOINTS:=4}" : "${CTX_CHECKPOINTS:=2}"
: "${NO_MMAP:=1}" : "${NO_MMAP:=1}"
: "${TEMPERATURE:=1.0}" : "${TEMPERATURE:=1.0}"
: "${TOP_K:=20}" : "${TOP_K:=20}"