frank-default-small: ubatch=1536, ctx-checkpoints=2 (24GB VRAM safety)

This commit is contained in:
Frank committed 2026-08-20 16:51:40 -07:00
1 parent 87dd4224b7
commit f8deaa798a
2 files changed
+4 -4

No files matched your search

+3 -3
View File
@@ -41,11 +41,11 @@ CACHE_TYPE_V=f16
VERBOSITY=4
# Store SWA/context checkpoints in RAM to hold peak GPU memory down.
CTX_CHECKPOINTS=4
CTX_CHECKPOINTS=2
# Big prefill batches (ubatch 1024, batch 4096).
# Big prefill batches (ubatch 1536, batch 4096).
BATCH_SIZE=4096
UBATCH_SIZE=1024
UBATCH_SIZE=1536
# Eager RAM load (MoE experts on CPU; ~31 GB RAM).
NO_MMAP=1