frank-default-small: ubatch=1536, ctx-checkpoints=2 (24GB VRAM safety)
This commit is contained in:
1 parent
87dd4224b7
commit
f8deaa798a
2 files changed
+4
-4
No files matched your search
@@ -41,11 +41,11 @@ CACHE_TYPE_V=f16
|
||||
|
||||
VERBOSITY=4
|
||||
# Store SWA/context checkpoints in RAM to hold peak GPU memory down.
|
||||
CTX_CHECKPOINTS=4
|
||||
CTX_CHECKPOINTS=2
|
||||
|
||||
# Big prefill batches (ubatch 1024, batch 4096).
|
||||
# Big prefill batches (ubatch 1536, batch 4096).
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=1024
|
||||
UBATCH_SIZE=1536
|
||||
|
||||
# Eager RAM load (MoE experts on CPU; ~31 GB RAM).
|
||||
NO_MMAP=1
|
||||
|
||||
Reference in new issue
Block a user