frank-default-small: ubatch=1024 (tight 24GB VRAM, 786k ctx pool)
This commit is contained in:
1 parent
26b42d2d6e
commit
87dd4224b7
2 files changed
+3
-3
No files matched your search
@@ -43,9 +43,9 @@ VERBOSITY=4
|
||||
# Store SWA/context checkpoints in RAM to hold peak GPU memory down.
|
||||
CTX_CHECKPOINTS=4
|
||||
|
||||
# Big prefill batches (ubatch 2048, batch 4096).
|
||||
# Big prefill batches (ubatch 1024, batch 4096).
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=2048
|
||||
UBATCH_SIZE=1024
|
||||
|
||||
# Eager RAM load (MoE experts on CPU; ~31 GB RAM).
|
||||
NO_MMAP=1
|
||||
|
||||
Reference in new issue
Block a user