perf: frank-default-small ctx 200000/stream, --no-mmap, batch 3072/1536
Lower each stream to 200000 ctx (pool 600000) to free KV VRAM for the larger batch. Add --no-mmap so the CPU-resident MoE experts load eagerly into RAM (avoids decode page faults). Keep KV on GPU. Experts already fully on CPU.
This commit is contained in:
1 parent
e45f457ad4
commit
9971f83e8b
2 files changed
+8
-2
No files matched your search
@@ -4,8 +4,9 @@ MODEL_DIR=/home/dank/models
|
||||
ALIAS=frank-default-small
|
||||
HOST=0.0.0.0
|
||||
PORT=7770
|
||||
# 3 parallel slots x 262144 ctx each = 786432 total context pool.
|
||||
CTX_SIZE=786432
|
||||
# 3 parallel slots x 200000 ctx each = 600000 total context pool.
|
||||
# (Reduced from 262144/stream per operator: allows the larger batch to fit VRAM.)
|
||||
CTX_SIZE=600000
|
||||
PARALLEL=3
|
||||
SPEC_TYPE=draft-dflash
|
||||
SPEC_N_MAX=7
|
||||
@@ -21,5 +22,8 @@ N_GPU_LAYERS=all
|
||||
BATCH_SIZE=3072
|
||||
UBATCH_SIZE=1536
|
||||
CTX_CHECKPOINTS=4
|
||||
# Load the model into RAM rather than lazy-mapping it. The MoE experts live on
|
||||
# CPU, so eager RAM load avoids page faults during decode (uses ~31GB RAM).
|
||||
NO_MMAP=1
|
||||
THREADS=8
|
||||
THREADS_HTTP=8
|
||||
Reference in new issue
Block a user