Offload one more MoE expert to CPU on frank-default-small (34→35 of 40)
- N_CPU_MOE 34→35: 5 expert layers on GPU instead of 6. Frees ~0.4-0.8 GiB VRAM for OOM safety after high-load sessions OOM'd at 32 and 33. - Sync stale doc references (deployment.md, Dankserver System Specs.md) that still claimed 32/8; specs now estimate ~1.8 GiB headroom at 35.
This commit is contained in:
1 parent
c96cb6a529
commit
209223cb4f
4 files changed
+9
-9
No files matched your search
@@ -8,15 +8,15 @@ PORT=7770
|
||||
# (Increased from 3×200k to add a 4th parallel stream.)
|
||||
CTX_SIZE=800000
|
||||
PARALLEL=4
|
||||
# Spec: ngram-mod. n_min=48, n_max=64, n_match=24 (defaults). 6 expert
|
||||
# layers back on GPU; 34 on CPU. Batch 8192 / ubatch 2048.
|
||||
# Spec: ngram-mod. n_min=48, n_max=64, n_match=24 (defaults). 5 expert
|
||||
# layers back on GPU; 35 on CPU. Batch 8192 / ubatch 2048.
|
||||
SPEC_TYPE=ngram-mod
|
||||
NGRAM_MOD_N_MIN=48
|
||||
NGRAM_MOD_N_MAX=64
|
||||
NGRAM_MOD_N_MATCH=24
|
||||
# Expert offload: 34 of 40 layers keep MoE weights on CPU; remaining
|
||||
# 6 layers have their expert weights resident on GPU for faster routing.
|
||||
N_CPU_MOE=34
|
||||
# Expert offload: 35 of 40 layers keep MoE weights on CPU; remaining
|
||||
# 5 layers have their expert weights resident on GPU for faster routing.
|
||||
N_CPU_MOE=35
|
||||
N_GPU_LAYERS=all
|
||||
# KV cache on GPU, K quantized to q8_0, V stays f16 (user-locked decision).
|
||||
CACHE_TYPE_K=q8_0
|
||||
|
||||
Reference in new issue
Block a user