feat: frank-default-small — 4 slots, batch 8192/2048, n-cpu-moe 32, ngram-mod

- PARALLEL=4 (800000 total ctx), batch=8192, ubatch=2048
- n-cpu-moe=32 (moved 8 expert layers back to GPU)
- ctx-checkpoints=4 for headroom during larger batch compute
- All other tunables unchanged (ngram-mod, no-mmap, no-kv-offload, q8_0/f16 KV)

scripts/frank-default-small.env

@@ -4,29 +4,29 @@ MODEL_DIR=/home/dank/models
 ALIAS=frank-default-small
 HOST=0.0.0.0
 PORT=7770
-# 3 parallel slots x 200000 ctx each = 600000 total context pool.
-# (Reduced from 262144/stream per operator: allows the larger batch to fit VRAM.)
-CTX_SIZE=600000
-PARALLEL=3
+# 4 parallel slots x 200000 ctx each = 800000 total context pool.
+# (Increased from 3×200k to add a 4th parallel stream.)
+CTX_SIZE=800000
+PARALLEL=4
+# Spec: ngram-mod. n_min=48, n_max=64, n_match=24 (defaults). 8 expert
+# layers back on GPU; rest on CPU. Batch 8192 / ubatch 2048.
 SPEC_TYPE=ngram-mod
-# Ngram-mod speculative decoding: look back this many tokens to guess the
-# next tokens. Defaults (48/64/24) are a reasonable starting point; tune if
-# you want more or fewer draft lookahead tokens.
 NGRAM_MOD_N_MIN=48
 NGRAM_MOD_N_MAX=64
 NGRAM_MOD_N_MATCH=24
+# Expert offload: 32 of 40 layers keep MoE weights on CPU; remaining
+# 8 layers have their expert weights resident on GPU for faster routing.
+N_CPU_MOE=32
+N_GPU_LAYERS=all
+# KV cache on GPU, K quantized to q8_0, V stays f16 (user-locked decision).
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=f16
-# All 40 layers keep their MoE expert weights on CPU (targeted expert offload).
-N_CPU_MOE=40
-# Every non-expert tensor (attention, norms, embedding, LM head) goes to GPU.
-N_GPU_LAYERS=all
-# Larger batches for prefill throughput. The bigger physical batch grows GPU
-# activation buffers; ctx-checkpoints parks attention checkpoints in RAM to
-# hold peak GPU memory down so the larger ubatch fits without OOM.
-BATCH_SIZE=3072
-UBATCH_SIZE=1536
+# Store SWA checkpoints in RAM to hold peak GPU memory down for the
+# larger batch/ubatch (leaves headroom for ctx-checkpoints operation).
 CTX_CHECKPOINTS=4
+# Larger batches / ubatch for big prefill workloads.
+BATCH_SIZE=8192
+UBATCH_SIZE=2048
 # Load the model into RAM rather than lazy-mapping it. The MoE experts live on
 # CPU, so eager RAM load avoids page faults during decode (uses ~31GB RAM).
 NO_MMAP=1