Use ngram-mod and prioritize attention experts

scripts/frank-default-small.env

@@ -30,24 +30,23 @@ LISTEN_PORT=7770
 
 # 3 parallel slots x 163840 ctx each (262144 lowered by 96k per stream for
 # cache and checkpoint headroom) = 491520 total context pool. The per-slot
-# 163840 will be advertised in models.json after validation.
+# 163840 is advertised in models.json.
 CTX_SIZE=491520
 PARALLEL=3
 
-# Spec: MoE model with a bundled MTP head -> native draft-mtp, n_max 4
-# (the knee). ngram under MTP hurts; MTP alone is the proven best for MoE.
-SPEC_TYPE=draft-mtp
-SPEC_DRAFT_N_MAX=4
-SPEC_DRAFT_N_MIN=0
-SPEC_DRAFT_P_MIN=0.0
+# Speculative decoding: ngram-mod only. MTP is intentionally disabled.
+SPEC_TYPE=ngram-mod
+SPEC_NGRAM_MOD_N_MIN=48
+SPEC_NGRAM_MOD_N_MAX=64
+SPEC_NGRAM_MOD_N_MATCH=24
 
 # Expert offload via explicit tensor override (replaces --n-cpu-moe). The Q6_K
-# layer audit found 630 MiB of expert tensors per block. Keep the selected
-# full-attention blocks 27 and 31 plus output-adjacent blocks 33-40 on CUDA,
-# including the bundled MTP block 40. Move all other expert tensors to CPU.
-# All non-expert tensors stay on the 24 GB card. Buffer type "CPU" is the CPU
-# buft name on this host.
-OVERRIDE_EXPERT='blk\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|28|29|30|32)\.ffn_(up|down|gate|gate_up)_(ch|)exps'
+# layer audit found 630 MiB of expert tensors per block. Keep every
+# full-attention block plus every block 35-40 on CUDA. The unique GPU set is
+# 3,7,11,15,19,23,27,31,35,36,37,38,39,40. Move all other expert tensors to
+# CPU. All non-expert tensors stay on the 24 GB card. Buffer type "CPU" is the
+# CPU buft name on this host.
+OVERRIDE_EXPERT='blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps'
 OVERRIDE_BUFT=CPU
 N_GPU_LAYERS=all
 
@@ -59,9 +58,10 @@ VERBOSITY=4
 # Store context checkpoints in RAM to hold peak GPU memory down.
 CTX_CHECKPOINTS=16
 
-# Big prefill batches.
+# Prefill batch sizing. ubatch 1024 is selected for reduced compute-memory
+# pressure and predictable latency under the ngram-mod deployment.
 BATCH_SIZE=4096
-UBATCH_SIZE=2048
+UBATCH_SIZE=1024
 
 # Eager RAM load (expert weights on CPU; ~31 GB RAM).
 NO_MMAP=1