Use ngram-mod and prioritize attention experts

scripts/frank-default-small.env

@@ -30,24 +30,23 @@ LISTEN_PORT=7770
 
 # 3 parallel slots x 163840 ctx each (262144 lowered by 96k per stream for
 # cache and checkpoint headroom) = 491520 total context pool. The per-slot
-# 163840 will be advertised in models.json after validation.
+# 163840 is advertised in models.json.
 CTX_SIZE=491520
 PARALLEL=3
 
-# Spec: MoE model with a bundled MTP head -> native draft-mtp, n_max 4
-# (the knee). ngram under MTP hurts; MTP alone is the proven best for MoE.
-SPEC_TYPE=draft-mtp
-SPEC_DRAFT_N_MAX=4
-SPEC_DRAFT_N_MIN=0
-SPEC_DRAFT_P_MIN=0.0
+# Speculative decoding: ngram-mod only. MTP is intentionally disabled.
+SPEC_TYPE=ngram-mod
+SPEC_NGRAM_MOD_N_MIN=48
+SPEC_NGRAM_MOD_N_MAX=64
+SPEC_NGRAM_MOD_N_MATCH=24
 
 # Expert offload via explicit tensor override (replaces --n-cpu-moe). The Q6_K
-# layer audit found 630 MiB of expert tensors per block. Keep the selected
-# full-attention blocks 27 and 31 plus output-adjacent blocks 33-40 on CUDA,
-# including the bundled MTP block 40. Move all other expert tensors to CPU.
-# All non-expert tensors stay on the 24 GB card. Buffer type "CPU" is the CPU
-# buft name on this host.
-OVERRIDE_EXPERT='blk\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|28|29|30|32)\.ffn_(up|down|gate|gate_up)_(ch|)exps'
+# layer audit found 630 MiB of expert tensors per block. Keep every
+# full-attention block plus every block 35-40 on CUDA. The unique GPU set is
+# 3,7,11,15,19,23,27,31,35,36,37,38,39,40. Move all other expert tensors to
+# CPU. All non-expert tensors stay on the 24 GB card. Buffer type "CPU" is the
+# CPU buft name on this host.
+OVERRIDE_EXPERT='blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps'
 OVERRIDE_BUFT=CPU
 N_GPU_LAYERS=all
 
@@ -59,9 +58,10 @@ VERBOSITY=4
 # Store context checkpoints in RAM to hold peak GPU memory down.
 CTX_CHECKPOINTS=16
 
-# Big prefill batches.
+# Prefill batch sizing. ubatch 1024 is selected for reduced compute-memory
+# pressure and predictable latency under the ngram-mod deployment.
 BATCH_SIZE=4096
-UBATCH_SIZE=2048
+UBATCH_SIZE=1024
 
 # Eager RAM load (expert weights on CPU; ~31 GB RAM).
 NO_MMAP=1

scripts/frank-default-small.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=llama.cpp Ornith-1.5-35B-A3B CRACK Q6_K MTP + Qwen-Sharp template (dankserver, frank-default-small)
+Description=llama.cpp Ornith-1.5-35B-A3B CRACK Q6_K ngram-mod + Qwen-Sharp template (dankserver, frank-default-small)
 After=network-online.target
 Wants=network-online.target
 

scripts/serve-frank-small.sh

@@ -4,17 +4,15 @@
 # experts on CPU).
 #
 # Runtime: upstream llama.cpp llama-server (b10121). The model is a 35B-total /
-# 3B-active MoE-dense hybrid with the MTP speculative head bundled in the GGUF.
+# 3B-active MoE-dense hybrid with an MTP head bundled in the GGUF; this
+# service uses ngram-mod instead of MTP speculation.
 #
 # Expert offload: the expert (FFN exps) tensors selected by the layer-bounded
 # --override-tensor pattern are pinned to CPU. Selected full-attention and
 # final output-adjacent blocks stay on CUDA with all non-expert tensors,
-# including the MTP head.
+# without relying on MTP speculation.
 #
-# Spec: MoE model with a bundled MTP head -> native draft-mtp (the head is
-# auto-detected from the target model; b10121 creates the MTP draft context
-# against the model file). ngram under MTP hurts (measured on the Qwen3.8
-# family), so MTP alone is kept. n_max 4 is the knee.
+# Speculative decoding: ngram-mod only. MTP is intentionally disabled.
 #
 # Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE
 # overrides that path.
@@ -40,21 +38,21 @@ fi
 : "${LISTEN_PORT:=7770}"
 : "${CTX_SIZE:=491520}"
 : "${PARALLEL:=3}"
-: "${SPEC_TYPE:=draft-mtp}"
-: "${SPEC_DRAFT_N_MAX:=4}"
-: "${SPEC_DRAFT_N_MIN:=0}"
-: "${SPEC_DRAFT_P_MIN:=0.0}"
+: "${SPEC_TYPE:=ngram-mod}"
+: "${SPEC_NGRAM_MOD_N_MIN:=48}"
+: "${SPEC_NGRAM_MOD_N_MAX:=64}"
+: "${SPEC_NGRAM_MOD_N_MATCH:=24}"
 : "${CACHE_TYPE_K:=f16}"
 : "${CACHE_TYPE_V:=f16}"
 : "${VERBOSITY:=4}"
 # Expert offload: --override-tensor regex pinned to the CPU buft. Same pattern
 # the internal --cpu-moe / --n-cpu-moe use; explicit here so we only move the
 # MoE expert weights and leave every other tensor on the GPU.
-: "${OVERRIDE_EXPERT:=blk\.(0|1|2|3|4|5|6|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|28|29|30|32)\.ffn_(up|down|gate|gate_up)_(ch|)exps}"
+: "${OVERRIDE_EXPERT:=blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps}"
 : "${OVERRIDE_BUFT:=CPU}"
 : "${N_GPU_LAYERS:=all}"
 : "${BATCH_SIZE:=4096}"
-: "${UBATCH_SIZE:=2048}"
+: "${UBATCH_SIZE:=1024}"
 : "${CTX_CHECKPOINTS:=16}"
 : "${NO_MMAP:=1}"
 : "${TEMPERATURE:=1.0}"
@@ -83,9 +81,9 @@ exec "$LLAMA_SERVER" \
     --mmproj "$MMPROJ" \
     --chat-template-file "$CHAT_TEMPLATE_FILE" \
     --spec-type "$SPEC_TYPE" \
-    --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
-    --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \
-    --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \
+    --spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \
+    --spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX" \
+    --spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
     --override-tensor "$OVERRIDE_EXPERT=$OVERRIDE_BUFT" \
     --cache-type-k "$CACHE_TYPE_K" \
     --cache-type-v "$CACHE_TYPE_V" \