From fd528ab36421964c2f9c2cdb6a21f2248a935e22 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Tue, 11 Aug 2026 02:29:32 -0700
Subject: [PATCH] =?UTF-8?q?chore:=20frank-default-small=20=E2=80=94=20ngra?=
 =?UTF-8?q?m-mod,=20drop=20DFlash=20draft?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- spec-type → ngram-mod (no separate draft model)
- drop --model-draft, --spec-draft-n-max; add --spec-ngram-mod-n-min/max/match
- DFlash batch default fallback SPEC_TYPE → ngram-mod
- env: SPEC_TYPE=ngram-mod, NGRAM_MOD_N_MIN=48, N_MAX=64, N_MATCH=24
- DRAFT_FILE env var emptied (not used under ngram-mod)
---
 scripts/frank-default-small.env    | 11 ++++++++---
 scripts/serve-llama-frank-small.sh | 18 ++++++++++--------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env
index affe1d7..cf5a547 100644
--- a/scripts/frank-default-small.env
+++ b/scripts/frank-default-small.env
@@ -1,4 +1,4 @@
-# Tunables for frank-default-small (HauhauCS Q6_K_P + DFlash draft, dankserver)
+# Tunables for frank-default-small (HauhauCS Q6_K_P, ngram-mod spec, dankserver)
 # Runtime: llama-server b10108; RTX 3090 24GB; Ryzen 7 7700X; 62GB RAM.
 MODEL_DIR=/home/dank/models
 ALIAS=frank-default-small
@@ -8,8 +8,13 @@ PORT=7770
 # (Reduced from 262144/stream per operator: allows the larger batch to fit VRAM.)
 CTX_SIZE=600000
 PARALLEL=3
-SPEC_TYPE=draft-dflash
-SPEC_N_MAX=7
+SPEC_TYPE=ngram-mod
+# Ngram-mod speculative decoding: look back this many tokens to guess the
+# next tokens. Defaults (48/64/24) are a reasonable starting point; tune if
+# you want more or fewer draft lookahead tokens.
+NGRAM_MOD_N_MIN=48
+NGRAM_MOD_N_MAX=64
+NGRAM_MOD_N_MATCH=24
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=f16
 # All 40 layers keep their MoE expert weights on CPU (targeted expert offload).
diff --git a/scripts/serve-llama-frank-small.sh b/scripts/serve-llama-frank-small.sh
index 0fa72f0..1a3c2b3 100755
--- a/scripts/serve-llama-frank-small.sh
+++ b/scripts/serve-llama-frank-small.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # Serve HauhauCS Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q6_K_P
-# with a DFlash draft model on dankserver (RTX 3090, MoE experts on CPU).
+# with ngram-mod speculative decoding on dankserver (RTX 3090, MoE experts on CPU).
 # Runtime: upstream llama.cpp llama-server (b10108+).
 #
 # Tunables come from frank-default-small.env in this directory unless the
@@ -19,15 +19,17 @@ fi
 # --- defaults (used when the env file does not set them) -----------------
 : "${MODEL_DIR:=/home/dank/models}"
 : "${MODEL_FILE:=Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q6_K_P.gguf}"
-: "${DRAFT_FILE:=Qwen3.6-35B-A3B-DFlash-Q8_0.gguf}"
+: "${DRAFT_FILE:=}"   # not used — ngram-mod reuses the loaded model for drafts
 : "${MMPROJ_FILE:=mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf}"
 : "${ALIAS:=frank-default-small}"
 : "${HOST:=0.0.0.0}"
 : "${PORT:=7770}"
 : "${CTX_SIZE:=262144}"
 : "${PARALLEL:=3}"
-: "${SPEC_TYPE:=draft-dflash}"
-: "${SPEC_N_MAX:=7}"
+: "${SPEC_TYPE:=ngram-mod}"
+: "${NGRAM_MOD_N_MIN:=48}"
+: "${NGRAM_MOD_N_MAX:=64}"
+: "${NGRAM_MOD_N_MATCH:=24}"
 : "${CACHE_TYPE_K:=q8_0}"
 : "${CACHE_TYPE_V:=f16}"
 : "${N_CPU_MOE:=40}"     # all 40 layers: MoE expert weights stay on CPU
@@ -41,10 +43,9 @@ fi
 
 LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp/build/bin/llama-server}"
 MODEL="$MODEL_DIR/$MODEL_FILE"
-DRAFT="$MODEL_DIR/$DRAFT_FILE"
 MMPROJ="$MODEL_DIR/$MMPROJ_FILE"
 
-for f in "$MODEL" "$DRAFT" "$MMPROJ"; do
+for f in "$MODEL" "$MMPROJ"; do
     if [[ ! -f "$f" ]]; then
         echo "missing required file: $f" >&2
         exit 1
@@ -53,11 +54,12 @@ done
 
 exec "$LLAMA_SERVER" \
     --model "$MODEL" \
-    --model-draft "$DRAFT" \
     --mmproj "$MMPROJ" \
     --jinja \
     --spec-type "$SPEC_TYPE" \
-    --spec-draft-n-max "$SPEC_N_MAX" \
+    --spec-ngram-mod-n-min "$NGRAM_MOD_N_MIN" \
+    --spec-ngram-mod-n-max "$NGRAM_MOD_N_MAX" \
+    --spec-ngram-mod-n-match "$NGRAM_MOD_N_MATCH" \
     --cache-type-k "$CACHE_TYPE_K" \
     --cache-type-v "$CACHE_TYPE_V" \
     --ctx-size "$CTX_SIZE" \