From 6451017d3c41cdc4097af730b84d798919eeba2d Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Thu, 13 Aug 2026 21:16:32 -0700
Subject: [PATCH] dankclaw: bench cycle 1 - switch spec to ngram-mod; draft
 flags conditional

DFlash causes premature EOS on a generating slot when another task lands
with high LCP prompt reuse (shared draft ctx corruption, reproduced
2026-08-14 3x). Benching ngram-mod vs MTP sidecar to pick the
replacement. serve script now passes --spec-draft-model only for
draft-* spec types.
---
 scripts/frank-default-large.env    | 24 +++++++++++++++---------
 scripts/serve-llama-frank-large.sh | 19 ++++++++++++++-----
 2 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env
index cb117d7..0d129ea 100644
--- a/scripts/frank-default-large.env
+++ b/scripts/frank-default-large.env
@@ -7,11 +7,19 @@ PORT=7770
 # 3 parallel slots x 200700 ctx each = 602100 total context pool.
 CTX_SIZE=602100
 PARALLEL=3
-# Spec: DFlash draft model. Bench (4k, ROCm) showed draft-dflash fastest
-# generation (20.2 t/s): DFlash+ngram 16.9, ngram-only 8.7. ngram layering
-# reduced throughput, so it is not layered in production.
-SPEC_TYPE=draft-dflash
-SPEC_DRAFT_N_MAX=8
+# Spec: speculative decoding mode.
+#   draft-dflash  - DFlash block-diffusion draft (benchmark winner pre-2026-08-14)
+#                   BUT: premature EOS under parallel load when a new task
+#                   reuses a cached prompt (shared draft ctx corruption). Retired.
+#   draft-mtp     - MTP sidecar draft (a4lg/Qwen3.6-27B-MTP-ONLY-Q8_0.gguf)
+#   ngram-mod     - draftless ngram matching (no draft model, no shared ctx)
+# Benched 2026-08-14: see docs/frank-default-large-benchmark.md for the winner.
+SPEC_TYPE=ngram-mod
+# Draft model file (only used when SPEC_TYPE contains 'draft-').
+# DFlash: Qwen3.6-27B-DFlash-Q8_0.gguf ; MTP: Qwen3.6-27B-MTP-ONLY-Q8_0.gguf
+DRAFT_FILE=Qwen3.6-27B-MTP-ONLY-Q8_0.gguf
+# Draft length per spec type: dflash 8, mtp 2-3.
+SPEC_DRAFT_N_MAX=3
 SPEC_NGRAM_MOD_N_MIN=48
 SPEC_NGRAM_MOD_N_MAX=64
 SPEC_NGRAM_MOD_N_MATCH=24
@@ -28,10 +36,8 @@ CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=q8_0
 # Batch sizing: empirically verified 2026-08-13 with caches dropped.
 # batch 8192 / ubatch 2048 loads and serves (61 GiB VRAM peak, ctx 602100).
-# The earlier "ubatch 2048 OOMs" notes in frank-default-large-benchmark.md
-# were confounded by page-cache pressure: with the model's pages fully
-# cached, the KV/graph allocation stalls (VRAM stuck at 28 GiB).
-# Drop caches before starting a load: sync && echo 3 > /proc/sys/vm/drop_caches.
+# ubatch is the compute-graph driver; larger ubatch = more VRAM, faster prefill.
+# Re-verified 2026-08-14 during the spec swap (see benchmark doc).
 BATCH_SIZE=8192
 UBATCH_SIZE=2048
 # Sampling settings — per HauhauCS Qwen3.6-27B model card.
diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh
index ab459bc..54bb437 100755
--- a/scripts/serve-llama-frank-large.sh
+++ b/scripts/serve-llama-frank-large.sh
@@ -48,16 +48,25 @@ LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp/build/bin/llama-server}"
 MODEL="$MODEL_DIR/$MODEL_FILE"
 DRAFT="$MODEL_DIR/$DRAFT_FILE"
 
-for f in "$MODEL" "$DRAFT"; do
-    if [[ ! -f "$f" ]]; then
-        echo "missing required file: $f" >&2
+if [[ ! -f "$MODEL" ]]; then
+    echo "missing required file: $MODEL" >&2
+    exit 1
+fi
+
+# Draft-model flags apply only to spec types that use a draft file
+# (draft-dflash, draft-mtp, draft-eagle3). ngram-mod is draftless.
+SPEC_ARGS=()
+if [[ "$SPEC_TYPE" == *draft-* ]]; then
+    if [[ ! -f "$DRAFT" ]]; then
+        echo "missing required file: $DRAFT" >&2
         exit 1
     fi
-done
+    SPEC_ARGS+=(--spec-draft-model "$DRAFT")
+fi
 
 exec "$LLAMA_SERVER" \
     --model "$MODEL" \
-    --spec-draft-model "$DRAFT" \
+    "${SPEC_ARGS[@]}" \
     --spec-type "$SPEC_TYPE" \
     --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
     --spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \