From aa2f4bfe39e997d4e1b5cc726777b007e1164660 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Tue, 11 Aug 2026 13:47:37 -0700
Subject: [PATCH] =?UTF-8?q?feat:=20frank-default-large=20=E2=86=92=20llama?=
 =?UTF-8?q?.cpp=20Qwen3.6-27B=20Q8=5FK=5FP=20+=20DFlash=20draft=20+=20ngra?=
 =?UTF-8?q?m-mod=20(bench=20pending)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/DFlash DSpark Speculative Decoding.md |  10 ++
 scripts/bench-frank-large.sh               | 150 +++++++++++++++++++++
 scripts/frank-default-large.env            |  31 +++++
 scripts/frank-default-large.service        |   7 +-
 scripts/serve-llama-frank-large.sh         |  74 ++++++++++
 5 files changed, 268 insertions(+), 4 deletions(-)
 create mode 100755 scripts/bench-frank-large.sh
 create mode 100644 scripts/frank-default-large.env
 create mode 100755 scripts/serve-llama-frank-large.sh

diff --git a/docs/DFlash DSpark Speculative Decoding.md b/docs/DFlash DSpark Speculative Decoding.md
index ad1c985..8cb31b8 100644
--- a/docs/DFlash DSpark Speculative Decoding.md	
+++ b/docs/DFlash DSpark Speculative Decoding.md	
@@ -75,6 +75,16 @@ vllm serve <target-model> \
 
 llama.cpp supports DFlash, DSpark, and MTP through `--spec-type`. When you pass `--spec-draft-hf`, llama.cpp auto-discovers the sidecar files and infers the type. The priority order is DSpark, DFlash, EAGLE-3, MTP.
 
+`--spec-type` accepts a **comma-separated list** of speculative implementations. `common_speculative_init` (in `common/speculative.cpp`) builds a vector of implementations and chains drafting through all of them (`common_speculative_draft_params.drafting` is "used to chain the drafts through all the available implementations"). This means you can **layer ngram-mod on top of a DFlash draft model** in one server process:
+
+```
+llama-server --hf-repo <target-GGUF> \
+    --spec-draft-hf <dflash-draft-GGUF> \
+    --spec-type draft-dflash,ngram-mod
+```
+
+The DFlash drafter proposes, then ngram-mod runs as an additional implementation in the chain. Verified against upstream `common/speculative.cpp` (b10133+).
+
 DFlash on llama.cpp:
 
 ```
diff --git a/scripts/bench-frank-large.sh b/scripts/bench-frank-large.sh
new file mode 100755
index 0000000..9d4f11b
--- /dev/null
+++ b/scripts/bench-frank-large.sh
@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+# bench-frank-large.sh — 64k-context benchmark for the frank-default-large stack.
+#
+# Measures generation throughput of Qwen3.6-27B Q8_K_P + DFlash + ngram-mod on
+# a single backend at 64k context. It starts llama-server on a scratch port,
+# drives one generation through the OpenAI-compatible API with a ~64k-token
+# prompt, and prints tokens/sec.
+#
+# Usage:
+#   ./scripts/bench-frank-large.sh <ROCm0|Vulkan0> [--generations N]
+#
+# Example:
+#   ./scripts/bench-frank-large.sh ROCm0
+#   ./scripts/bench-frank-large.sh Vulkan0 --generations 3
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DEVICE="${1:?usage: bench-frank-large.sh <ROCm0|Vulkan0> [--generations N]}"
+GENERATIONS=1
+if [ "${2:-}" = "--generations" ]; then
+    GENERATIONS="${3:?--generations needs a count}"
+fi
+
+MODEL_DIR=/home/dank/models
+MODEL_FILE=Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf
+DRAFT_FILE=Qwen3.6-27B-DFlash-Q8_0.gguf
+LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
+MODEL="$MODEL_DIR/$MODEL_FILE"
+DRAFT="$MODEL_DIR/$DRAFT_FILE"
+
+PORT=7890
+CTX=65536
+PARALLEL=1
+BATCH=2048
+UBATCH=512
+OUT_TOKENS=256
+
+for f in "$MODEL" "$DRAFT"; do
+    if [[ ! -f "$f" ]]; then
+        echo "missing required file: $f" >&2
+        exit 1
+    fi
+done
+
+echo "=== 64k benchmark: device=$DEVICE model=$MODEL_FILE ==="
+
+# --- start the server ----------------------------------------------------
+"$LLAMA_SERVER" \
+    --model "$MODEL" \
+    --spec-draft-model "$DRAFT" \
+    --spec-type "draft-dflash,ngram-mod" \
+    --spec-draft-n-max 8 \
+    --spec-ngram-mod-n-min 48 \
+    --spec-ngram-mod-n-max 64 \
+    --spec-ngram-mod-n-match 24 \
+    --device "$DEVICE" \
+    --gpu-layers all \
+    --cache-type-k q8_0 \
+    --cache-type-v f16 \
+    --ctx-size "$CTX" \
+    --parallel "$PARALLEL" \
+    --batch-size "$BATCH" \
+    --ubatch-size "$UBATCH" \
+    --alias bench \
+    --host 127.0.0.1 \
+    --port "$PORT" \
+    > "/tmp/frankbench-$DEVICE.log" 2>&1 &
+SERVER_PID=$!
+echo "server pid $SERVER_PID on 127.0.0.1:$PORT"
+
+trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
+
+# --- wait for ready ------------------------------------------------------
+echo -n "waiting for server"
+ready=0
+for i in $(seq 1 400); do
+    if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
+        echo " [ready]"
+        ready=1
+        break
+    fi
+    echo -n "."
+    sleep 2
+done
+if [ "$ready" != "1" ]; then
+    echo ""
+    echo "FAIL: server did not become ready"
+    tail -25 "/tmp/frankbench-$DEVICE.log"
+    exit 1
+fi
+
+# --- drive N generations and measure --------------------------------------
+python3 - "$DEVICE" "$CTX" "$OUT_TOKENS" "$GENERATIONS" "$PORT" <<'PY'
+import sys, time, json, urllib.request, statistics
+
+device       = sys.argv[1]
+ctx          = int(sys.argv[2])
+out_tokens   = int(sys.argv[3])
+generations  = int(sys.argv[4])
+port         = sys.argv[5]
+
+# ~64k-token prompt. One prose unit is ~110 chars ≈ 28 tokens for this model.
+# We aim for a prompt of ~60k tokens so generation stays within the 64k ctx.
+unit = ("The careful study of natural language reveals patterns of meaning "
+        "that emerge from the deliberate arrangement of words and phrases. ")
+# approximate: unit is 160 chars, ~40 tokens; repeat to ~1500 units = ~60k tokens
+prompt = unit * 1500
+
+def req(prompt, n):
+    body = json.dumps({
+        "prompt": prompt,
+        "max_tokens": n,
+        "temperature": 0.0,
+        "stream": False,
+    }).encode()
+    r = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
+                               data=body, headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(r, timeout=1800) as resp:
+        return json.load(resp)
+
+# prefill timing
+t0 = time.time()
+first = req(prompt, 0)
+t_pre = time.time() - t0
+used = first.get("usage", {})
+prompt_tokens = used.get("prompt_tokens", 0)
+print(f"prefill: {prompt_tokens} tokens in {t_pre:.2f}s "
+      f"({prompt_tokens/t_pre:.1f} tok/s)")
+
+# generation timing
+rates = []
+total_gentok = 0
+total_time = 0.0
+for i in range(generations):
+    t0 = time.time()
+    out = req(prompt, out_tokens)
+    dt = time.time() - t0
+    uit = out.get("usage", {})
+    comp = uit.get("completion_tokens", 0)
+    gt = dt - t_pre  # approximate generation time = total - prefill
+    r = comp / gt if gt > 0 else 0
+    rates.append(r)
+    total_gentok += comp
+    total_time += gt
+    print(f"gen {i+1}: {comp} tokens in {gt:.2f}s ({r:.1f} tok/s)")
+
+avg = statistics.mean(rates) if rates else 0
+print(f"\nRESULT device={device} prompt_tokens={prompt_tokens} "
+      f"avg_gen_tok_s={avg:.1f} over {generations} runs")
+PY
diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env
new file mode 100644
index 0000000..abb3fb7
--- /dev/null
+++ b/scripts/frank-default-large.env
@@ -0,0 +1,31 @@
+# Tunables for frank-default-large (HauhauCS Qwen3.6-27B Q8_K_P, DFlash+ngram-mod, dankclaw)
+# Runtime: llama.cpp llama-server; AMD Strix Halo (gfx1151) 96GB UMA; Ryzen AI MAX+ 395.
+MODEL_DIR=/home/dank/models
+ALIAS=frank-default-large
+HOST=0.0.0.0
+PORT=7770
+# 6 parallel slots x 65536 ctx each = 393216 total context pool.
+CTX_SIZE=393216
+PARALLEL=6
+# Spec: DFlash draft model layered with ngram-mod. --spec-type accepts a
+# comma-separated list; llama.cpp chains the implementations (verified in
+# common/speculative.cpp). --spec-draft-n-max must stay below the DFlash
+# block size (BF16 sidecar = 8 token block), so 8 is the ceiling.
+SPEC_TYPE=draft-dflash,ngram-mod
+SPEC_DRAFT_N_MAX=8
+SPEC_NGRAM_MOD_N_MIN=48
+SPEC_NGRAM_MOD_N_MAX=64
+SPEC_NGRAM_MOD_N_MATCH=24
+# Backend device for offload, from --list-devices. Pick the faster of:
+#   ROCm0    (HIP / ROCm backend)
+#   Vulkan0  (RADV Vulkan backend)
+DEVICE=ROCm0
+# Dense model (Qwen3.6-27B, not MoE): no MoE CPU offload. Full offload.
+N_GPU_LAYERS=all
+# KV on GPU, quantized (K q8_0, V f16); mirrors the small service profile.
+CACHE_TYPE_K=q8_0
+CACHE_TYPE_V=f16
+BATCH_SIZE=2048
+UBATCH_SIZE=512
+THREADS=8
+THREADS_HTTP=8
diff --git a/scripts/frank-default-large.service b/scripts/frank-default-large.service
index f4e513b..0f29d3b 100644
--- a/scripts/frank-default-large.service
+++ b/scripts/frank-default-large.service
@@ -1,16 +1,15 @@
 [Unit]
-Description=vLLM Ling-3.0-flash INT4 (CIRU gfx1151) -- frank-default-large
+Description=llama.cpp Qwen3.6-27B Q8_K_P DFlash+ngram-mod (frank-default-large)
 After=network.target
 
 [Service]
 Type=simple
 User=dank
 Group=dank
-WorkingDirectory=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native
 Environment="HOST=0.0.0.0"
 Environment="PORT=7770"
-Environment="SERVED_MODEL_NAME=frank-default-large"
-ExecStart=/home/dank/vllmScripts/scripts/serve-ling3-vllm.sh
+Environment="FRANK_LARGE_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env"
+ExecStart=/home/dank/vllmScripts/scripts/serve-llama-frank-large.sh
 Restart=on-failure
 RestartSec=10
 
diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh
new file mode 100755
index 0000000..8fa6806
--- /dev/null
+++ b/scripts/serve-llama-frank-large.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Serve HauhauCS Qwen3.6-27B-Uncensored-HauhauCS-Aggressive:Q8_K_P
+# with DFlash draft + ngram-mod speculative decoding on dankclaw
+# (AMD Strix Halo gfx1151, ROCm or Vulkan backend).
+# Runtime: llama.cpp llama-server.
+#
+# Tunables come from frank-default-large.env in this directory unless the
+# FRANK_LARGE_ENV_FILE environment variable overrides that path.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -----------------
+: "${MODEL_DIR:=/home/dank/models}"
+: "${MODEL_FILE:=Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf}"
+: "${DRAFT_FILE:=Qwen3.6-27B-DFlash-Q8_0.gguf}"
+: "${ALIAS:=frank-default-large}"
+: "${HOST:=0.0.0.0}"
+: "${PORT:=7770}"
+: "${CTX_SIZE:=393216}"
+: "${PARALLEL:=6}"
+: "${SPEC_TYPE:=draft-dflash,ngram-mod}"
+: "${SPEC_DRAFT_N_MAX:=8}"
+: "${SPEC_NGRAM_MOD_N_MIN:=48}"
+: "${SPEC_NGRAM_MOD_N_MAX:=64}"
+: "${SPEC_NGRAM_MOD_N_MATCH:=24}"
+: "${DEVICE:=ROCm0}"
+: "${N_GPU_LAYERS:=all}"
+: "${CACHE_TYPE_K:=q8_0}"
+: "${CACHE_TYPE_V:=f16}"
+: "${BATCH_SIZE:=2048}"
+: "${UBATCH_SIZE:=512}"
+: "${THREADS:=8}"
+: "${THREADS_HTTP:=8}"
+
+LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp/build/bin/llama-server}"
+MODEL="$MODEL_DIR/$MODEL_FILE"
+DRAFT="$MODEL_DIR/$DRAFT_FILE"
+
+for f in "$MODEL" "$DRAFT"; do
+    if [[ ! -f "$f" ]]; then
+        echo "missing required file: $f" >&2
+        exit 1
+    fi
+done
+
+exec "$LLAMA_SERVER" \
+    --model "$MODEL" \
+    --spec-draft-model "$DRAFT" \
+    --spec-type "$SPEC_TYPE" \
+    --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
+    --spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \
+    --spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX" \
+    --spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
+    --device "$DEVICE" \
+    --gpu-layers "$N_GPU_LAYERS" \
+    --cache-type-k "$CACHE_TYPE_K" \
+    --cache-type-v "$CACHE_TYPE_V" \
+    --ctx-size "$CTX_SIZE" \
+    --parallel "$PARALLEL" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --threads "$THREADS" \
+    --threads-http "$THREADS_HTTP" \
+    --alias "$ALIAS" \
+    --host "$HOST" \
+    --port "$PORT"