From 1ca3db4ae247de7b2342b5fec908130b1f0d60f7 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Sun, 30 Aug 2026 01:24:40 -0700
Subject: [PATCH] frank-default-large: ubatch 512, disable speculative decoding

- UBATCH_SIZE: 4096 -> 512 (reduce GPU memory pressure)
- SPEC_TYPE: ngram-mod -> none (disable speculative decoding entirely)
---
 scripts/frank-default-large.env | 8 ++++----
 scripts/serve-frank-large.sh    | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env
index 93d1ef4..81e21e4 100644
--- a/scripts/frank-default-large.env
+++ b/scripts/frank-default-large.env
@@ -59,7 +59,7 @@ BIND_ADDR=0.0.0.0
 LISTEN_PORT=7770
 
 # Serving: 401408-token context, TWO slots (parallel 2), batch 4096 /
-# ubatch 4096, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
+# ubatch 512, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
 # prompt cache. --fit off: auto-fit segfaults on this model's
 # hyper-connection/indexer config; -ngl 999 puts every layer on the
 # SVM-backed GPU pool.
@@ -71,7 +71,7 @@ CTX_SIZE=401408
 PARALLEL=2
 
 BATCH_SIZE=4096
-UBATCH_SIZE=4096
+UBATCH_SIZE=512
 NUM_THREADS=16
 THREAD_BATCH=16
 CACHE_TYPE_K=f16
@@ -104,10 +104,10 @@ HSA_OVERRIDE_GFX_VERSION=11.5.1
 VERBOSITY=4
 
 
-# Speculative decoding: ngram-mod only. The MTP draft head is retained
+# Speculative decoding: disabled.
 DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
 DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf
-SPEC_TYPE=ngram-mod
+SPEC_TYPE=none
 SPEC_DRAFT_N_MAX=3
 SPEC_DRAFT_N_MIN=0
 SPEC_DRAFT_P_MIN=0.75
diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh
index 751982b..f6e5e23 100755
--- a/scripts/serve-frank-large.sh
+++ b/scripts/serve-frank-large.sh
@@ -47,8 +47,8 @@ fi
 : "${LISTEN_PORT:=7770}"
 : "${CTX_SIZE:=401408}"
 : "${PARALLEL:=2}"
-: "${BATCH_SIZE:=4096}"
-: "${UBATCH_SIZE:=4096}"
+:: "${BATCH_SIZE:=4096}"
+:: "${UBATCH_SIZE:=512}"
 : "${NUM_THREADS:=16}"
 : "${THREAD_BATCH:=16}"
 : "${CACHE_TYPE_K:=f16}"
@@ -67,8 +67,8 @@ fi
 : "${PRESENCE_PENALTY:=1.5}"
 : "${REASONING:=on}"
 : "${REASONING_FORMAT:=deepseek}"
-: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
-: "${SPEC_TYPE:=ngram-mod}"
+:: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
+:: "${SPEC_TYPE:=none}"
 : "${DRAFT_MODEL_DIR:=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST}"
 : "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}"
 : "${SPEC_DRAFT_N_MAX:=3}"