From 40d50fed065e9d6bb007705eaf3299e1f1858d25 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Sat, 22 Aug 2026 01:03:08 -0700
Subject: [PATCH] =?UTF-8?q?large:=20Kairic=20chat=20profile=20=E2=80=94=20?=
 =?UTF-8?q?reasoning=20on/deepseek,=20batch=204096/2048,=20preserve=5Fthin?=
 =?UTF-8?q?king?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Local overrides to the pinned runner for production chat:
  - --reasoning on --reasoning-format deepseek (thinking to
    message.reasoning_content, matching the retired v3 service so the
    fleet's reasoning clients work)
  - --reasoning-budget -1 (unrestricted thinking)
  - --chat-template-kwargs '{"preserve_thinking":"true"}' (preserve
    thinking tokens in the conversation history across turns)
  - batch 4096 / ubatch 2048 (larger than the runner's 2048/512; unified
    memory has headroom)

The Kairic IU4 lane (--kairic-edge) and the MTP draft fast path are
unaffected.
---
 scripts/frank-default-large.env | 14 +++++++++++---
 scripts/serve-frank-large.sh    | 24 +++++++++++++++---------
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env
index 8f22c50..55dadef 100644
--- a/scripts/frank-default-large.env
+++ b/scripts/frank-default-large.env
@@ -50,8 +50,10 @@ LISTEN_PORT=52625
 # while the prompt cache is enabled.
 CTX_SIZE=262144
 PARALLEL=1
-BATCH_SIZE=2048
-UBATCH_SIZE=512
+# Local override: the pinned runner uses 2048/512; production chat uses a
+# larger 4096/2048 batch (unified memory has headroom).
+BATCH_SIZE=4096
+UBATCH_SIZE=2048
 TARGET_THREADS=16
 BATCH_THREADS=32
 CACHE_TYPE_K=f16
@@ -70,4 +72,10 @@ SPEC_DRAFT_P_SPLIT=0.10
 # profile (temp 0 / top-p 1 / top-k 0 / min-p 0) as the server default.
 # Clients override per request; the model card recommends Qwen live
 # values for ordinary chat (temp 0.7, top-p 0.8, top-k 20, min-p 0,
-# presence_penalty 1.5). Reasoning is off in the qualified profile.
+# presence_penalty 1.5).
+#
+# Reasoning: the qualified profile runs reasoning off, but production
+# chat runs --reasoning on --reasoning-format deepseek (thinking goes to
+# message.reasoning_content, matching the retired v3 service) with
+# --reasoning-budget -1 (unrestricted) and chat_template_kwargs
+# preserve_thinking=true (preserve thinking tokens across turns).
diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh
index 9819078..3024ddd 100755
--- a/scripts/serve-frank-large.sh
+++ b/scripts/serve-frank-large.sh
@@ -47,8 +47,8 @@ fi
 : "${LISTEN_PORT:=52625}"
 : "${CTX_SIZE:=262144}"
 : "${PARALLEL:=1}"
-: "${BATCH_SIZE:=2048}"
-: "${UBATCH_SIZE:=512}"
+: "${BATCH_SIZE:=4096}"
+: "${UBATCH_SIZE:=2048}"
 : "${TARGET_THREADS:=16}"
 : "${BATCH_THREADS:=32}"
 : "${CACHE_TYPE_K:=f16}"
@@ -122,10 +122,15 @@ export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0
 export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0
 export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
 
-# Pinned runner flags, verbatim (qualified 262K one-slot profile, native
-# MTP depth 4, deterministic sampler, reasoning off). Sampling is
-# overridden per request by clients; the card recommends Qwen live values
-# (temp 0.7, top-p 0.8, top-k 20, min-p 0, presence_penalty 1.5).
+# Pinned runner flags (qualified 262K one-slot profile, native MTP depth
+# 4, deterministic sampler). Local overrides for production chat:
+#   - reasoning on + deepseek format (thinking goes to message.reasoning_content,
+#     matching the retired v3 service so the fleet's reasoning clients work)
+#   - batch 4096 / ubatch 2048 (larger than the runner's 2048/512)
+#   - chat_template_kwargs preserve_thinking=true (preserve thinking tokens
+#     in the conversation history across turns)
+# Sampling is overridden per request by clients; the card recommends Qwen
+# live values (temp 0.7, top-p 0.8, top-k 20, min-p 0, presence_penalty 1.5).
 exec "$LLAMA_BIN" \
     -m "$MODEL" \
     --alias "$ALIAS" \
@@ -163,6 +168,7 @@ exec "$LLAMA_BIN" \
     --top-p 1.0 \
     --top-k 0 \
     --min-p 0.0 \
-    --reasoning off \
-    --reasoning-format none \
-    --reasoning-budget -1
+    --reasoning on \
+    --reasoning-format deepseek \
+    --reasoning-budget -1 \
+    --chat-template-kwargs '{"preserve_thinking":"true"}'