diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 8f22c50..55dadef 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -50,8 +50,10 @@ LISTEN_PORT=52625 # while the prompt cache is enabled. CTX_SIZE=262144 PARALLEL=1 -BATCH_SIZE=2048 -UBATCH_SIZE=512 +# Local override: the pinned runner uses 2048/512; production chat uses a +# larger 4096/2048 batch (unified memory has headroom). +BATCH_SIZE=4096 +UBATCH_SIZE=2048 TARGET_THREADS=16 BATCH_THREADS=32 CACHE_TYPE_K=f16 @@ -70,4 +72,10 @@ SPEC_DRAFT_P_SPLIT=0.10 # profile (temp 0 / top-p 1 / top-k 0 / min-p 0) as the server default. # Clients override per request; the model card recommends Qwen live # values for ordinary chat (temp 0.7, top-p 0.8, top-k 20, min-p 0, -# presence_penalty 1.5). Reasoning is off in the qualified profile. +# presence_penalty 1.5). +# +# Reasoning: the qualified profile runs reasoning off, but production +# chat runs --reasoning on --reasoning-format deepseek (thinking goes to +# message.reasoning_content, matching the retired v3 service) with +# --reasoning-budget -1 (unrestricted) and chat_template_kwargs +# preserve_thinking=true (preserve thinking tokens across turns). diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 9819078..3024ddd 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -47,8 +47,8 @@ fi : "${LISTEN_PORT:=52625}" : "${CTX_SIZE:=262144}" : "${PARALLEL:=1}" -: "${BATCH_SIZE:=2048}" -: "${UBATCH_SIZE:=512}" +: "${BATCH_SIZE:=4096}" +: "${UBATCH_SIZE:=2048}" : "${TARGET_THREADS:=16}" : "${BATCH_THREADS:=32}" : "${CACHE_TYPE_K:=f16}" @@ -122,10 +122,15 @@ export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0 export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0 export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0 -# Pinned runner flags, verbatim (qualified 262K one-slot profile, native -# MTP depth 4, deterministic sampler, reasoning off). Sampling is -# overridden per request by clients; the card recommends Qwen live values -# (temp 0.7, top-p 0.8, top-k 20, min-p 0, presence_penalty 1.5). +# Pinned runner flags (qualified 262K one-slot profile, native MTP depth +# 4, deterministic sampler). Local overrides for production chat: +# - reasoning on + deepseek format (thinking goes to message.reasoning_content, +# matching the retired v3 service so the fleet's reasoning clients work) +# - batch 4096 / ubatch 2048 (larger than the runner's 2048/512) +# - chat_template_kwargs preserve_thinking=true (preserve thinking tokens +# in the conversation history across turns) +# Sampling is overridden per request by clients; the card recommends Qwen +# live values (temp 0.7, top-p 0.8, top-k 20, min-p 0, presence_penalty 1.5). exec "$LLAMA_BIN" \ -m "$MODEL" \ --alias "$ALIAS" \ @@ -163,6 +168,7 @@ exec "$LLAMA_BIN" \ --top-p 1.0 \ --top-k 0 \ --min-p 0.0 \ - --reasoning off \ - --reasoning-format none \ - --reasoning-budget -1 + --reasoning on \ + --reasoning-format deepseek \ + --reasoning-budget -1 \ + --chat-template-kwargs '{"preserve_thinking":"true"}'