From 7aee1df7c36c20e22399b31bd4ff81beb814a496 Mon Sep 17 00:00:00 2001 From: Frank Date: Sat, 22 Aug 2026 00:28:01 -0700 Subject: [PATCH] large: disable target greedy argmax fast path for chat (Kairic) The pinned runner sets LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1, but the exact-Q8 argmax fast path fails closed on any non-greedy request. The model card's own chat instructions override sampling per request with Qwen live values (temp 0.7, top-p 0.8, top-k 20, min-p 0, presence_penalty 1.5), which the fast path rejects (400). Production chat forces the fast path to 0; the Kairic IU4 lane (--kairic-edge) and the MTP draft fast path are unaffected. --- scripts/serve-frank-large.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index e803b31..9819078 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -88,7 +88,16 @@ export LD_LIBRARY_PATH="$(dirname "$LLAMA_BIN"):${ROCM_LIB}/lib:${ROCM_CORE}/lib export HIP_VISIBLE_DEVICES=0 export HSA_OVERRIDE_GFX_VERSION=11.5.1 export GGML_CUDA_GRAPH_OPT=0 -export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1 +# The pinned runner sets the target exact-Q8 greedy argmax fast path +# (LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1). That fast path fails closed on +# any non-greedy request (temperature != 0, top_k > 1, top_p < 1, penalties, +# grammar, logit bias, or reasoning budget). The model card's own chat +# instructions override sampling per request with Qwen live values (temp +# 0.7, top-p 0.8, top-k 20, min-p 0, presence_penalty 1.5), which the fast +# path rejects. Production chat therefore forces it to 0. This does NOT +# affect the Kairic Edge IU4 lane (prompt/verification route via +# --kairic-edge) nor the MTP draft fast path below. +export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0 export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 export PROMPTFORGE_TARGET_ONLY=0 export PROMPTFORGE_MODE=iu4_ffn