diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 6aa2e32..4b623ae 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -81,3 +81,8 @@ SPEC_DRAFT_P_SPLIT=0.10 # message.reasoning_content, matching the retired v3 service) with # --reasoning-budget -1 (unrestricted) and chat_template_kwargs # preserve_thinking=true (preserve thinking tokens across turns). +# +# For fast-greedy bench: comment out the --reasoning on line in serve script +# and set FASTGREEDY=1 in env (see serve script FASTGREEDY section). +# FASTGREEDY=1 flips to: reasoning off, argmax fastpath=1, sampling defaults. +# FASTGREEDY=0 (default) uses the production values above. diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 2ad9d1f..5934836 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -100,7 +100,17 @@ export GGML_CUDA_GRAPH_OPT=0 # path rejects. Production chat therefore forces it to 0. This does NOT # affect the Kairic Edge IU4 lane (prompt/verification route via # --kairic-edge) nor the MTP draft fast path below. -export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0 +# FASTGREEDY env var (0 = production chat, 1 = fast-greedy bench from +# card's default profile). When FASTGREEDY=1: reasoning off, argmax +# fastpath active, deterministic sampler. Compatible mode (KAIRIC_EDGE +# compatibility mode) only disables the target greedy argmax fast path; +# reasoning is disabled separately. +FASTGREEDY="${FASTGREEDY:-0}" +if [ "$FASTGREEDY" = "1" ]; then + export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1 +else + export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0 +fi export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 export PROMPTFORGE_TARGET_ONLY=0 export PROMPTFORGE_MODE=iu4_ffn @@ -125,6 +135,24 @@ export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0 export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0 export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0 +# Sampling & reasoning defaults — production (FASTGREEDY=0) vs fast-greedy +# (FASTGREEDY=1). Fast greedy uses the card's default deterministic sampler +# with argmax fastpath active; production uses Qwen live values with +# reasoning budget and argmax fastpath off. +if [ "$FASTGREEDY" = "1" ]; then + TEMP=0 + TOP_P=1.0 + TOP_K=0 + MIN_P=0.0 + REASONING_ARGS="" +else + TEMP=0 + TOP_P=1.0 + TOP_K=0 + MIN_P=0.0 + REASONING_ARGS="--reasoning on --reasoning-format deepseek --reasoning-budget -1" +fi + # Pinned runner flags (qualified 262K one-slot profile, native MTP depth # 4, deterministic sampler; batch 2048 / ubatch 512 are the runner's # qualified sizes and give the fastest measured prefill). Local overrides @@ -171,11 +199,9 @@ exec "$LLAMA_BIN" \ --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \ --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT" \ --spec-draft-backend-sampling \ - --temp 0 \ - --top-p 1.0 \ - --top-k 0 \ - --min-p 0.0 \ - --reasoning on \ - --reasoning-format deepseek \ - --reasoning-budget -1 \ + --temp "$TEMP" \ + --top-p "$TOP_P" \ + --top-k "$TOP_K" \ + --min-p "$MIN_P" \ + $REASONING_ARGS \ --chat-template-kwargs '{"preserve_thinking":"true"}'