Add FASTGREEDY mode to serve script: reasoning off + argmax fastpath on for fast-greedy bench

FASTGREEDY=1 flips the serve script to the card's default profile:
- reasoning off (no logit budget overhead)
- LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1 (exact-Q8 GPU argmax)
- deterministic sampler (temp 0, top_p 1, top_k 0, min_p 0)

This mirrors the card's 'fast greedy (default)' benchmark config.
Production (FASTGREEDY=0) unchanged.
This commit is contained in:
Frank
2026-08-22 12:54:03 -07:00
parent 8b3cd2ff79
commit e0329caef8
2 changed files with 39 additions and 8 deletions
+5
View File
@@ -81,3 +81,8 @@ SPEC_DRAFT_P_SPLIT=0.10
# message.reasoning_content, matching the retired v3 service) with
# --reasoning-budget -1 (unrestricted) and chat_template_kwargs
# preserve_thinking=true (preserve thinking tokens across turns).
#
# For fast-greedy bench: comment out the --reasoning on line in serve script
# and set FASTGREEDY=1 in env (see serve script FASTGREEDY section).
# FASTGREEDY=1 flips to: reasoning off, argmax fastpath=1, sampling defaults.
# FASTGREEDY=0 (default) uses the production values above.
+34 -8
View File
@@ -100,7 +100,17 @@ export GGML_CUDA_GRAPH_OPT=0
# path rejects. Production chat therefore forces it to 0. This does NOT
# affect the Kairic Edge IU4 lane (prompt/verification route via
# --kairic-edge) nor the MTP draft fast path below.
export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0
# FASTGREEDY env var (0 = production chat, 1 = fast-greedy bench from
# card's default profile). When FASTGREEDY=1: reasoning off, argmax
# fastpath active, deterministic sampler. Compatible mode (KAIRIC_EDGE
# compatibility mode) only disables the target greedy argmax fast path;
# reasoning is disabled separately.
FASTGREEDY="${FASTGREEDY:-0}"
if [ "$FASTGREEDY" = "1" ]; then
export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1
else
export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0
fi
export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1
export PROMPTFORGE_TARGET_ONLY=0
export PROMPTFORGE_MODE=iu4_ffn
@@ -125,6 +135,24 @@ export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0
export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0
export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
# Sampling & reasoning defaults — production (FASTGREEDY=0) vs fast-greedy
# (FASTGREEDY=1). Fast greedy uses the card's default deterministic sampler
# with argmax fastpath active; production uses Qwen live values with
# reasoning budget and argmax fastpath off.
if [ "$FASTGREEDY" = "1" ]; then
TEMP=0
TOP_P=1.0
TOP_K=0
MIN_P=0.0
REASONING_ARGS=""
else
TEMP=0
TOP_P=1.0
TOP_K=0
MIN_P=0.0
REASONING_ARGS="--reasoning on --reasoning-format deepseek --reasoning-budget -1"
fi
# Pinned runner flags (qualified 262K one-slot profile, native MTP depth
# 4, deterministic sampler; batch 2048 / ubatch 512 are the runner's
# qualified sizes and give the fastest measured prefill). Local overrides
@@ -171,11 +199,9 @@ exec "$LLAMA_BIN" \
--spec-draft-p-min "$SPEC_DRAFT_P_MIN" \
--spec-draft-p-split "$SPEC_DRAFT_P_SPLIT" \
--spec-draft-backend-sampling \
--temp 0 \
--top-p 1.0 \
--top-k 0 \
--min-p 0.0 \
--reasoning on \
--reasoning-format deepseek \
--reasoning-budget -1 \
--temp "$TEMP" \
--top-p "$TOP_P" \
--top-k "$TOP_K" \
--min-p "$MIN_P" \
$REASONING_ARGS \
--chat-template-kwargs '{"preserve_thinking":"true"}'