diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 47564de..e6f4bf0 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -87,15 +87,14 @@ HSA_OVERRIDE_GFX_VERSION=11.5.1 VERBOSITY=4 -# MTP draft head (speculative decoding). The MTP head from -# Qwen/Qwen3.8-Flash-Next, exported as a standalone Q8_0 GGUF sidecar by -# EasiiX/Qwen3.8-Flash-Next-MTP-Strix-Halo-GGUF. Architecture qwen4exp, -# matching the main model. Measured +50% decode on code at temp 0 on -# Strix Halo (gfx1151). Set SPEC_TYPE=none to disable. -DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next -DRAFT_MODEL_FILE=mtp-Qwen3.8-Flash-Next-Q8_0.gguf +# MTP draft head (speculative decoding). agentionai +# Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST-GGUF, 2.28 GiB, ROCmFP4-FAST +# quant matched to the target. n_max 3 per the HF card (n_max 4 +# measured worse). Set SPEC_TYPE=none to disable spec decoding. +DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST +DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf SPEC_TYPE=draft-mtp,ngram-mod -SPEC_DRAFT_N_MAX=4 +SPEC_DRAFT_N_MAX=3 SPEC_DRAFT_N_MIN=0 SPEC_DRAFT_P_MIN=0.75 SPEC_DRAFT_NGL=999 diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 5dfea74..58d4ed0 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -21,10 +21,11 @@ # hyper-connection/indexer configuration; -ngl 999 places all layers on # the GPU (SVM-backed). # -# MTP draft head: EasiiX/Qwen3.8-Flash-Next-MTP-Strix-Halo-GGUF (Q8_0, -# 4.1 GB). Architecture qwen4exp, matching the main model. Speculative -# decoding via --spec-type draft-mtp,ngram-mod (n_max 4, p_min 0.75). -# Set SPEC_TYPE=none in the env to disable. +# MTP draft head: agentionai/Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST-GGUF +# (ROCmFP4-FAST, 2.28 GiB). Speculative decoding via --spec-type +# draft-mtp,ngram-mod (n_max 3, p_min 0.75). The earlier EasiiX Q8_0 +# draft was incompatible (missing blk.0.hc_attn_norm.weight). Set +# SPEC_TYPE=none in the env to disable spec decoding. # # The Qwen-Sharp chat template (scripts/chat-template.jinja) is applied at # runtime so the model emits reasoning in DeepSeek format; the reasoning @@ -70,8 +71,8 @@ fi : "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}" : "${SPEC_TYPE:=draft-mtp,ngram-mod}" : "${DRAFT_MODEL_DIR:=$MODEL_DIR}" -: "${DRAFT_MODEL_FILE:=mtp-Qwen3.8-Flash-Next-Q8_0.gguf}" -: "${SPEC_DRAFT_N_MAX:=4}" +: "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}" +: "${SPEC_DRAFT_N_MAX:=3}" : "${SPEC_DRAFT_N_MIN:=0}" : "${SPEC_DRAFT_P_MIN:=0.75}" : "${SPEC_DRAFT_NGL:=999}"