diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index e6f4bf0..2633505 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -87,13 +87,14 @@ HSA_OVERRIDE_GFX_VERSION=11.5.1 VERBOSITY=4 -# MTP draft head (speculative decoding). agentionai -# Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST-GGUF, 2.28 GiB, ROCmFP4-FAST -# quant matched to the target. n_max 3 per the HF card (n_max 4 -# measured worse). Set SPEC_TYPE=none to disable spec decoding. +# MTP draft head (speculative decoding) — BOTH attempted heads are +# INCOMPATIBLE with the STRIX_LEAN Uncensored model (missing tensor +# blk.0.hc_attn_norm.weight): the EasiiX Q8_0 and the agentionai +# ROCmFP4-FAST. Using ngram-mod only until a compatible MTP head +# is found. Set SPEC_TYPE=none to disable spec decoding entirely. DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf -SPEC_TYPE=draft-mtp,ngram-mod +SPEC_TYPE=ngram-mod SPEC_DRAFT_N_MAX=3 SPEC_DRAFT_N_MIN=0 SPEC_DRAFT_P_MIN=0.75 diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 58d4ed0..948541d 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -21,11 +21,11 @@ # hyper-connection/indexer configuration; -ngl 999 places all layers on # the GPU (SVM-backed). # -# MTP draft head: agentionai/Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST-GGUF -# (ROCmFP4-FAST, 2.28 GiB). Speculative decoding via --spec-type -# draft-mtp,ngram-mod (n_max 3, p_min 0.75). The earlier EasiiX Q8_0 -# draft was incompatible (missing blk.0.hc_attn_norm.weight). Set -# SPEC_TYPE=none in the env to disable spec decoding. +# MTP draft head: both attempted heads (EasiiX Q8_0 and agentionai +# ROCmFP4-FAST) are INCOMPATIBLE with the STRIX_LEAN model — the +# qwen4exp binary expects blk.0.hc_attn_norm.weight which neither +# draft GGUF carries. Using ngram-mod only. Set SPEC_TYPE=none in +# the env to disable spec decoding. # # The Qwen-Sharp chat template (scripts/chat-template.jinja) is applied at # runtime so the model emits reasoning in DeepSeek format; the reasoning @@ -69,7 +69,7 @@ fi : "${REASONING:=on}" : "${REASONING_FORMAT:=deepseek}" : "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}" -: "${SPEC_TYPE:=draft-mtp,ngram-mod}" +: "${SPEC_TYPE:=ngram-mod}" : "${DRAFT_MODEL_DIR:=$MODEL_DIR}" : "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}" : "${SPEC_DRAFT_N_MAX:=3}"