frank-default-large: add MTP draft head (EasiiX Q8_0, draft-mtp+ngram-mod)

Add speculative decoding via EasiiX/Qwen3.8-Flash-Next-MTP-Strix-Halo-GGUF
Q8_0 sidecar (4.1 GB, qwen4exp architecture matching the main model).

- serve-frank-large.sh: SPEC_ARGS array, draft model existence check,
  spec defaults (n_max 4, p_min 0.75, ngl 999, ROCm0)
- frank-default-large.env: MTP tunables section, fix 'no MTP' comment
- kingjones-strix-rocmfp4.md: MTP Draft Head section, spec flags in
  serving profile, EasiiX + PR #27739 citations

Set SPEC_TYPE=none in the env to disable speculative decoding.
This commit is contained in:
Frank committed 2026-08-27 23:52:42 -07:00
1 parent cadc7f2768
commit fc75ca6ed8
3 files changed
+107 -5

No files matched your search

+40 -1
View File
@@ -21,6 +21,11 @@
# hyper-connection/indexer configuration; -ngl 999 places all layers on
# the GPU (SVM-backed).
#
# MTP draft head: EasiiX/Qwen3.8-Flash-Next-MTP-Strix-Halo-GGUF (Q8_0,
# 4.1 GB). Architecture qwen4exp, matching the main model. Speculative
# decoding via --spec-type draft-mtp,ngram-mod (n_max 4, p_min 0.75).
# Set SPEC_TYPE=none in the env to disable.
#
# The Qwen-Sharp chat template (scripts/chat-template.jinja) is applied at
# runtime so the model emits reasoning in DeepSeek format; the reasoning
# trail is preserved across turns via CHAT_TEMPLATE_KWARGS.
@@ -63,6 +68,16 @@ fi
: "${REASONING:=on}"
: "${REASONING_FORMAT:=deepseek}"
: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
: "${SPEC_TYPE:=draft-mtp,ngram-mod}"
: "${DRAFT_MODEL_DIR:=$MODEL_DIR}"
: "${DRAFT_MODEL_FILE:=mtp-Qwen3.8-Flash-Next-Q8_0.gguf}"
: "${SPEC_DRAFT_N_MAX:=4}"
: "${SPEC_DRAFT_N_MIN:=0}"
: "${SPEC_DRAFT_P_MIN:=0.75}"
: "${SPEC_DRAFT_NGL:=999}"
: "${SPEC_DRAFT_DEVICE:=ROCm0}"
: "${SPEC_DRAFT_TYPE_K:=f16}"
: "${SPEC_DRAFT_TYPE_V:=f16}"
MODEL="$MODEL_DIR/$MODEL_FILE"
for f in "$LLAMA_BIN" "$MODEL" "$CHAT_TEMPLATE_FILE"; do
@@ -76,6 +91,29 @@ if [[ ! -x "$LLAMA_BIN" ]]; then
exit 1
fi
# --- speculative decoding (MTP draft head) --------------------------------
# The MTP head is loaded as a separate GGUF sidecar via --model-draft.
# Set SPEC_TYPE=none in the env to disable speculative decoding entirely.
SPEC_ARGS=()
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
DRAFT_MODEL="$DRAFT_MODEL_DIR/$DRAFT_MODEL_FILE"
if [[ ! -r "$DRAFT_MODEL" ]]; then
echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
exit 1
fi
SPEC_ARGS=(
--spec-type "$SPEC_TYPE"
--model-draft "$DRAFT_MODEL"
--spec-draft-n-max "$SPEC_DRAFT_N_MAX"
--spec-draft-n-min "$SPEC_DRAFT_N_MIN"
--spec-draft-p-min "$SPEC_DRAFT_P_MIN"
--spec-draft-ngl "$SPEC_DRAFT_NGL"
--spec-draft-device "$SPEC_DRAFT_DEVICE"
--spec-draft-type-k "$SPEC_DRAFT_TYPE_K"
--spec-draft-type-v "$SPEC_DRAFT_TYPE_V"
)
fi
SERVER_DIR="$(cd -- "$(dirname -- "$LLAMA_BIN")" && pwd)"
ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
@@ -117,4 +155,5 @@ exec /usr/bin/env \
--presence-penalty "$PRESENCE_PENALTY" \
--reasoning "$REASONING" \
--reasoning-format "$REASONING_FORMAT" \
--reasoning-budget -1
--reasoning-budget -1 \
"${SPEC_ARGS[@]}"