frank-secondary: swap to Gemma4-26B-A4B Q4_K_M, 1x200704, q8_0 KV, MTP via separate assistant file

- MODEL_FILE: Swift-1.5-Qwen3.8-27B RCO-IQ3_XXS -> Gemma4-26B-A4B Q4_K_M
- CTX_SIZE: 229376 (2x114,688) -> 200704 (1x200,704)
- PARALLEL: 2 -> 1
- CACHE_TYPE_K/V: q8_0 (unchanged, FP8 class)
- SPEC_TYPE: draft-mtp (unchanged)
- SPEC_DRAFT_FILE: new — separate MTP assistant file (mtp-gemma-4-26B-A4B-it.gguf)
- serve script: add --spec-draft-model support for separate draft files
This commit is contained in:
Frank committed 2026-10-07 12:41:07 -07:00
1 parent 41d2f56269
commit 5b7b3ccb90
2 files changed
+24 -82

No files matched your search

+5
View File
@@ -67,6 +67,7 @@ fi
# not carry half a spec pair.
: "${SPEC_TYPE:=draft-mtp}"
: "${SPEC_DRAFT_N_MAX:=3}"
: "${SPEC_DRAFT_FILE:=}"
: "${NGL:=99}"
# Vision projector. No colon on purpose: an explicitly empty value means
# "text-only, on purpose" and must NOT be replaced by the default path.
@@ -89,6 +90,10 @@ fi
# baseline must not carry half a spec pair.
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
EXTRA_ARGS+=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
if [[ -n "$SPEC_DRAFT_FILE" ]]; then
[[ -e "$SPEC_DRAFT_FILE" ]] || { echo "frank-secondary: missing $SPEC_DRAFT_FILE" >&2; exit 1; }
EXTRA_ARGS+=(--spec-draft-model "$SPEC_DRAFT_FILE")
fi
fi
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA