frank-secondary: swap to Gemma4-26B-A4B Q4_K_M, 1x200704, q8_0 KV, MTP via separate assistant file
- MODEL_FILE: Swift-1.5-Qwen3.8-27B RCO-IQ3_XXS -> Gemma4-26B-A4B Q4_K_M - CTX_SIZE: 229376 (2x114,688) -> 200704 (1x200,704) - PARALLEL: 2 -> 1 - CACHE_TYPE_K/V: q8_0 (unchanged, FP8 class) - SPEC_TYPE: draft-mtp (unchanged) - SPEC_DRAFT_FILE: new — separate MTP assistant file (mtp-gemma-4-26B-A4B-it.gguf) - serve script: add --spec-draft-model support for separate draft files
This commit is contained in:
1 parent
41d2f56269
commit
5b7b3ccb90
2 files changed
+24
-82
No files matched your search
@@ -67,6 +67,7 @@ fi
|
||||
# not carry half a spec pair.
|
||||
: "${SPEC_TYPE:=draft-mtp}"
|
||||
: "${SPEC_DRAFT_N_MAX:=3}"
|
||||
: "${SPEC_DRAFT_FILE:=}"
|
||||
: "${NGL:=99}"
|
||||
# Vision projector. No colon on purpose: an explicitly empty value means
|
||||
# "text-only, on purpose" and must NOT be replaced by the default path.
|
||||
@@ -89,6 +90,10 @@ fi
|
||||
# baseline must not carry half a spec pair.
|
||||
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
|
||||
EXTRA_ARGS+=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
||||
if [[ -n "$SPEC_DRAFT_FILE" ]]; then
|
||||
[[ -e "$SPEC_DRAFT_FILE" ]] || { echo "frank-secondary: missing $SPEC_DRAFT_FILE" >&2; exit 1; }
|
||||
EXTRA_ARGS+=(--spec-draft-model "$SPEC_DRAFT_FILE")
|
||||
fi
|
||||
fi
|
||||
|
||||
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
||||
|
||||
Reference in new issue
Block a user