frank-small: wire SPEC_DRAFT_N_MAX and POLL_LEVEL knobs
Both were implicit build defaults (3 and 50); the throughput sweep needs to move them without editing the exec line each time. Defaults preserve current behavior exactly. Additive only.
This commit is contained in:
@@ -90,8 +90,12 @@ NGL=999
|
||||
FLASH_ATTN=on
|
||||
|
||||
# Speculative decoding: the MTP sidecar (draft context) chained with the
|
||||
# draftless ngram-mod matcher. Draft depth stays llama.cpp's default of 3.
|
||||
# draftless ngram-mod matcher. Draft depth and busy-wait level are made
|
||||
# explicit here for tuning (both were the implicit build defaults: 3 and 50;
|
||||
# the Gemma4 sidecar carries 4 nextn layers).
|
||||
SPEC_TYPE=draft-mtp,ngram-mod
|
||||
SPEC_DRAFT_N_MAX=3
|
||||
POLL_LEVEL=50
|
||||
|
||||
# Greedy default (fleet contract); reasoning is extracted into
|
||||
# reasoning_content with an unlimited budget (the model decides).
|
||||
|
||||
@@ -60,6 +60,11 @@ fi
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${BATCH_SIZE:=2048}"
|
||||
: "${UBATCH_SIZE:=512}"
|
||||
# Decode-shape knobs: the MTP drafter's per-round proposal length (build
|
||||
# default 3; the Gemma4 sidecar carries 4 nextn layers) and the engine's
|
||||
# busy-wait level (build default 50).
|
||||
: "${SPEC_DRAFT_N_MAX:=3}"
|
||||
: "${POLL_LEVEL:=50}"
|
||||
: "${NUM_THREADS:=8}"
|
||||
: "${THREAD_BATCH:=8}"
|
||||
: "${NGL:=999}"
|
||||
@@ -90,7 +95,7 @@ fi
|
||||
|
||||
SPEC_ARGS=()
|
||||
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
|
||||
SPEC_ARGS=(--spec-type "$SPEC_TYPE")
|
||||
SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
||||
fi
|
||||
|
||||
DRAFT_ARGS=()
|
||||
@@ -122,6 +127,7 @@ exec "$LLAMA_BIN" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--threads "$NUM_THREADS" \
|
||||
--threads-batch "$THREAD_BATCH" \
|
||||
--poll "$POLL_LEVEL" \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
"${SPEC_ARGS[@]}" \
|
||||
|
||||
Reference in New Issue
Block a user