frank-small: wire SPEC_DRAFT_N_MAX and POLL_LEVEL knobs
Both were implicit build defaults (3 and 50); the throughput sweep needs to move them without editing the exec line each time. Defaults preserve current behavior exactly. Additive only.
This commit is contained in:
1 parent
23f09788e9
commit
4d13333da7
2 files changed
+12
-2
No files matched your search
@@ -90,8 +90,12 @@ NGL=999
|
|||||||
FLASH_ATTN=on
|
FLASH_ATTN=on
|
||||||
|
|
||||||
# Speculative decoding: the MTP sidecar (draft context) chained with the
|
# Speculative decoding: the MTP sidecar (draft context) chained with the
|
||||||
# draftless ngram-mod matcher. Draft depth stays llama.cpp's default of 3.
|
# draftless ngram-mod matcher. Draft depth and busy-wait level are made
|
||||||
|
# explicit here for tuning (both were the implicit build defaults: 3 and 50;
|
||||||
|
# the Gemma4 sidecar carries 4 nextn layers).
|
||||||
SPEC_TYPE=draft-mtp,ngram-mod
|
SPEC_TYPE=draft-mtp,ngram-mod
|
||||||
|
SPEC_DRAFT_N_MAX=3
|
||||||
|
POLL_LEVEL=50
|
||||||
|
|
||||||
# Greedy default (fleet contract); reasoning is extracted into
|
# Greedy default (fleet contract); reasoning is extracted into
|
||||||
# reasoning_content with an unlimited budget (the model decides).
|
# reasoning_content with an unlimited budget (the model decides).
|
||||||
|
|||||||
@@ -60,6 +60,11 @@ fi
|
|||||||
: "${CACHE_TYPE_V:=f16}"
|
: "${CACHE_TYPE_V:=f16}"
|
||||||
: "${BATCH_SIZE:=2048}"
|
: "${BATCH_SIZE:=2048}"
|
||||||
: "${UBATCH_SIZE:=512}"
|
: "${UBATCH_SIZE:=512}"
|
||||||
|
# Decode-shape knobs: the MTP drafter's per-round proposal length (build
|
||||||
|
# default 3; the Gemma4 sidecar carries 4 nextn layers) and the engine's
|
||||||
|
# busy-wait level (build default 50).
|
||||||
|
: "${SPEC_DRAFT_N_MAX:=3}"
|
||||||
|
: "${POLL_LEVEL:=50}"
|
||||||
: "${NUM_THREADS:=8}"
|
: "${NUM_THREADS:=8}"
|
||||||
: "${THREAD_BATCH:=8}"
|
: "${THREAD_BATCH:=8}"
|
||||||
: "${NGL:=999}"
|
: "${NGL:=999}"
|
||||||
@@ -90,7 +95,7 @@ fi
|
|||||||
|
|
||||||
SPEC_ARGS=()
|
SPEC_ARGS=()
|
||||||
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
|
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
|
||||||
SPEC_ARGS=(--spec-type "$SPEC_TYPE")
|
SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
||||||
fi
|
fi
|
||||||
|
|
||||||
DRAFT_ARGS=()
|
DRAFT_ARGS=()
|
||||||
@@ -122,6 +127,7 @@ exec "$LLAMA_BIN" \
|
|||||||
--ubatch-size "$UBATCH_SIZE" \
|
--ubatch-size "$UBATCH_SIZE" \
|
||||||
--threads "$NUM_THREADS" \
|
--threads "$NUM_THREADS" \
|
||||||
--threads-batch "$THREAD_BATCH" \
|
--threads-batch "$THREAD_BATCH" \
|
||||||
|
--poll "$POLL_LEVEL" \
|
||||||
--cache-type-k "$CACHE_TYPE_K" \
|
--cache-type-k "$CACHE_TYPE_K" \
|
||||||
--cache-type-v "$CACHE_TYPE_V" \
|
--cache-type-v "$CACHE_TYPE_V" \
|
||||||
"${SPEC_ARGS[@]}" \
|
"${SPEC_ARGS[@]}" \
|
||||||
|
|||||||
Reference in new issue
Block a user