dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-default-small: disable speculative decoding to keep 3 slots

Browse Source 
ik_llama limits speculative decoding to a single server slot (-np 1) and
hard-exits at load if a spec stage is configured with --parallel > 1:
"Speculative decoding is currently limited to a single server slot
(-np 1)." This service keeps PARALLEL=3 (one slot per user), so set
SPEC_TYPE=none to disable speculation. It is a low-parameter MoE (3B
active of 35B), so throughput is fine without it. MTP stays disabled.
KB and tests updated.
This commit is contained in:
Frank committed 2026-08-26 03:38:25 -07:00
1 parent 0e5c37f2f8
commit 4144f0f0e8
5 files changed
+44 -34
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/frank_small_config.bats
+6 -5
View File
@@ -40,17 +40,18 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
[[ "$N_GPU_LAYERS" == 999 ]]
}
@test "frank-default-small uses ngram-mod without MTP" {
# ik_llama main uses the canonical --spec-type ngram-mod:<k=v> form; the
# legacy --spec-ngram-mod-n-* flags are rejected.
grep -q '^SPEC_TYPE=ngram-mod:n_min=48,n_max=64,ngram_size_n=24$' "$ENV_FILE"
@test "frank-default-small disables speculative decoding for 3 parallel slots" {
# ik_llama limits speculative decoding to -np 1 and hard-exits with
# --parallel > 1; this service keeps 3 parallel slots and runs SPEC_TYPE=none.
grep -q '^SPEC_TYPE=none$' "$ENV_FILE"
grep -q '^PARALLEL=3$' "$ENV_FILE"
! grep -q '^SPEC_NGRAM_MOD_N_MIN=' "$ENV_FILE"
! grep -q '^SPEC_NGRAM_MOD_N_MAX=' "$ENV_FILE"
! grep -q '^SPEC_NGRAM_MOD_N_MATCH=' "$ENV_FILE"
! grep -q -- 'SPEC_DRAFT_' "$ENV_FILE"
! grep -q -- '--spec-type.*draft-mtp' "$SERVE_FILE"
grep -q -- '--spec-type' "$SERVE_FILE"
! grep -q -- '--spec-ngram-mod-' "$SERVE_FILE"
! grep -q -- 'ngram' "$SERVE_FILE"
}
@test "frank-default-small uses ubatch 1024" {