frank-default-small: 491520 context pool (163840 per slot x 3)
--ctx-size is the total pool divided among --parallel slots, so with PARALLEL=3 the 160k target must be a 491520 pool (163840/slot) to actually deliver 160k context per slot. Env, serve default, bats updated.
This commit is contained in:
@@ -43,11 +43,12 @@ ALIAS=frank-default-small
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
# Three parallel slots, each with 160000 ctx. With PARALLEL=3 ik_llama
|
||||
# limits speculative decoding to a single slot, so SPEC stays none (MTP is
|
||||
# also disabled). The 3 slots triple the KV cache, so enough expert weights
|
||||
# are CPU-pinned for the cache to fit on the 24 GB card.
|
||||
CTX_SIZE=160000
|
||||
# Three parallel slots sharing a 491520 total context pool (491520/3 =
|
||||
# 163840 per slot — the requested 160k). With PARALLEL=3 ik_llama limits
|
||||
# speculative decoding to a single slot, so SPEC stays none (MTP is also
|
||||
# disabled). The 3 slots + 480k pool make the KV cache large, so expert
|
||||
# weights are CPU-pinned for the cache to fit on the 24 GB card.
|
||||
CTX_SIZE=491520
|
||||
PARALLEL=3
|
||||
|
||||
# Speculative decoding: DISABLED. ik_llama limits spec to a single slot; at
|
||||
|
||||
@@ -49,7 +49,7 @@ fi
|
||||
: "${ALIAS:=frank-default-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7770}"
|
||||
: "${CTX_SIZE:=160000}"
|
||||
: "${CTX_SIZE:=491520}"
|
||||
: "${PARALLEL:=3}"
|
||||
: "${SPEC_TYPE:=none}"
|
||||
: "${CACHE_TYPE_K:=q8_0}"
|
||||
|
||||
@@ -14,9 +14,10 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
|
||||
grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf' "$SERVE_FILE"
|
||||
}
|
||||
|
||||
@test "frank-default-small uses three 160000 context slots" {
|
||||
grep -q '^CTX_SIZE=160000$' "$ENV_FILE"
|
||||
@test "frank-default-small uses three 163840-slot context (491520 pool)" {
|
||||
grep -q '^CTX_SIZE=491520$' "$ENV_FILE"
|
||||
grep -q '^PARALLEL=3$' "$ENV_FILE"
|
||||
grep -q '163840 per slot' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "frank-default-small uses K q8_0 + V q5_0 KV cache with Hadamard" {
|
||||
|
||||
Reference in New Issue
Block a user