frank-default-small: 491520 context pool (163840 per slot x 3)
--ctx-size is the total pool divided among --parallel slots, so with PARALLEL=3 the 160k target must be a 491520 pool (163840/slot) to actually deliver 160k context per slot. Env, serve default, bats updated.
This commit is contained in:
1 parent
04a8fb7806
commit
8f860d24a6
3 files changed
+10
-8
No files matched your search
@@ -43,11 +43,12 @@ ALIAS=frank-default-small
|
|||||||
BIND_ADDR=0.0.0.0
|
BIND_ADDR=0.0.0.0
|
||||||
LISTEN_PORT=7770
|
LISTEN_PORT=7770
|
||||||
|
|
||||||
# Three parallel slots, each with 160000 ctx. With PARALLEL=3 ik_llama
|
# Three parallel slots sharing a 491520 total context pool (491520/3 =
|
||||||
# limits speculative decoding to a single slot, so SPEC stays none (MTP is
|
# 163840 per slot — the requested 160k). With PARALLEL=3 ik_llama limits
|
||||||
# also disabled). The 3 slots triple the KV cache, so enough expert weights
|
# speculative decoding to a single slot, so SPEC stays none (MTP is also
|
||||||
# are CPU-pinned for the cache to fit on the 24 GB card.
|
# disabled). The 3 slots + 480k pool make the KV cache large, so expert
|
||||||
CTX_SIZE=160000
|
# weights are CPU-pinned for the cache to fit on the 24 GB card.
|
||||||
|
CTX_SIZE=491520
|
||||||
PARALLEL=3
|
PARALLEL=3
|
||||||
|
|
||||||
# Speculative decoding: DISABLED. ik_llama limits spec to a single slot; at
|
# Speculative decoding: DISABLED. ik_llama limits spec to a single slot; at
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ fi
|
|||||||
: "${ALIAS:=frank-default-small}"
|
: "${ALIAS:=frank-default-small}"
|
||||||
: "${BIND_ADDR:=0.0.0.0}"
|
: "${BIND_ADDR:=0.0.0.0}"
|
||||||
: "${LISTEN_PORT:=7770}"
|
: "${LISTEN_PORT:=7770}"
|
||||||
: "${CTX_SIZE:=160000}"
|
: "${CTX_SIZE:=491520}"
|
||||||
: "${PARALLEL:=3}"
|
: "${PARALLEL:=3}"
|
||||||
: "${SPEC_TYPE:=none}"
|
: "${SPEC_TYPE:=none}"
|
||||||
: "${CACHE_TYPE_K:=q8_0}"
|
: "${CACHE_TYPE_K:=q8_0}"
|
||||||
|
|||||||
@@ -14,9 +14,10 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
|
|||||||
grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf' "$SERVE_FILE"
|
grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf' "$SERVE_FILE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "frank-default-small uses three 160000 context slots" {
|
@test "frank-default-small uses three 163840-slot context (491520 pool)" {
|
||||||
grep -q '^CTX_SIZE=160000$' "$ENV_FILE"
|
grep -q '^CTX_SIZE=491520$' "$ENV_FILE"
|
||||||
grep -q '^PARALLEL=3$' "$ENV_FILE"
|
grep -q '^PARALLEL=3$' "$ENV_FILE"
|
||||||
|
grep -q '163840 per slot' "$ENV_FILE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "frank-default-small uses K q8_0 + V q5_0 KV cache with Hadamard" {
|
@test "frank-default-small uses K q8_0 + V q5_0 KV cache with Hadamard" {
|
||||||
|
|||||||
Reference in new issue
Block a user