diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 3081d90..193e535 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -43,11 +43,12 @@ ALIAS=frank-default-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7770 -# Three parallel slots, each with 160000 ctx. With PARALLEL=3 ik_llama -# limits speculative decoding to a single slot, so SPEC stays none (MTP is -# also disabled). The 3 slots triple the KV cache, so enough expert weights -# are CPU-pinned for the cache to fit on the 24 GB card. -CTX_SIZE=160000 +# Three parallel slots sharing a 491520 total context pool (491520/3 = +# 163840 per slot — the requested 160k). With PARALLEL=3 ik_llama limits +# speculative decoding to a single slot, so SPEC stays none (MTP is also +# disabled). The 3 slots + 480k pool make the KV cache large, so expert +# weights are CPU-pinned for the cache to fit on the 24 GB card. +CTX_SIZE=491520 PARALLEL=3 # Speculative decoding: DISABLED. ik_llama limits spec to a single slot; at diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 865f63d..4a4cc45 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -49,7 +49,7 @@ fi : "${ALIAS:=frank-default-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7770}" -: "${CTX_SIZE:=160000}" +: "${CTX_SIZE:=491520}" : "${PARALLEL:=3}" : "${SPEC_TYPE:=none}" : "${CACHE_TYPE_K:=q8_0}" diff --git a/tests/frank_small_config.bats b/tests/frank_small_config.bats index 59f316d..875661f 100644 --- a/tests/frank_small_config.bats +++ b/tests/frank_small_config.bats @@ -14,9 +14,10 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf' "$SERVE_FILE" } -@test "frank-default-small uses three 160000 context slots" { - grep -q '^CTX_SIZE=160000$' "$ENV_FILE" +@test "frank-default-small uses three 163840-slot context (491520 pool)" { + grep -q '^CTX_SIZE=491520$' "$ENV_FILE" grep -q '^PARALLEL=3$' "$ENV_FILE" + grep -q '163840 per slot' "$ENV_FILE" } @test "frank-default-small uses K q8_0 + V q5_0 KV cache with Hadamard" {