dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-default-small: PARALLEL=3 160k, KV K q8_0 / V q5_0 + Hadamard

Browse Source 
Reconfigure to 3 parallel 160000-ctx slots (was 1 slot). KV cache is
now asymmetric: K at q8_0, V at q5_0, both with Hadamard. The 3 slots
triple the KV, so expert blocks 27-34 are pinned back to CPU (freed
~8.7 GiB) leaving output-adjacent 35-40 on CUDA. Q8_0 model,
split-mode none, spec disabled (ik limits spec to 1 slot). AGENTS.md:
drop the scratch-port rule (test/bench processes cannot share the
single GPU with the running service anyway). Env, serve script, bats,
docs updated.
This commit is contained in:
Frank committed 2026-08-26 05:43:35 -07:00
1 parent 8c41c821c4
commit 04a8fb7806
5 files changed
+84 -68
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/frank_small_config.bats
+18 -13
View File
@@ -14,41 +14,46 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf' "$SERVE_FILE"
}
@test "frank-default-small uses a single 160000 context slot" {
@test "frank-default-small uses three 160000 context slots" {
grep -q '^CTX_SIZE=160000$' "$ENV_FILE"
grep -q '^PARALLEL=1$' "$ENV_FILE"
grep -q '^PARALLEL=3$' "$ENV_FILE"
}
@test "frank-default-small uses q6_0 KV cache with Hadamard" {
grep -q '^CACHE_TYPE_K=q6_0$' "$ENV_FILE"
grep -q '^CACHE_TYPE_V=q6_0$' "$ENV_FILE"
@test "frank-default-small uses K q8_0 + V q5_0 KV cache with Hadamard" {
grep -q '^CACHE_TYPE_K=q8_0$' "$ENV_FILE"
grep -q '^CACHE_TYPE_V=q5_0$' "$ENV_FILE"
grep -q '^K_CACHE_HADAMARD=1$' "$ENV_FILE"
grep -q '^V_CACHE_HADAMARD=1$' "$ENV_FILE"
}
@test "frank-default-small keeps full-attention and final blocks on CUDA" {
source "$ENV_FILE"
[[ "$OVERRIDE_EXPERT" == 'blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26)\.ffn_(up|down|gate|gate_up)_(ch|)exps' ]]
[[ "$OVERRIDE_EXPERT" == 'blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|27|28|29|30|31|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps' ]]
# output-adjacent final blocks stay on CUDA
[[ "$OVERRIDE_EXPERT" != *'|35|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|36|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|37|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|38|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|39|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|40|'* ]]
# full-attention blocks (3,7,11,15,19,23) are untouched (no exps)
[[ "$OVERRIDE_EXPERT" != *'|3|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|7|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|11|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|15|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|19|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|23|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|27|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|31|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|35|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|36|'* ]]
[[ "$OVERRIDE_EXPERT" != *'|40|'* ]]
[[ "$OVERRIDE_BUFT" == CPU ]]
# ik_llama --gpu-layers/-ngl requires an integer ("all" is rejected); 999
# loads as many layers as possible while OVERRIDE_EXPERT pins exps to CPU.
[[ "$N_GPU_LAYERS" == 999 ]]
}
@test "frank-default-small disables speculative decoding (single slot)" {
@test "frank-default-small disables speculative decoding (3 parallel slots)" {
# ik_llama limits speculative decoding to -np 1; with PARALLEL=3 a
# configured spec stage hard-exits, so SPEC_TYPE=none.
grep -q '^SPEC_TYPE=none$' "$ENV_FILE"
grep -q '^PARALLEL=1$' "$ENV_FILE"
grep -q '^PARALLEL=3$' "$ENV_FILE"
! grep -q '^SPEC_NGRAM_MOD_N_MIN=' "$ENV_FILE"
! grep -q '^SPEC_NGRAM_MOD_N_MAX=' "$ENV_FILE"
! grep -q '^SPEC_NGRAM_MOD_N_MATCH=' "$ENV_FILE"