Reduce small model context and rebalance GPU layers

This commit is contained in:
Frank
2026-08-25 00:59:41 -07:00
parent 7a17984dbc
commit 339325719e
7 changed files with 38 additions and 35 deletions
+5 -5
View File
@@ -14,15 +14,15 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf' "$SERVE_FILE"
}
@test "frank-default-small uses 196608 context per slot across three slots" {
grep -q '^CTX_SIZE=589824$' "$ENV_FILE"
grep -q '196608 ctx each' "$ENV_FILE"
@test "frank-default-small uses 163840 context per slot across three slots" {
grep -q '^CTX_SIZE=491520$' "$ENV_FILE"
grep -q '163840 ctx each' "$ENV_FILE"
grep -q '^PARALLEL=3$' "$ENV_FILE"
}
@test "frank-default-small keeps final five expert blocks on CUDA" {
@test "frank-default-small keeps final eight expert blocks on CUDA" {
grep -q '^OVERRIDE_EXPERT=' "$ENV_FILE"
grep -Fq '34|35)\.ffn_' "$ENV_FILE"
grep -Fq '31|32)\.ffn_' "$ENV_FILE"
grep -q '^OVERRIDE_BUFT=CPU$' "$ENV_FILE"
grep -q '^N_GPU_LAYERS=all$' "$ENV_FILE"
}