Reduce small model context and rebalance GPU layers
This commit is contained in:
@@ -14,15 +14,15 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
|
||||
grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf' "$SERVE_FILE"
|
||||
}
|
||||
|
||||
@test "frank-default-small uses 196608 context per slot across three slots" {
|
||||
grep -q '^CTX_SIZE=589824$' "$ENV_FILE"
|
||||
grep -q '196608 ctx each' "$ENV_FILE"
|
||||
@test "frank-default-small uses 163840 context per slot across three slots" {
|
||||
grep -q '^CTX_SIZE=491520$' "$ENV_FILE"
|
||||
grep -q '163840 ctx each' "$ENV_FILE"
|
||||
grep -q '^PARALLEL=3$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "frank-default-small keeps final five expert blocks on CUDA" {
|
||||
@test "frank-default-small keeps final eight expert blocks on CUDA" {
|
||||
grep -q '^OVERRIDE_EXPERT=' "$ENV_FILE"
|
||||
grep -Fq '34|35)\.ffn_' "$ENV_FILE"
|
||||
grep -Fq '31|32)\.ffn_' "$ENV_FILE"
|
||||
grep -q '^OVERRIDE_BUFT=CPU$' "$ENV_FILE"
|
||||
grep -q '^N_GPU_LAYERS=all$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user