Reduce frank-default-large context for KV headroom
This commit is contained in:
1 parent
7ca9dc248e
commit
ee3c3de750
5 files changed
+18
-11
No files matched your search
@@ -39,6 +39,11 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
|
||||
grep -q '^MTP_BACKEND_SAMPLING=on$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "frank-default-large fits the model and KV cache in unified memory" {
|
||||
grep -q '^CTX_SIZE=196608$' "$ENV_FILE"
|
||||
grep -q ': "${CTX_SIZE:=262144}"' "$SERVE_FILE"
|
||||
}
|
||||
|
||||
@test "frank-default-large exposes a swappable ubatch" {
|
||||
grep -q '^UBATCH_SIZE=256$' "$ENV_FILE"
|
||||
grep -q ': "${UBATCH_SIZE:=256}"' "$SERVE_FILE"
|
||||
|
||||
Reference in new issue
Block a user