perf: frank-default-large batch capped at 2048/2048 — 3x200700 ctx KV ~50GB leaves no room for batch 12288 (OOM)

This commit is contained in:
Frank committed 2026-08-11 17:11:04 -07:00
1 parent bf64c0d9c2
commit 62cc2b850a
2 files changed
+8 -4

No files matched your search

+2 -2
View File
@@ -35,8 +35,8 @@ fi
: "${N_GPU_LAYERS:=all}"
: "${CACHE_TYPE_K:=q8_0}"
: "${CACHE_TYPE_V:=f16}"
: "${BATCH_SIZE:=12288}"
: "${UBATCH_SIZE:=4096}"
: "${BATCH_SIZE:=2048}"
: "${UBATCH_SIZE:=2048}"
: "${THREADS:=8}"
: "${THREADS_HTTP:=8}"