perf: frank-default-large batch capped at 2048/2048 — 3x200700 ctx KV ~50GB leaves no room for batch 12288 (OOM)
This commit is contained in:
1 parent
bf64c0d9c2
commit
62cc2b850a
2 files changed
+8
-4
No files matched your search
@@ -35,8 +35,8 @@ fi
|
||||
: "${N_GPU_LAYERS:=all}"
|
||||
: "${CACHE_TYPE_K:=q8_0}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${BATCH_SIZE:=12288}"
|
||||
: "${UBATCH_SIZE:=4096}"
|
||||
: "${BATCH_SIZE:=2048}"
|
||||
: "${UBATCH_SIZE:=2048}"
|
||||
: "${THREADS:=8}"
|
||||
: "${THREADS_HTTP:=8}"
|
||||
|
||||
|
||||
Reference in new issue
Block a user