perf: frank-default-large batch 12288 / ubatch 4096

2026-08-11 16:36:39 -07:00
parent f5a959bfa2
commit bf64c0d9c2
2 changed files with 4 additions and 4 deletions
@@ -23,7 +23,7 @@ N_GPU_LAYERS=all
 # KV on GPU, quantized (K q8_0, V f16).
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=f16
-BATCH_SIZE=2048
-UBATCH_SIZE=512
+BATCH_SIZE=12288
+UBATCH_SIZE=4096
 THREADS=8
 THREADS_HTTP=8
@@ -35,8 +35,8 @@ fi
 : "${N_GPU_LAYERS:=all}"
 : "${CACHE_TYPE_K:=q8_0}"
 : "${CACHE_TYPE_V:=f16}"
-: "${BATCH_SIZE:=2048}"
-: "${UBATCH_SIZE:=512}"
+: "${BATCH_SIZE:=12288}"
+: "${UBATCH_SIZE:=4096}"
 : "${THREADS:=8}"
 : "${THREADS_HTTP:=8}"