perf: frank-default-large batch 12288 / ubatch 4096

scripts/frank-default-large.env

@@ -23,7 +23,7 @@ N_GPU_LAYERS=all
 # KV on GPU, quantized (K q8_0, V f16).
 CACHE_TYPE_K=q8_0
 CACHE_TYPE_V=f16
-BATCH_SIZE=2048
+BATCH_SIZE=12288
-UBATCH_SIZE=512
+UBATCH_SIZE=4096
 THREADS=8
 THREADS_HTTP=8

scripts/serve-llama-frank-large.sh

@@ -35,8 +35,8 @@ fi
 : "${N_GPU_LAYERS:=all}"
 : "${CACHE_TYPE_K:=q8_0}"
 : "${CACHE_TYPE_V:=f16}"
-: "${BATCH_SIZE:=2048}"
+: "${BATCH_SIZE:=12288}"
-: "${UBATCH_SIZE:=512}"
+: "${UBATCH_SIZE:=4096}"
 : "${THREADS:=8}"
 : "${THREADS_HTTP:=8}"