large: batch profile flipped to the runner's qualified 2048/512 (fastest measured prefill, no decode change)

This commit is contained in:
Frank committed 2026-08-22 03:30:58 -07:00
1 parent 3eb29de6ca
commit 8b3cd2ff79
4 files changed
+35 -34

No files matched your search

+6 -5
View File
@@ -50,11 +50,12 @@ LISTEN_PORT=52625
# while the prompt cache is enabled.
CTX_SIZE=262144
PARALLEL=1
# Local override: the pinned runner uses 2048/512; production chat uses a
# larger 4096/4096 batch (unified memory has headroom; matches the retired
# v3 profile).
BATCH_SIZE=4096
UBATCH_SIZE=4096
# Batch profile: the pinned runner's qualified sizes (2048/512). Benchmarks
# on 2026-08-22: ubatch 512 gives the fastest prefill at every depth
# (524/485/323 t/s at 8k/16k/64k vs 266/258/203 for ubatch 4096) with no
# decode difference. Do not raise ubatch without re-benching prefill.
BATCH_SIZE=2048
UBATCH_SIZE=512
TARGET_THREADS=16
BATCH_THREADS=32
CACHE_TYPE_K=f16