large: batch profile flipped to the runner's qualified 2048/512 (fastest measured prefill, no decode change)
This commit is contained in:
1 parent
3eb29de6ca
commit
8b3cd2ff79
4 files changed
+35
-34
No files matched your search
@@ -50,11 +50,12 @@ LISTEN_PORT=52625
|
||||
# while the prompt cache is enabled.
|
||||
CTX_SIZE=262144
|
||||
PARALLEL=1
|
||||
# Local override: the pinned runner uses 2048/512; production chat uses a
|
||||
# larger 4096/4096 batch (unified memory has headroom; matches the retired
|
||||
# v3 profile).
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=4096
|
||||
# Batch profile: the pinned runner's qualified sizes (2048/512). Benchmarks
|
||||
# on 2026-08-22: ubatch 512 gives the fastest prefill at every depth
|
||||
# (524/485/323 t/s at 8k/16k/64k vs 266/258/203 for ubatch 4096) with no
|
||||
# decode difference. Do not raise ubatch without re-benching prefill.
|
||||
BATCH_SIZE=2048
|
||||
UBATCH_SIZE=512
|
||||
TARGET_THREADS=16
|
||||
BATCH_THREADS=32
|
||||
CACHE_TYPE_K=f16
|
||||
|
||||
Reference in new issue
Block a user