diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index b6b2273..1c67057 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -37,10 +37,13 @@ N_GPU_LAYERS=all # mixed types silently disable FA (long-context decode collapses). CACHE_TYPE_K=q8_0 CACHE_TYPE_V=q8_0 -# Batch sizing (start conservative; tune after verifying memory): -# ubatch is the compute-graph driver; larger = more VRAM, faster prefill. -BATCH_SIZE=2048 -UBATCH_SIZE=512 +# Batch sizing (verified 2026-08-14 on Qwen3.8 UD-Q8_K_XL): +# ubatch 128 with batch>=2048 -> premature-EOS under concurrency (unsafe). +# ubatch 512 -> MTP acceptance collapses (~25%) -> gen ~9 t/s (slow). +# ubatch 256 -> ~30 t/s gen, concurrent-safe, ~330 t/s prefill. WINNER. +# batch 8192 keeps single-shot prefill flexibility; ubatch drives the graph. +BATCH_SIZE=8192 +UBATCH_SIZE=256 # Sampling settings — per Qwen3.8-27B model card thinking mode. # Thinking: temp=1.0, top_p=0.95, top_k=20, min_p=0. Clients may override. # WARNING: do not name this variable TEMP. The ROCm HIP runtime reads the