From 75bb9bb53e2867b23dd3831ce9b2fb68e93a537f Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 14 Aug 2026 13:59:20 -0700 Subject: [PATCH] dankclaw Qwen3.8: batch 8192/ubatch 256 - fixes MTP collapse + premature-EOS Found: large ubatch (512) collapses MTP acceptance to ~25% (gen ~9 t/s); ubatch 128 with batch>=2048 triggers premature instant-EOS under 2 concurrent slots. ubatch 256 = ~30 t/s gen, concurrent-safe, ~330 t/s prefill. --- scripts/frank-default-large.env | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index b6b2273..1c67057 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -37,10 +37,13 @@ N_GPU_LAYERS=all # mixed types silently disable FA (long-context decode collapses). CACHE_TYPE_K=q8_0 CACHE_TYPE_V=q8_0 -# Batch sizing (start conservative; tune after verifying memory): -# ubatch is the compute-graph driver; larger = more VRAM, faster prefill. -BATCH_SIZE=2048 -UBATCH_SIZE=512 +# Batch sizing (verified 2026-08-14 on Qwen3.8 UD-Q8_K_XL): +# ubatch 128 with batch>=2048 -> premature-EOS under concurrency (unsafe). +# ubatch 512 -> MTP acceptance collapses (~25%) -> gen ~9 t/s (slow). +# ubatch 256 -> ~30 t/s gen, concurrent-safe, ~330 t/s prefill. WINNER. +# batch 8192 keeps single-shot prefill flexibility; ubatch drives the graph. +BATCH_SIZE=8192 +UBATCH_SIZE=256 # Sampling settings — per Qwen3.8-27B model card thinking mode. # Thinking: temp=1.0, top_p=0.95, top_k=20, min_p=0. Clients may override. # WARNING: do not name this variable TEMP. The ROCm HIP runtime reads the