From b0b840b1491a785a21ecf972a2679d47ea2e7d7a Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 5 Oct 2026 17:32:22 -0700 Subject: [PATCH] frank-primary: document FRANK_PRIMARY_MNBT=2048 and the chunked-prefill flag surface --- scripts/frank-primary.env | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index ce160b2..4bb5e97 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -29,7 +29,7 @@ # container (GPU_MAX_HW_QUEUES) is held here under the prefix and passed on the # run line as `-e GPU_MAX_HW_QUEUES=…`. # - +# # --- runtime ---------------------------------------------------------------- # Digest-pinned: the floating :latest tag moved once already. Local alias # qwen-fp8-runtime:tuned-20260929 (image ID 3322dbbbc96d) points at the same @@ -93,6 +93,19 @@ FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8" # GATHER 32x64 stays: R4D_LRU_GATHER_CHUNKS=32 / R4D_LRU_GATHER_LANES=64 # measured +18.7% decode vs default geometry and the win grows with depth. # Do not drop it. +# CHUNKED PREFILL: the fork exposes the standard scheduler flags. --max-num-batched-tokens +# (MNBT) is the lever; --enable-chunked-prefill is an auto-flag (True in SchedulerConfig) +# and --prefill-schedule-interval is the gap between iterations. The running shape carries +# --max-num-batched-tokens 2048 with --max-model-len 200704, so chunked prefill is ON and +# already active: a 200704-token prompt is prefilled in ~98 iterations of 2048 tokens. +# MEASURED 2026-09-29 on the ad hoc qwen-fp8 container (same fork, same payload, TP=2): +# MNBT 8192 vs the default 2048 is NEUTRAL on decode at every depth, and small values +# re-chunk prefill only — they hurt TTFT and never touch decode, because a decode step is +# max_num_seqs x (spec+1) tokens, independent of MNBT. There is NO --ubatch-size +# equivalent in vLLM: that is a llama.cpp compute-buffer concept (the frank-small pair +# doubled on 2026-10-05 has no counterpart here). MNBT is therefore NOT a lever on this +# shape — the deployed 2048 is the fork's default — and every change re-enters the +# compile RANGE path, so FRANK_PRIMARY_CACHE_DIR pays one recompilation. # MNBT 2048 (Frank's call, 2026-10-03; the lock was 1024). The measured law # still holds: MNBT 512 doubles TTFT at 4064 ctx (2.46 s vs 1.36 s), 1024-4096 # are interchangeable on decode, 12288 costs 2.5% plus arena. 2048 is ARENA-FREE @@ -335,7 +348,7 @@ FRANK_PRIMARY_GEN_CONFIG= # --- client contract: reasoning split, tools, preserve_thinking ---------------- # Without --reasoning-parser the engine leaves the thinking text INLINE in # `content` with a dangling `` and a null `reasoning` field (measured -# 2026-10-01: content = 'We need answer...Brief.\n\n\n391') — the +# 2026-10-01: content = 'We need answer...Brief.\n\n391') — the # malformed thinking block clients choke on. The qwen3 parser splits it # properly. The GLM rollback (d91379d) had dropped all three of these; the # working values are the ones ecaad36 shipped for this model. @@ -346,4 +359,4 @@ FRANK_PRIMARY_TOOL_PARSER=qwen3_xml # preserve_thinking keeps prior thinking blocks in the rendered prompt across # turns; enable_thinking makes thinking the server-side default. Verified the # payload's tokenizer_config template consumes both keys. -FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS='{"enable_thinking": true, "preserve_thinking": true}' \ No newline at end of file +FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS='{"enable_thinking": true, "preserve_thinking": true}'