From 23f09788e99598bcbd819ba4bbe0e764846c4431 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 17 Sep 2026 02:56:36 -0700 Subject: [PATCH] frank-medium: MTP drafter, 393216 pool (2x196k), split 34:31 DFlash2's sidecar (1.92 GiB + buffers on ROCm1) kept the 393216 pool off the table; the MTP head lives in the target file, so its cost is one nextn layer's KV (4096 B/pos) plus small buffers. Fit: ROCm0 29.77 / ROCm1 28.32 GiB, over the floor, 34:31 the balance point at this pool. n_max 3 is the measured MTP winner. The seam's fit is now drafter-aware, the slack is the deployed integer gap (1483 MiB), the ctx cap 393216 with the rope caveat (top 131k extrapolates on the base rope). --- scripts/frank-medium.env | 71 ++++++++++++----------- tests/frank_medium_unit.bats | 105 +++++++++++++++++++---------------- 2 files changed, 92 insertions(+), 84 deletions(-) diff --git a/scripts/frank-medium.env b/scripts/frank-medium.env index bcc28d5..9da5828 100644 --- a/scripts/frank-medium.env +++ b/scripts/frank-medium.env @@ -141,30 +141,26 @@ HIP_VISIBLE_DEVICES=0,1 DEVICES=ROCm0,ROCm1 SPLIT_MODE=layer -# 36:29 -- one layer over the 35:30 balance, moved to buy headroom on ROCm1, -# the card that carries the drafter, its KV, the output layers and the heavier -# compute buffers, now that the context grew to 327680 (see below). Fit at -# the deployed point (36:29, ctx 327680, batch 6144 x ubatch 512, two slots, -# DFlash2 + ngram-mod, mmproj on ROCm0): ROCm0 29.19 / ROCm1 26.04 GiB of -# 31.86, i.e. 2.67 / 5.82 GiB free and a 3.15 GiB gap BY CONSTRUCTION -- the -# balance point at this context is 34:31 (0.11 GiB gap) and the operator chose -# headroom on the drafter card over balance. See SIZING above for the -# measurement table; one -ts step is one LAYER, worth ~0.43 GiB (DeltaNet) or -# ~1.43 GiB (attention), so do not move it without re-measuring. -TENSOR_SPLIT=36,29 +# 34:31 -- the balance point at the 393216 pool (fit gap 1.44 GiB; MTP's head +# and its KV ride ROCm1 with the last layers, so the split sends MORE layers +# to ROCm0 than the 36:29 of the DFlash2/327680 configuration). One -ts step +# is one LAYER, worth ~0.43 GiB (DeltaNet) or ~1.43 GiB (attention), so do not +# move it without re-measuring. +TENSOR_SPLIT=34,31 NGL=999 # Auto-fit has no business resizing a context we set explicitly. FIT=off FLASH_ATTN=on -# 327680 positions over TWO slots, so each conversation gets 163840 -- +# 393216 positions over TWO slots, so each conversation gets 196608 -- # --parallel DIVIDES --ctx-size, so more slots would shrink every # conversation rather than grow the cache. The checkpoint's trained window is # 262144 and the GGUF carries NO rope scaling (freq_base 1e7, no YaRN keys), -# so positions beyond 262144 extrapolate on the base rope: it runs, and 1.25x -# is a modest ask of it, but the extra 65k is untrained territory. -CTX_SIZE=327680 +# so positions beyond 262144 extrapolate on the base rope: it runs, and 1.5x +# is a modest ask of a rope, but the top 131k of the pool is untrained +# territory. +CTX_SIZE=393216 PARALLEL=2 # MEASURED 2026-09-16 with scripts/bench-batch-ubatch.sh on this box: llama-bench, @@ -238,34 +234,36 @@ REASONING_BUDGET=-1 CHAT_TEMPLATE_FILE= CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}' -# Speculative decoding: the DFlash2 block-diffusion drafter PLUS the draftless -# n-gram matcher. DFlash2's GGUF carries block_size 8, so at most 7 draft -# tokens per verification step (llama.cpp asserts n_max <= block_size - 1). -# The drafter rides ROCm1, the card that ends up holding the target's last -# layers. -SPEC_TYPE=draft-dflash,ngram-mod -SPEC_DRAFT_N_MAX=7 +# Speculative decoding: the model's OWN MTP head -- the blk..nextn.* +# tensors inside the target file. No sidecar file, no second copy of the +# weights: the draft context runs on the main model (shares_model), so the +# cost is one nextn layer's KV (4096 B/position) plus small buffers on the +# card that holds the last layers. This is what makes the 393216 pool fit: +# the DFlash2 sidecar's 1.92 GiB file and its buffers are off ROCm1. +# n_max: 3 is the measured MTP winner (see the table below -- n=7 accepts 33 % +# of 147, n=3 accepts 58 % of 72). The drafter rides ROCm1. +SPEC_TYPE=draft-mtp +SPEC_DRAFT_N_MAX=3 SPEC_DRAFT_NGL=999 SPEC_DRAFT_DEVICE=ROCm1 SPEC_DRAFT_TYPE_K=f16 SPEC_DRAFT_TYPE_V=f16 -# MTP (the model's own head) vs the drafter -- MEASURED 2026-09-16 (temperature -# 0, 1k prompt / 512 out; the autoregressive baseline and both MTP depths were -# run against the deployed configuration in one session, same binary, same -# prompts): +# MTP (the model's own head) vs the DFlash2 drafter -- MEASURED 2026-09-16 +# (temperature 0, 1k prompt / 512 out; the autoregressive baseline and both +# MTP depths were run in one session, same binary, same prompts): # # none 17.7 tok/s card1 26.07 GB # draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB # draft-mtp n=7 26.7 tok/s 33 % accepted card1 29.65 GB # draft-dflash 7 43.5 tok/s 41 % accepted card1 29.42 GB # -# All four produced IDENTICAL greedy text, so verification is lossless whichever -# proposer runs. DFlash2 is worth 2.46x over autoregressive and 34 % over the -# best MTP; deeper MTP drafting costs more than it returns (n=7 accepts 33 % of -# 147 proposals, n=3 accepts 58 % of 72). The MTP head is not free either: -# selecting it loads blk.64.nextn.* and adds ~3.3 GB to the card that holds it, -# against 2.06 GB for the drafter's file. Do not switch this without +# All four produced IDENTICAL greedy text, so verification is lossless +# whichever proposer runs. DFlash2 is the faster proposer (2.46x over +# autoregressive, 34 % over the best MTP); the operator switched to MTP on +# 2026-09-17 for the VRAM, not the speed: the MTP head adds ~3.3 GB to the +# card that holds it, against 2.06 GB for the drafter's file, and it is the +# only drafter that fits the 393216 pool. Do not switch this without # re-measuring. # # THE TWO CANNOT BE CHAINED. `--spec-type draft-mtp,draft-dflash` dies at load: @@ -278,11 +276,12 @@ SPEC_DRAFT_TYPE_V=f16 # tuning question. See serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md. # ngram-mod: the draftless n-gram matcher (CPU hash table in host RAM, no VRAM, -# no model file, no draft context). It joins the chain unconditionally -- +# no model file, no draft context). It joins the chain unconditionally -- # unlike MTP, which needs a draft context and therefore cannot share one with -# DFlash2. Verified on the loaded server: with ngram-mod as the ONLY spec type, -# a prose prompt produces ZERO drafts (plain autoregressive) while a -# prompt-echo prompt accepts 100 % of them, so its drafts are unmistakable. +# DFlash2. NOT CURRENTLY ACTIVE: SPEC_TYPE is draft-mtp alone, and the serve +# script only passes --spec-ngram-mod-* when the type contains "ngram". The +# window below is retained because the DFlash2 + ngram chain was measured free +# on prose and 2.6x on prompt-echo -- one SPEC_TYPE change brings it back. # # WINDOW MEASURED 2026-09-16 (chain at each window, decode tok/s): # diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats index 401db5d..c21912f 100755 --- a/tests/frank_medium_unit.bats +++ b/tests/frank_medium_unit.bats @@ -72,22 +72,24 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE" } -@test "the drafter runs in a chain with the draftless n-gram matcher" { - # The chain is the measured winner (see the runbook's A/B): free where the - # matcher cannot match, ~2.6x the drafter alone on prompt-echo traffic. A - # bare `draft-dflash` here would silently give that up. - grep -qE '^SPEC_TYPE=draft-dflash,ngram-mod$' "$ENV_FILE" - grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE" +@test "the drafter is the model's own MTP head, at the measured depth" { + # MTP's head lives in the target file (blk..nextn.*): no sidecar, no + # second copy of the weights, and it is the only drafter that fits the + # 393216 pool. n_max 3 is the measured MTP winner -- n=7 accepts 33 % of + # 147, n=3 accepts 58 % of 72 (the env's table; identical greedy text at + # every depth, so this is pure speed). + grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE" grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE" - grep -q -- '--spec-draft-model "$DRAFT_MODEL"' "$SERVE" - # ngram-mod's own window -- and only when the type asks for it, so a pure - # DFlash or MTP configuration is not quietly handed n-gram parameters. + local n_max + n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")" + [ -n "$n_max" ] && [ "$n_max" -eq 3 ] + # The DFlash2 sidecar stays on disk and in the fetch script -- one + # SPEC_TYPE change brings the faster chain back -- and ngram-mod's window + # is retained with it, so neither may be dropped from the env. + grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE" grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE" - grep -q '^SPEC_NGRAM_N_MIN=48$' "$ENV_FILE" - grep -q '^SPEC_NGRAM_N_MAX=64$' "$ENV_FILE" - grep -q -- '--spec-ngram-mod-n-match "$SPEC_NGRAM_N_MATCH"' "$SERVE" - grep -q -- '--spec-ngram-mod-n-min "$SPEC_NGRAM_N_MIN"' "$SERVE" - grep -q -- '--spec-ngram-mod-n-max "$SPEC_NGRAM_N_MAX"' "$SERVE" + # ...and neither is loaded: ngram-mod's flags only go out when the type + # contains ngram, so a pure MTP configuration is not handed them. python3 -c " lines = open('$SERVE').read().splitlines() guard = next(i for i, l in enumerate(lines) if '*ngram*' in l) @@ -96,11 +98,6 @@ end = next(i for i in range(guard + 1, len(lines)) if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent) assert '--spec-ngram-mod-n-match' in chr(10).join(lines[guard:end]), 'ngram flags must sit in the ngram branch' " - # DFlash2 drafts into a block of 8, so at most 7 draft tokens per - # verification step -- llama.cpp asserts n_max <= block_size - 1. - local n_max - n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")" - [ -n "$n_max" ] && [ "$n_max" -ge 1 ] && [ "$n_max" -le 7 ] } @test "a draft MODEL is passed only for DFlash types, never for MTP" { @@ -150,7 +147,8 @@ print('draft-model wiring ok') split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")" batch="$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" ubatch="$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" - [ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] + spec_type="$(sed -nE 's/^SPEC_TYPE=(.+)$/\1/p' "$ENV_FILE")" + [ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] && [ -n "$spec_type" ] [ -n "$split_hi" ] && [ -n "$split_lo" ] && [ -n "$batch" ] && [ -n "$ubatch" ] [ "$parallel" -ge 1 ] [ "$batch" -ge "$ubatch" ] @@ -160,18 +158,20 @@ print('draft-model wiring ok') # measured 6.44 / 6.90 GiB free. The fit reproduces card 1 to 0.00 GiB # and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the # deployed point is re-measured on the host after every split/ctx/batch - # move (the 2026-09-17 move to 36:29 / 327680 / mmproj is the current one). + # move (the current one is 34:31 / 393216 / MTP, 2026-09-17). # # card 0 = its share of the weights and of the KV (qwen35 spends KV on # its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim # x 2 B = 64 KiB per token at f16) plus ~1.4e6 B per ubatch unit # of compute buffers - # card 1 = the same for its own share, plus the WHOLE drafter: its weights - # (2.06e9 B), its KV -- BOUNDED BY ITS SLIDING WINDOW, so - # 20480 B/pos x 2048 positions rather than the context -- and - # ~2.5e6 B per ubatch unit of buffers on top of a ~0.89 GiB - # constant. Its buffers are the heavier ones because it holds the - # output layers and the drafter's graphs. + # card 1 = the same for its own share, plus the drafter. The drafter's + # charge depends on WHAT it is: DFlash2 is a sidecar file, so + # its weights (2.06e9 B) plus window-bounded KV (20480 B/pos x + # 2048) plus ~0.89 GiB of constant buffers; MTP's head lives in + # the target file (already charged above), so its cost is one + # nextn layer's KV (4096 B/position) plus small buffers. Its + # compute buffers are the heavier ones either way, because it + # holds the output layers and the drafter's graphs. # # BOTH ubatch coefficients are measured at the deployed shape, from the # 1536 -> 512 move: 1.30 GiB on card 0 and 2.41 GiB on card 1. That @@ -185,45 +185,54 @@ print('draft-model wiring ok') # a one-layer move creates, and so trips on moves a measurement might bless. # That is the intent: moving the split means re-measuring, not re-arguing. # - # The slack is one DeltaNet layer, rounded up. At this shape the balance - # cannot be tuned finer than one layer, so 35:30's 0.46 GiB gap IS the - # achievable optimum and a tighter slack would only reject it. + # The slack is the DEPLOYED gap, rounded up: at each shape the balance + # cannot be tuned finer than one layer, so the deployed split's gap IS + # what the check must accept, and a tighter slack would only reject it. # # What this model cannot see is CONTIGUITY -- the mechanism by which the # DSpark-era 8192/2048 failed, a 2020 MiB pp buffer that found no room while # every total looked fine. Total bytes are not the whole story for one big # allocation. - local kv_pos=65536 draft_kv_pos=20480 floor slack + local kv_pos=65536 draft_kv_pos=20480 mtp_kv_pos=4096 floor slack local card_hi card_lo diff local mmproj=931145888 # mmproj-BF16.gguf, autoloaded onto the primary device floor=$(( 1536 * 1024 * 1024 )) # 1.5 GiB per card - # One DeltaNet layer at the old shape was the finest the balance could be - # tuned, so the slack used to be one DeltaNet layer. The 2026-09-17 move - # to 36:29 at ctx 327680 traded balance for headroom on the drafter card - # (the balance point at this context is 34:31, gap 0.11 GiB; 36:29's gap - # is 3.15 GiB BY CONSTRUCTION, and the operator chose it). The slack is - # that gap, rounded up: tighter rejects the deployed split, looser stops - # being a balance check at all. - slack=$(( 3225 * 1024 * 1024 )) + # The current point is 34:31 at ctx 393216 with MTP: the fit gap is 1.44 + # GiB, which is also the balance point at this pool, so the slack is that + # gap rounded up. (The 36:29 / 327680 / DFlash2 configuration's slack was + # 3225 MiB -- its 3.15 GiB gap was by construction, and is no longer the + # deployed shape.) + slack=$(( 1483 * 1024 * 1024 )) card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \ + kv_pos * ctx * split_hi / (split_hi + split_lo) \ + 381898377 \ + mmproj \ + 1363 * 1024 * ubatch )) - card_lo=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \ - + DRAFT_BYTES \ - + kv_pos * ctx * split_lo / (split_hi + split_lo) \ - + draft_kv_pos * (ctx < 2048 ? ctx : 2048) \ - + 955046413 \ - + 2468 * 1024 * ubatch )) + if [[ "$spec_type" == *dflash* || "$spec_type" == *dspark* ]]; then + card_lo=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \ + + DRAFT_BYTES \ + + kv_pos * ctx * split_lo / (split_hi + split_lo) \ + + draft_kv_pos * (ctx < 2048 ? ctx : 2048) \ + + 955046413 \ + + 2468 * 1024 * ubatch )) + else + # MTP: the head is in the target file (charged above); one nextn + # layer's KV plus ~0.2 GiB of buffers (estimate -- re-measure on the + # host after a drafter swap, as always). + card_lo=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \ + + kv_pos * ctx * split_lo / (split_hi + split_lo) \ + + mtp_kv_pos * ctx \ + + 209715200 \ + + 2468 * 1024 * ubatch )) + fi [ "$card_hi" -le $(( CARD_BYTES - floor )) ] [ "$card_lo" -le $(( CARD_BYTES - floor )) ] diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff )) [ "$diff" -le "$slack" ] # ...and the window may not wander far past what the checkpoint was - # trained for (262144; the GGUF carries no rope scaling, so the extra 65k - # extrapolates on the base rope). - [ "$ctx" -le 327680 ] + # trained for (262144; the GGUF carries no rope scaling, so the top 131k + # of the 393216 pool extrapolates on the base rope). + [ "$ctx" -le 393216 ] } @test "vision is enabled: the BF16 projector loads with the model" {