diff --git a/scripts/frank-medium.env b/scripts/frank-medium.env index d052eeb..091fbb3 100644 --- a/scripts/frank-medium.env +++ b/scripts/frank-medium.env @@ -137,21 +137,24 @@ REASONING_BUDGET=-1 CHAT_TEMPLATE_FILE= CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}' -# Speculative decoding: the DFlash2 block-diffusion drafter. Its GGUF carries -# block_size 8, so at most 7 draft tokens per verification step (llama.cpp -# asserts n_max <= block_size - 1); 7 is also what the drafter was published -# with. The drafter rides ROCm1, the card that ends up holding the target's -# last layers -- the hidden states the proposal consumes. -SPEC_TYPE=draft-dflash +# Speculative decoding: the DFlash2 block-diffusion drafter PLUS the draftless +# n-gram matcher. Measured against the drafter alone on 2026-09-16 (tables +# below): the chain is byte-identical to DFlash2 by itself where n-grams cannot +# match, and ~2.6x it where they can. DFlash2's GGUF carries block_size 8, so at +# most 7 draft tokens per verification step (llama.cpp asserts +# n_max <= block_size - 1); 7 is also what the drafter was published with. The +# drafter rides ROCm1, the card that ends up holding the target's last layers. +SPEC_TYPE=draft-dflash,ngram-mod SPEC_DRAFT_N_MAX=7 SPEC_DRAFT_NGL=999 SPEC_DRAFT_DEVICE=ROCm1 SPEC_DRAFT_TYPE_K=f16 SPEC_DRAFT_TYPE_V=f16 -# MTP vs the drafter, MEASURED 2026-09-16 (temperature 0, 1k prompt / 512 out; -# the autoregressive baseline and both MTP depths were run against the deployed -# configuration in one session, same binary, same prompts): +# MTP (the model's own head) vs the drafter -- MEASURED 2026-09-16 (temperature +# 0, 1k prompt / 512 out; the autoregressive baseline and both MTP depths were +# run against the deployed configuration in one session, same binary, same +# prompts): # # none 17.7 tok/s card1 26.07 GB # draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB @@ -175,12 +178,29 @@ SPEC_DRAFT_TYPE_V=f16 # the target model ..."), so a chain of the two is a structural dead end, not a # tuning question. See serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md. -# ngram-mod: the draftless n-gram matcher (CPU hash table, no VRAM, no model -# file). It is a legitimate second chain member -- unlike MTP it needs no draft -# context, so `--spec-type draft-dflash,ngram-mod` is structurally allowed. The -# window below is the fleet's tuned one (n_min/n_max 48/64 match long echoes, -# which is the only traffic where n-grams pay). Measured against DFlash2 on -# 2026-09-16 -- see the runbook for the verdict before enabling it. +# ngram-mod: the draftless n-gram matcher (CPU hash table in host RAM, no VRAM, +# no model file, no draft context). It joins the chain unconditionally -- +# unlike MTP, which needs a draft context and therefore cannot share one with +# DFlash2. Verified on the loaded server: with ngram-mod as the ONLY spec type, +# a prose prompt produces ZERO drafts (plain autoregressive) while a +# prompt-echo prompt accepts 100 % of them, so its drafts are unmistakable. +# +# WINDOW MEASURED 2026-09-16 (chain at each window, decode tok/s): +# +# n_min/n_max prose 1k/512 prompt-echo 1.5k/400 +# 48/64 43.9 200.6 <- deployed +# 16/32 36.0 161.0 +# 8/16 36.2 113.8 +# +# The long window is the right one, and not only because it wins on echo: a +# SHORT window makes the matcher propose on short fragments of ordinary prose, +# where the proposals are rejected and the verification still costs -- which is +# the "ngram hurts" result the fleet measured on other payloads twice before. +# With 48/64 the chain is byte-identical to DFlash2 alone on prose (same 119 +# drafts / 49 accepted) and on the identity prompt (70/53). +# +# Do not shorten these without re-running scripts/bench-spec-ab.py: the +# no-penalty property is what makes the chain safe as a default. SPEC_NGRAM_N_MATCH=24 SPEC_NGRAM_N_MIN=48 SPEC_NGRAM_N_MAX=64 diff --git a/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md b/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md index fcad3e5..60710d2 100644 --- a/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md +++ b/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md @@ -242,6 +242,61 @@ same binary, identical prompts at `temperature 0`, unit stopped between runs the draft model (`creating MTP draft context against the target model ...`). That is structural, not a tuning question. +### Chaining the draftless n-gram matcher (deployed) + +`ngram-mod` **can** ride beside DFlash2, and it is enabled. The reason is in +`common/speculative.cpp`: the n-gram types are added to the config vector +unconditionally while every model-backed drafter is gated on +`ctx_dft != nullptr` — so n-gram needs no draft context and cannot collide with +the drafter over one. (MTP is the opposite: it *is* context-backed, which is why +`draft-mtp,draft-dflash` cannot load.) The vector's fixed order — ngram types +first, then the drafters — also means CLI order is irrelevant. + +Measured with `scripts/bench-spec-ab.py`, decode tok/s, three shapes that a +proposer can and cannot match: + +| Configuration | prose 1k/512 | echo 1.5k/400 (copy 0.94) | structured/400 | identity | +|---|---:|---:|---:|---:| +| `draft-dflash@7` | 43.77 | 76.82 | 91.08 | 70.37 | +| `ngram-mod` alone | 17.72 *(0 drafts)* | 102.50 | 17.80 *(0 drafts)* | 17.77 | +| **`draft-dflash,ngram-mod@7`** | **43.65** | **200.58** | 91.22 | 70.40 | + +- **On prompt-echo traffic the chain is 2.6× the drafter alone** (200.6 vs 76.8) + and twice `ngram-mod` by itself, at 100 % acceptance. +- **On prose it is indistinguishable from DFlash2 alone** — the same 119 drafts + and 49 accepts, the same 43.7 tok/s. That is the property that makes it safe + as a default: where the matcher cannot match, it costs nothing. +- `ngram-mod` alone is a *negative control*: zero drafts on prose and identity + (17.7 tok/s = the autoregressive baseline), 100 % acceptance on echo. Nothing + else could have proposed those tokens, which is how its drafts are identified + at all — see the trap below. +- VRAM is unchanged (29.42 GB on card 1 with or without it): the matcher is a + 64 MiB host-RAM hash table. +- `structured` (25 records with unique ids) rewards no proposer: its lines + differ, so there is no long span to match. It is a negative control for + *self-repetition*, not evidence about it. + +**The window matters, and shorter is not better.** The chain at three windows: + +| `SPEC_NGRAM_N_MIN`/`N_MAX` | prose 1k/512 | echo 1.5k/400 | +|---|---:|---:| +| **48/64 — deployed** | **43.9** | **200.6** | +| 16/32 | 36.0 | 161.0 | +| 8/16 | 36.2 | 113.8 | + +A short window makes the matcher propose on ordinary prose fragments, where the +proposals are rejected and the verification is still paid — the "ngram hurts" +result this fleet measured twice before on other payloads. At 48/64 the chain +keeps DFlash2's exact numbers on prose and wins on echo. + +> **Operational trap: `ngram-mod` prints no startup banner in this build.** +> `journalctl -u frank-medium` shows `adding speculative implementation +> 'draft-dflash'` and nothing for the matcher — the only evidence it is running +> is behavioural (`draft_n` > 0 where text repeats) or the per-task line +> `slot print_timing: ... draft acceptance = 1.00000 (256 accepted / 256 +> generated), mean len = 65.00`. Do not conclude from the load log that the +> chain is not engaged. + ### The agent-loop shape (fleet harness, 2 families × 2 turns) `scripts/bench-turn-latency.py http://127.0.0.1:7770 2 2 LAYERSPLIT` — two diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats index 6fcf7f8..d57e4cf 100755 --- a/tests/frank_medium_unit.bats +++ b/tests/frank_medium_unit.bats @@ -61,11 +61,30 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE" } -@test "the DFlash2 drafter is a draft model at a depth its block allows" { - grep -q '^SPEC_TYPE=draft-dflash$' "$ENV_FILE" +@test "the drafter runs in a chain with the draftless n-gram matcher" { + # The chain is the measured winner (see the runbook's A/B): free where the + # matcher cannot match, ~2.6x the drafter alone on prompt-echo traffic. A + # bare `draft-dflash` here would silently give that up. + grep -qE '^SPEC_TYPE=draft-dflash,ngram-mod$' "$ENV_FILE" grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE" grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE" grep -q -- '--spec-draft-model "$DRAFT_MODEL"' "$SERVE" + # ngram-mod's own window -- and only when the type asks for it, so a pure + # DFlash or MTP configuration is not quietly handed n-gram parameters. + grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE" + grep -q '^SPEC_NGRAM_N_MIN=48$' "$ENV_FILE" + grep -q '^SPEC_NGRAM_N_MAX=64$' "$ENV_FILE" + grep -q -- '--spec-ngram-mod-n-match "$SPEC_NGRAM_N_MATCH"' "$SERVE" + grep -q -- '--spec-ngram-mod-n-min "$SPEC_NGRAM_N_MIN"' "$SERVE" + grep -q -- '--spec-ngram-mod-n-max "$SPEC_NGRAM_N_MAX"' "$SERVE" + python3 -c " +lines = open('$SERVE').read().splitlines() +guard = next(i for i, l in enumerate(lines) if '*ngram*' in l) +indent = len(lines[guard]) - len(lines[guard].lstrip()) +end = next(i for i in range(guard + 1, len(lines)) + if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent) +assert '--spec-ngram-mod-n-match' in chr(10).join(lines[guard:end]), 'ngram flags must sit in the ngram branch' +" # DFlash2 drafts into a block of 8, so at most 7 draft tokens per # verification step -- llama.cpp asserts n_max <= block_size - 1. local n_max