From 34253a9903395f6247e449e51c68bb5e8cc02a11 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2026 16:15:59 -0700 Subject: [PATCH] frank-medium: the drafter beats MTP and MTP beats nothing -- measured, not assumed Four configurations of the same binary, identical prompts at temperature 0, the unit stopped between runs, on the deployed payload: none 17.7 tok/s card1 26.07 GB draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB draft-mtp n=7 26.7 tok/s 33 % accepted card1 29.65 GB draft-dflash 7 43.5 tok/s 41 % accepted card1 29.42 GB DFlash2 is worth 2.46x over autoregressive and 34 % over the best MTP, and all four produced IDENTICAL greedy text, so the 'lossless verification' claim in the runbook is now a check rather than an assumption. MTP is not the cheaper option either: its head loads ~3.3 GB against the drafter's 2.06 GB file, and deeper drafting loses (n=7 accepts a third of its proposals, n=3 more than half). The two CANNOT be chained, and the failure is structural rather than tunable: 'spec-type draft-mtp,draft-dflash' dies at load with 'context type MTP requested but model doesn't contain MTP layers', because a chain has ONE draft context and MTP builds against whatever model that context holds -- here the DFlash2 file. The serve script therefore passes --spec-draft-model in the dflash branch ONLY, and the seam now asserts exactly that (the wiring whose violation is this error), plus that the deployment stays on the measured winner. scripts/bench-spec-ab.py makes the comparison reproducible: the benchmarking policy wants the runtime's own harness for this, and the numbers above were otherwise unreproducible one-offs. --- scripts/bench-spec-ab.py | 192 ++++++++++++++++++ scripts/frank-medium.env | 26 +++ .../LLM/frank-medium-llamacpp-dflash2.md | 31 +++ tests/frank_medium_unit.bats | 20 ++ 4 files changed, 269 insertions(+) create mode 100755 scripts/bench-spec-ab.py diff --git a/scripts/bench-spec-ab.py b/scripts/bench-spec-ab.py new file mode 100755 index 0000000..1b4a758 --- /dev/null +++ b/scripts/bench-spec-ab.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""bench-spec-ab.py — settle "which proposer should this endpoint use?" with data. + +Runs one payload through several speculative-decoding configurations of the same +binary, one at a time, with the unit stopped between runs: on a single-GPU host +there is no room for two copies of a 29 GiB model, and the benchmarking policy +applies. Every configuration gets identical prompts at temperature 0, and the +greedy text of a fixed prompt is compared across them — speculation may buy +speed, never change the answer. + + bench-spec-ab.py [--base-url URL] + bench-spec-ab.py /home/dank/serverAppConfigs/scripts/frank-medium.env \ + /home/dank/serverAppConfigs/scripts/frank-medium.sh + +The configurations are the ones that exist for THIS payload (frank-medium, +2026-09-16): the DFlash2 drafter, the model's own MTP head at two depths, and +autoregressive. The systemd unit is restored at the end, so the endpoint comes +back on whatever the env file says whatever happens here. + +Result on this box (1k prompt / 512 out, temperature 0): + + none 17.7 tok/s card1 26.07 GB + draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB + draft-mtp n=7 26.7 tok/s 33 % accepted card1 29.65 GB + draft-dflash 7 43.5 tok/s 41 % accepted card1 29.42 GB + +...and `draft-mtp,draft-dflash` fails at load: a chain has ONE draft context and +MTP builds against whatever model that context holds. +""" +import argparse +import json +import re +import subprocess +import sys +import time +import urllib.request + +FILLER = ("The quick brown fox jumps over the lazy dog near the river bank. " + "A cache is a component that stores data so future requests are faster. ") +TOKENS_PER_REP = 28 +UNIT = "frank-medium.service" +MODEL = "frank-medium" +SPEC_GREP = "adding speculative implementation|- n_max=|- block_size=|creating MTP" + +CONFIGS = [ + ("none", {"SPEC_TYPE": "none"}), + ("mtp-n3", {"SPEC_TYPE": "draft-mtp", "SPEC_DRAFT_N_MAX": "3"}), + ("mtp-n7", {"SPEC_TYPE": "draft-mtp", "SPEC_DRAFT_N_MAX": "7"}), + ("dflash-n7", {"SPEC_TYPE": "draft-dflash", "SPEC_DRAFT_N_MAX": "7"}), +] + + +def log(msg, logfile): + line = "[" + time.strftime("%H:%M:%S") + "] " + str(msg) + print(line, flush=True) + with open(logfile, "a") as fh: + fh.write(line + "\n") + + +def sh(cmd): + return subprocess.run(cmd, shell=True, capture_output=True, text=True).stdout.strip() + + +def write_env(src, name, overrides): + path = "/tmp/spec-ab-" + name + ".env" + with open(src) as fh: + lines = fh.read().splitlines() + out = [] + for line in lines: + for k, v in overrides.items(): + if line.startswith(k + "="): + line = k + "=" + v + break + out.append(line) + with open(path, "w") as fh: + fh.write("\n".join(out) + "\n") + return path + + +def post(base, body, timeout=1200): + req = urllib.request.Request( + base + "/v1/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r) + + +def vram(): + out = sh("rocm-smi --showmeminfo vram 2>/dev/null | grep 'Used Memory'") + return [round(int(x) / 1e9, 2) for x in re.findall(r":\s+(\d+)\s*$", out, re.M)][:2] + + +def wait_health(base, timeout=420): + t0 = time.time() + while time.time() - t0 < timeout: + if not sh("pgrep -x llama-server >/dev/null && echo up"): + return False + try: + with urllib.request.urlopen(base + "/health", timeout=5) as r: + if r.status == 200 and json.load(r).get("status") == "ok": + return True + except Exception: + time.sleep(3) + return False + + +def stop_unit(): + sh("sudo systemctl stop " + UNIT) + sh("pkill -x llama-server") + for _ in range(40): + if not sh("pgrep -x llama-server || true"): + return + time.sleep(1) + + +def shape(base, prompt_tokens, max_tokens): + t = post(base, {"model": MODEL, + "messages": [{"role": "user", + "content": FILLER * (prompt_tokens // TOKENS_PER_REP) + + "\n\nReply with a single short sentence."}], + "max_tokens": max_tokens, "temperature": 0.0})["timings"] + return {"shape": "decode " + str(prompt_tokens) + "/" + str(max_tokens), + "prefill_tps": round(1000 * t["prompt_n"] / t["prompt_ms"], 1), + "predicted_n": t["predicted_n"], + "decode_tps": round(t["predicted_per_second"], 2), + "draft_n": t.get("draft_n", 0), + "accepted": t.get("draft_n_accepted", 0)} + + +def measure(base): + shape(base, 4000, 32) # warmup + rows = [shape(base, 1000, 512), shape(base, 16000, 16)] # decode, prefill + d = post(base, {"model": MODEL, + "messages": [{"role": "user", + "content": "List the first five prime numbers, comma separated, then stop."}], + "max_tokens": 192, "temperature": 0.0}) + msg = d["choices"][0]["message"] + return rows, (msg.get("content") or "").strip() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("env_file") + ap.add_argument("serve_script") + ap.add_argument("--base-url", default="http://127.0.0.1:7770") + ap.add_argument("--log", default="/tmp/bench-spec-ab.log") + args = ap.parse_args() + + base, logfile = args.base_url, args.log + results = {} + log("spec A/B start (unit " + UNIT + " stopped for the duration)", logfile) + stop_unit() + for name, over in CONFIGS: + env_path = write_env(args.env_file, name, over) + logf = "/tmp/spec-ab-" + name + ".log" + sh("cd " + sh("dirname " + args.serve_script) + " && FRANK_MEDIUM_ENV_FILE=" + env_path + + " setsid bash " + args.serve_script + " > " + logf + " 2>&1 < /dev/null &") + if not wait_health(base): + log(name + ": FAILED to become healthy -- " + sh("tail -2 " + logf), logfile) + results[name] = {"error": "not healthy", "tail": sh("tail -4 " + logf)} + stop_unit() + continue + rows, text = measure(base) + results[name] = {"vram_gb": vram(), "rows": rows, "answer": text[:240], + "spec_log": sh("grep -E '" + SPEC_GREP + "' " + logf + " | head -4")} + log(name + ": " + " | ".join( + r["shape"] + " prefill " + str(r["prefill_tps"]) + " decode " + str(r["decode_tps"]) + + " draft " + str(r["draft_n"]) + "/" + str(r["accepted"]) for r in rows) + + " | vram " + str(vram()) + " | answer=" + repr(text[:40]), logfile) + stop_unit() + + with open("/tmp/bench-spec-ab.json", "w") as fh: + json.dump(results, fh, indent=2) + answers = {v.get("answer") for v in results.values() if v.get("answer")} + if len(answers) > 1: + log("WARNING: configurations disagree on greedy text -- speculation is " + "changing answers: " + repr(answers), logfile) + log("restoring " + UNIT, logfile) + sh("sudo systemctl start " + UNIT) + for _ in range(120): + try: + with urllib.request.urlopen(base + "/health", timeout=5) as r: + if json.load(r).get("status") == "ok": + break + except Exception: + time.sleep(5) + log("done; results in /tmp/bench-spec-ab.json", logfile) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/frank-medium.env b/scripts/frank-medium.env index 7ff6a10..8e7e30e 100644 --- a/scripts/frank-medium.env +++ b/scripts/frank-medium.env @@ -149,6 +149,32 @@ SPEC_DRAFT_DEVICE=ROCm1 SPEC_DRAFT_TYPE_K=f16 SPEC_DRAFT_TYPE_V=f16 +# MTP vs the drafter, MEASURED 2026-09-16 (temperature 0, 1k prompt / 512 out; +# the autoregressive baseline and both MTP depths were run against the deployed +# configuration in one session, same binary, same prompts): +# +# none 17.7 tok/s card1 26.07 GB +# draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB +# draft-mtp n=7 26.7 tok/s 33 % accepted card1 29.65 GB +# draft-dflash 7 43.5 tok/s 41 % accepted card1 29.42 GB +# +# All four produced IDENTICAL greedy text, so verification is lossless whichever +# proposer runs. DFlash2 is worth 2.46x over autoregressive and 34 % over the +# best MTP; deeper MTP drafting costs more than it returns (n=7 accepts 33 % of +# 147 proposals, n=3 accepts 58 % of 72). The MTP head is not free either: +# selecting it loads blk.64.nextn.* and adds ~3.3 GB to the card that holds it, +# against 2.06 GB for the drafter's file. Do not switch this without +# re-measuring. +# +# THE TWO CANNOT BE CHAINED. `--spec-type draft-mtp,draft-dflash` dies at load: +# context type MTP requested but model doesn't contain MTP layers +# failed to create MTP context +# A chain has ONE draft context, and MTP builds against whatever model that +# context holds -- here the DFlash2 file, which has no MTP layers. MTP's head is +# only loaded when MTP IS the draft model ("creating MTP draft context against +# the target model ..."), so a chain of the two is a structural dead end, not a +# tuning question. See serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md. + # Text only: the server must not autoload a vision tower. Vision stays on # frank-small (Dankclaw, RTX 3090), which is the fleet's image endpoint. NO_MMPROJ=1 diff --git a/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md b/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md index 5a006a0..fcad3e5 100644 --- a/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md +++ b/serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md @@ -211,6 +211,37 @@ vLLM predecessor's 42.8 tok/s single-stream (two replicas) and the ROCmFPX llama.cpp measured here at 35.7-46.2 tok/s. One process, one model, both cards, no proxy. +### What the drafter is worth — and why MTP is not the alternative + +Measured 2026-09-16 with `scripts/bench-spec-ab.py`, four configurations of the +same binary, identical prompts at `temperature 0`, unit stopped between runs +(one host, one copy of a 29 GiB model). 1k-token prompt, 512-token budget: + +| Configuration | Decode | Drafting | VRAM (card 1) | +|---|---:|---|---:| +| `none` — autoregressive | 17.7 tok/s | — | 26.07 GB | +| `draft-mtp` n_max 3 (the model's own head) | 32.4 tok/s | 58 % of 72 | 29.35 GB | +| `draft-mtp` n_max 7 | 26.7 tok/s | 33 % of 147 | 29.65 GB | +| **`draft-dflash` n_max 7 — deployed** | **43.5 tok/s** | 41 % of 119 | 29.42 GB | + +- **DFlash2 is worth 2.46× over autoregressive and 34 % over the best MTP.** + Deeper MTP drafting costs more than it returns: at depth 7 only a third of + the proposals survive, at depth 3 more than half do. +- **All four produced identical greedy text** (`2, 3, 5, 7, 11`): speculation + buys speed, never a different answer. The harness warns loudly if a future + configuration ever breaks that. +- **MTP is not the cheaper option.** Selecting it loads `blk.64.nextn.*` from + the target and adds ~3.3 GB to the card that holds it, against 2.06 GB for + the drafter's file — which is also why the load log says + `unused tensor blk.64.nextn... ignoring` under `draft-dflash`: correct + behaviour, not a truncated download. +- **They cannot be chained.** `--spec-type draft-mtp,draft-dflash` dies at load + with `context type MTP requested but model doesn't contain MTP layers`. A + chain has ONE draft context, and MTP builds against whatever model that + context holds — here the DFlash2 file. MTP's head is loaded only when MTP *is* + the draft model (`creating MTP draft context against the target model ...`). + That is structural, not a tuning question. + ### The agent-loop shape (fleet harness, 2 families × 2 turns) `scripts/bench-turn-latency.py http://127.0.0.1:7770 2 2 LAYERSPLIT` — two diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats index fc8f4bf..6fcf7f8 100755 --- a/tests/frank_medium_unit.bats +++ b/tests/frank_medium_unit.bats @@ -73,6 +73,26 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter [ -n "$n_max" ] && [ "$n_max" -ge 1 ] && [ "$n_max" -le 7 ] } +@test "a draft MODEL is passed only for DFlash types, never for MTP" { + # MTP builds its context against the TARGET model (its head lives in that + # GGUF). Passing --spec-draft-model alongside it points the draft context at + # the drafter's file, which has no MTP layers, and the server dies at load: + # context type MTP requested but model doesn't contain MTP layers + # failed to create MTP context + # So the flag must appear exactly once, inside the dflash branch. + python3 -c " +lines = open('$SERVE').read().splitlines() +guard = next(i for i, l in enumerate(lines) if '*dflash*' in l) +indent = len(lines[guard]) - len(lines[guard].lstrip()) +end = next(i for i in range(guard + 1, len(lines)) + if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent) +block = chr(10).join(lines[guard:end]) +assert '--spec-draft-model' in block, 'the dflash branch must pass the draft model' +assert sum(l.count('--spec-draft-model') for l in lines) == 1, 'only that branch may pass it' +print('draft-model wiring ok') +" +} + @test "the runtime is pinned to a commit and to gfx1201 alone" { # A floating ref would silently move the service between engine versions, # and an unpinned arch would build for a sibling GPU that loads and then