diff --git a/scripts/bench-spec-ab.py b/scripts/bench-spec-ab.py index f22931b..73294eb 100755 --- a/scripts/bench-spec-ab.py +++ b/scripts/bench-spec-ab.py @@ -17,25 +17,28 @@ Config syntax: `name=SPEC_TYPE[@n_max]`, semicolon-separated. The systemd unit i restored at the end, so the endpoint comes back on whatever the env file says whatever happens here. -TWO SHAPES, because a single one answers the wrong question: +THREE SHAPES, because a single one answers the wrong question. A proposer can +only draft text it has seen, so the shape decides the verdict: - prose a 1k-token prompt asking for a sentence — nothing to copy, so only a - model-backed proposer can help. This is the shape that compares - drafters. - echo a ~1.5k-token passage of unique lines with "repeat it back verbatim". - The output is almost entirely a copy of the prompt, which is the only - traffic where an n-gram matcher has anything to match. Measuring - ngram-mod on prose alone would call it useless; measuring DFlash on - echo alone would call it wasteful. + prose a 1k prompt asking for one sentence. Nothing to copy and nothing + repetitive, so only a model-backed proposer can help: this is the + shape that compares drafters. + echo a ~1.5k passage of unique lines, "repeat it back verbatim". The + output IS the prompt, which is ngram-mod's home turf. + structured "25 JSON records, one per line" — repetitive OUTPUT with no prompt + overlap, which is what an ngram matcher is really for in an + agentic fleet (config edits, tables, record lists). -Results on Dankserver (frank-medium payload, 2026-09-16, temperature 0): +echo and structured disable thinking (`chat_template_kwargs`), otherwise the +model thinks the answer into `reasoning_content`, `content` comes back empty and +the cross-configuration identity check passes vacuously — which is exactly how +this harness first "verified" an echo it never looked at. - prose 1k/512 none 17.7 | mtp n=3 32.4 | mtp n=7 26.7 | dflash=7 43.5 tok/s - card 1 VRAM none 26.07 | mtp n=3 29.35 | mtp n=7 29.65 | dflash 29.42 GB - -...and `draft-mtp,draft-dflash` fails at load: a chain has ONE draft context and -MTP builds against whatever model that context holds. ngram-mod has no such -constraint — it is draftless and is added to the chain unconditionally. +Results on Dankserver (frank-medium payload, 2026-09-16, temperature 0). prose +1k/512 decode: none 17.7 | mtp n=3 32.4 | mtp n=7 26.7 | dflash=7 43.5 tok/s. +`draft-mtp,draft-dflash` fails at load (a chain has ONE draft context and MTP +builds against whatever model that context holds); ngram-mod has no such +constraint — it is draftless and joins the chain unconditionally. """ import argparse import json @@ -51,7 +54,8 @@ FILLER = ("The quick brown fox jumps over the lazy dog near the river bank. " TOKENS_PER_REP = 28 UNIT = "frank-medium.service" MODEL = "frank-medium" -SPEC_GREP = "adding speculative implementation|- n_max=|- block_size=|creating MTP|ngram_mod with" +SPEC_GREP = ("adding speculative implementation|- n_max=|- block_size=|creating MTP|" + "draft acceptance") DEFAULT_CONFIGS = [ ("none", {"SPEC_TYPE": "none"}), @@ -73,10 +77,14 @@ def echo_passage(lines=90): for i in range(lines)) -ECHO_INSTRUCTION = ("\n\nRepeat the passage above verbatim, every line, exactly as written, " - "with no commentary before or after it.") -PROSE_INSTRUCTION = "\n\nReply with a single short sentence." -IDENTITY_PROMPT = "List the first five prime numbers, comma separated, then stop." +ECHO = (echo_passage() + "\n\nRepeat the passage above verbatim, every line, exactly as " + "written, with no commentary before or after it.") +STRUCTURED = ("Output exactly 25 JSON records, one per line, no prose and no code fence. " + 'Each line must look like {"id": N, "state": "active", "region": "us-west-2"} ' + "with N running from 1 to 25.") +PROSE = FILLER * (1000 // TOKENS_PER_REP) + "\n\nReply with a single short sentence." +IDENTITY = "List the first five prime numbers, comma separated, then stop." +NO_THINK = {"chat_template_kwargs": {"enable_thinking": False}} def log(msg, logfile): @@ -159,11 +167,15 @@ def stop_unit(): time.sleep(1) -def run_shape(base, prompt, max_tokens, label): - t = post(base, {"model": MODEL, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, "temperature": 0.0}) - tm = t["timings"] +def run_shape(base, prompt, max_tokens, label, extra=None): + body = {"model": MODEL, "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, "temperature": 0.0} + if extra: + body.update(extra) + d = post(base, body) + tm = d["timings"] + msg = d["choices"][0]["message"] + content = (msg.get("content") or "").strip() row = {"shape": label, "prompt_n": tm["prompt_n"], "prefill_tps": round(1000 * tm["prompt_n"] / tm["prompt_ms"], 1), @@ -171,18 +183,26 @@ def run_shape(base, prompt, max_tokens, label): "decode_tps": round(tm["predicted_per_second"], 2), "draft_n": tm.get("draft_n", 0), "accepted": tm.get("draft_n_accepted", 0), - "answer": (t["choices"][0]["message"].get("content") or "").strip()} + "answer": content, + "reasoning_len": len(msg.get("reasoning_content") or "")} + if label.startswith("echo"): + # How much of the answer really is the passage: a config that "wins" by + # answering something else is not a win. + want = {l.strip() for l in echo_passage().splitlines()} + got = [l.strip() for l in content.splitlines() if l.strip()] + row["copy_ratio"] = round(sum(1 for l in got if l in want) / len(got), 3) if got else 0.0 return row def measure(base): - run_shape(base, FILLER * (4000 // TOKENS_PER_REP) + PROSE_INSTRUCTION, 32, "warmup") - rows = [ - run_shape(base, FILLER * (1000 // TOKENS_PER_REP) + PROSE_INSTRUCTION, 512, "prose 1k/512"), - run_shape(base, echo_passage() + ECHO_INSTRUCTION, 400, "echo 1.5k/400"), - run_shape(base, IDENTITY_PROMPT, 192, "identity"), + run_shape(base, FILLER * (4000 // TOKENS_PER_REP) + + "\n\nReply with a single short sentence.", 32, "warmup") + return [ + run_shape(base, PROSE, 512, "prose 1k/512"), + run_shape(base, ECHO, 400, "echo 1.5k/400", NO_THINK), + run_shape(base, STRUCTURED, 400, "structured/400", NO_THINK), + run_shape(base, IDENTITY, 192, "identity"), ] - return rows def main(): @@ -219,22 +239,26 @@ def main(): continue rows = measure(base) results[name] = {"vram_gb": vram(), "rows": rows, - "spec_log": sh("grep -E '" + SPEC_GREP + "' " + logf + " | head -6")} + "spec_log": sh("grep -E '" + SPEC_GREP + "' " + logf + " | tail -4")} for r in rows: log(name + " | " + r["shape"] + " prefill " + str(r["prefill_tps"]) + " tok/s | decode " + str(r["decode_tps"]) + " tok/s | draft " + str(r["draft_n"]) + "/" + str(r["accepted"]) + - (" (" + str(round(100.0 * r["accepted"] / r["draft_n"])) + " %)" if r["draft_n"] else ""), - logfile) + (" (" + str(round(100.0 * r["accepted"] / r["draft_n"])) + " %)" if r["draft_n"] else "") + + (" | copy " + str(r["copy_ratio"]) if "copy_ratio" in r else "") + + (" | EMPTY ANSWER" if not r["answer"] else ""), logfile) log(name + " | vram " + str(results[name]["vram_gb"]), logfile) stop_unit() with open(args.json, "w") as fh: json.dump(results, fh, indent=2) - for shape in ("prose 1k/512", "echo 1.5k/400", "identity"): - texts = {v["rows"][i]["answer"] for v in results.values() if "rows" in v - for i, r in enumerate(v["rows"]) if r["shape"] == shape} - if len(texts) > 1: + for shape in ("prose 1k/512", "echo 1.5k/400", "structured/400", "identity"): + texts = [r["answer"] for v in results.values() if "rows" in v + for r in v["rows"] if r["shape"] == shape] + if any(not t for t in texts): + log("WARNING: '" + shape + "' produced an empty answer in some configuration -- " + "its identity check proves nothing", logfile) + elif len(set(texts)) > 1: log("WARNING: configurations disagree on greedy text for '" + shape + "' -- speculation is changing answers", logfile) log("restoring " + UNIT, logfile)