From 21daf07713be7a296a8dabf9f85c6ceaa1152a7d Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2026 16:19:15 -0700 Subject: [PATCH] frank-medium: ngram-mod is a legal chain member -- wire it and measure it Unlike MTP, ngram-mod needs no draft context: common/speculative.cpp adds the ngram types unconditionally while the model-backed drafters are gated on ctx_dft != nullptr, so 'draft-dflash,ngram-mod' is structurally allowed (and the config vector's fixed order -- ngram first, then the drafters -- makes CLI order irrelevant). The harness grows the two things this question needs: --configs, and an ECHO shape. ngram-mod can only ever match text that is present in the prompt, so measuring it on prose calls it useless and measuring DFlash on echo calls it wasteful; both shapes now run, and the greedy text of each is compared across configurations because speculation may not change answers. --- scripts/bench-spec-ab.py | 169 +++++++++++++++++++++++----------- scripts/frank-medium.env | 10 ++ scripts/serve-frank-medium.sh | 13 +++ 3 files changed, 136 insertions(+), 56 deletions(-) diff --git a/scripts/bench-spec-ab.py b/scripts/bench-spec-ab.py index 1b4a758..4984cc0 100755 --- a/scripts/bench-spec-ab.py +++ b/scripts/bench-spec-ab.py @@ -5,27 +5,37 @@ Runs one payload through several speculative-decoding configurations of the same binary, one at a time, with the unit stopped between runs: on a single-GPU host there is no room for two copies of a 29 GiB model, and the benchmarking policy applies. Every configuration gets identical prompts at temperature 0, and the -greedy text of a fixed prompt is compared across them — speculation may buy -speed, never change the answer. +greedy text is compared across them — speculation may buy speed, never change +the answer. - bench-spec-ab.py [--base-url URL] + bench-spec-ab.py [options] bench-spec-ab.py /home/dank/serverAppConfigs/scripts/frank-medium.env \ - /home/dank/serverAppConfigs/scripts/frank-medium.sh + /home/dank/serverAppConfigs/scripts/serve-frank-medium.sh \ + --configs "dflash=draft-dflash@7;ngram=ngram-mod@64;chain=draft-dflash,ngram-mod@7" -The configurations are the ones that exist for THIS payload (frank-medium, -2026-09-16): the DFlash2 drafter, the model's own MTP head at two depths, and -autoregressive. The systemd unit is restored at the end, so the endpoint comes -back on whatever the env file says whatever happens here. +Config syntax: `name=SPEC_TYPE[@n_max]`, semicolon-separated. The systemd unit is +restored at the end, so the endpoint comes back on whatever the env file says +whatever happens here. -Result on this box (1k prompt / 512 out, temperature 0): +TWO SHAPES, because a single one answers the wrong question: - none 17.7 tok/s card1 26.07 GB - draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB - draft-mtp n=7 26.7 tok/s 33 % accepted card1 29.65 GB - draft-dflash 7 43.5 tok/s 41 % accepted card1 29.42 GB + prose a 1k-token prompt asking for a sentence — nothing to copy, so only a + model-backed proposer can help. This is the shape that compares + drafters. + echo a ~1.5k-token passage of unique lines with "repeat it back verbatim". + The output is almost entirely a copy of the prompt, which is the only + traffic where an n-gram matcher has anything to match. Measuring + ngram-mod on prose alone would call it useless; measuring DFlash on + echo alone would call it wasteful. + +Results on Dankserver (frank-medium payload, 2026-09-16, temperature 0): + + prose 1k/512 none 17.7 | mtp n=3 32.4 | mtp n=7 26.7 | dflash=7 43.5 tok/s + card 1 VRAM none 26.07 | mtp n=3 29.35 | mtp n=7 29.65 | dflash 29.42 GB ...and `draft-mtp,draft-dflash` fails at load: a chain has ONE draft context and -MTP builds against whatever model that context holds. +MTP builds against whatever model that context holds. ngram-mod has no such +constraint — it is draftless and is added to the chain unconditionally. """ import argparse import json @@ -40,15 +50,33 @@ FILLER = ("The quick brown fox jumps over the lazy dog near the river bank. " TOKENS_PER_REP = 28 UNIT = "frank-medium.service" MODEL = "frank-medium" -SPEC_GREP = "adding speculative implementation|- n_max=|- block_size=|creating MTP" +SPEC_GREP = "adding speculative implementation|- n_max=|- block_size=|creating MTP|ngram_mod with" -CONFIGS = [ +DEFAULT_CONFIGS = [ ("none", {"SPEC_TYPE": "none"}), ("mtp-n3", {"SPEC_TYPE": "draft-mtp", "SPEC_DRAFT_N_MAX": "3"}), ("mtp-n7", {"SPEC_TYPE": "draft-mtp", "SPEC_DRAFT_N_MAX": "7"}), ("dflash-n7", {"SPEC_TYPE": "draft-dflash", "SPEC_DRAFT_N_MAX": "7"}), ] +WORDS = ["latency", "throughput", "cache", "adapter", "scheduler", "allocator", + "queue", "kernel", "buffer", "shard", "replica", "window"] + + +def echo_passage(lines=90): + """Unique lines: no long span repeats inside the prompt, so an n-gram can + only fire when the OUTPUT copies the INPUT.""" + return "\n".join( + "Record %03d: the %s of subsystem %d reports state %d with checksum %05d." + % (i, WORDS[i % len(WORDS)], i % 7, i % 3, (i * 7919) % 100000) + for i in range(lines)) + + +ECHO_INSTRUCTION = ("\n\nRepeat the passage above verbatim, every line, exactly as written, " + "with no commentary before or after it.") +PROSE_INSTRUCTION = "\n\nReply with a single short sentence." +IDENTITY_PROMPT = "List the first five prime numbers, comma separated, then stop." + def log(msg, logfile): line = "[" + time.strftime("%H:%M:%S") + "] " + str(msg) @@ -61,8 +89,25 @@ def sh(cmd): return subprocess.run(cmd, shell=True, capture_output=True, text=True).stdout.strip() +def parse_configs(spec): + out = [] + for chunk in spec.split(";"): + chunk = chunk.strip() + if not chunk: + continue + name, _, rest = chunk.partition("=") + n_max = None + if "@" in rest: + rest, _, n_max = rest.partition("@") + over = {"SPEC_TYPE": rest.strip()} + if n_max: + over["SPEC_DRAFT_N_MAX"] = n_max.strip() + out.append((name.strip(), over)) + return out + + def write_env(src, name, overrides): - path = "/tmp/spec-ab-" + name + ".env" + path = "/tmp/spec-ab-" + name.replace("/", "_") + ".env" with open(src) as fh: lines = fh.read().splitlines() out = [] @@ -77,7 +122,7 @@ def write_env(src, name, overrides): return path -def post(base, body, timeout=1200): +def post(base, body, timeout=1800): req = urllib.request.Request( base + "/v1/chat/completions", data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) @@ -113,29 +158,30 @@ def stop_unit(): time.sleep(1) -def shape(base, prompt_tokens, max_tokens): +def run_shape(base, prompt, max_tokens, label): t = post(base, {"model": MODEL, - "messages": [{"role": "user", - "content": FILLER * (prompt_tokens // TOKENS_PER_REP) + - "\n\nReply with a single short sentence."}], - "max_tokens": max_tokens, "temperature": 0.0})["timings"] - return {"shape": "decode " + str(prompt_tokens) + "/" + str(max_tokens), - "prefill_tps": round(1000 * t["prompt_n"] / t["prompt_ms"], 1), - "predicted_n": t["predicted_n"], - "decode_tps": round(t["predicted_per_second"], 2), - "draft_n": t.get("draft_n", 0), - "accepted": t.get("draft_n_accepted", 0)} + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, "temperature": 0.0}) + tm = t["timings"] + row = {"shape": label, + "prompt_n": tm["prompt_n"], + "prefill_tps": round(1000 * tm["prompt_n"] / tm["prompt_ms"], 1), + "predicted_n": tm["predicted_n"], + "decode_tps": round(tm["predicted_per_second"], 2), + "draft_n": tm.get("draft_n", 0), + "accepted": tm.get("draft_n_accepted", 0), + "answer": (t["choices"][0]["message"].get("content") or "").strip()} + return row def measure(base): - shape(base, 4000, 32) # warmup - rows = [shape(base, 1000, 512), shape(base, 16000, 16)] # decode, prefill - d = post(base, {"model": MODEL, - "messages": [{"role": "user", - "content": "List the first five prime numbers, comma separated, then stop."}], - "max_tokens": 192, "temperature": 0.0}) - msg = d["choices"][0]["message"] - return rows, (msg.get("content") or "").strip() + run_shape(base, FILLER * (4000 // TOKENS_PER_REP) + PROSE_INSTRUCTION, 32, "warmup") + rows = [ + run_shape(base, FILLER * (1000 // TOKENS_PER_REP) + PROSE_INSTRUCTION, 512, "prose 1k/512"), + run_shape(base, echo_passage() + ECHO_INSTRUCTION, 400, "echo 1.5k/400"), + run_shape(base, IDENTITY_PROMPT, 192, "identity"), + ] + return rows def main(): @@ -144,37 +190,48 @@ def main(): ap.add_argument("serve_script") ap.add_argument("--base-url", default="http://127.0.0.1:7770") ap.add_argument("--log", default="/tmp/bench-spec-ab.log") + ap.add_argument("--configs", default=None, + help="name=SPEC_TYPE[@n_max] joined by ';' (default: the four " + "configs measured on 2026-09-16)") + ap.add_argument("--json", default="/tmp/bench-spec-ab.json") args = ap.parse_args() + configs = parse_configs(args.configs) if args.configs else DEFAULT_CONFIGS base, logfile = args.base_url, args.log results = {} - log("spec A/B start (unit " + UNIT + " stopped for the duration)", logfile) + log("spec A/B start: " + ", ".join(n for n, _ in configs) + + " (unit " + UNIT + " stopped for the duration)", logfile) stop_unit() - for name, over in CONFIGS: + for name, over in configs: env_path = write_env(args.env_file, name, over) - logf = "/tmp/spec-ab-" + name + ".log" + logf = "/tmp/spec-ab-" + name.replace("/", "_") + ".log" sh("cd " + sh("dirname " + args.serve_script) + " && FRANK_MEDIUM_ENV_FILE=" + env_path + " setsid bash " + args.serve_script + " > " + logf + " 2>&1 < /dev/null &") if not wait_health(base): - log(name + ": FAILED to become healthy -- " + sh("tail -2 " + logf), logfile) - results[name] = {"error": "not healthy", "tail": sh("tail -4 " + logf)} + log(name + ": FAILED to become healthy -- " + sh("tail -3 " + logf), logfile) + results[name] = {"error": "not healthy", "tail": sh("tail -6 " + logf)} stop_unit() continue - rows, text = measure(base) - results[name] = {"vram_gb": vram(), "rows": rows, "answer": text[:240], - "spec_log": sh("grep -E '" + SPEC_GREP + "' " + logf + " | head -4")} - log(name + ": " + " | ".join( - r["shape"] + " prefill " + str(r["prefill_tps"]) + " decode " + str(r["decode_tps"]) + - " draft " + str(r["draft_n"]) + "/" + str(r["accepted"]) for r in rows) + - " | vram " + str(vram()) + " | answer=" + repr(text[:40]), logfile) + rows = measure(base) + results[name] = {"vram_gb": vram(), "rows": rows, + "spec_log": sh("grep -E '" + SPEC_GREP + "' " + logf + " | head -6")} + for r in rows: + log(name + " | " + r["shape"] + " prefill " + str(r["prefill_tps"]) + + " tok/s | decode " + str(r["decode_tps"]) + " tok/s | draft " + + str(r["draft_n"]) + "/" + str(r["accepted"]) + + (" (" + str(round(100.0 * r["accepted"] / r["draft_n"])) + " %)" if r["draft_n"] else ""), + logfile) + log(name + " | vram " + str(results[name]["vram_gb"]), logfile) stop_unit() - with open("/tmp/bench-spec-ab.json", "w") as fh: + with open(args.json, "w") as fh: json.dump(results, fh, indent=2) - answers = {v.get("answer") for v in results.values() if v.get("answer")} - if len(answers) > 1: - log("WARNING: configurations disagree on greedy text -- speculation is " - "changing answers: " + repr(answers), logfile) + for shape in ("prose 1k/512", "echo 1.5k/400", "identity"): + texts = {v["rows"][i]["answer"] for v in results.values() if "rows" in v + for i, r in enumerate(v["rows"]) if r["shape"] == shape} + if len(texts) > 1: + log("WARNING: configurations disagree on greedy text for '" + shape + + "' -- speculation is changing answers", logfile) log("restoring " + UNIT, logfile) sh("sudo systemctl start " + UNIT) for _ in range(120): @@ -184,9 +241,9 @@ def main(): break except Exception: time.sleep(5) - log("done; results in /tmp/bench-spec-ab.json", logfile) + log("done; results in " + args.json, logfile) return 0 if __name__ == "__main__": - sys.exit(main()) + sys.exit(main()) \ No newline at end of file diff --git a/scripts/frank-medium.env b/scripts/frank-medium.env index 8e7e30e..d052eeb 100644 --- a/scripts/frank-medium.env +++ b/scripts/frank-medium.env @@ -175,6 +175,16 @@ SPEC_DRAFT_TYPE_V=f16 # the target model ..."), so a chain of the two is a structural dead end, not a # tuning question. See serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md. +# ngram-mod: the draftless n-gram matcher (CPU hash table, no VRAM, no model +# file). It is a legitimate second chain member -- unlike MTP it needs no draft +# context, so `--spec-type draft-dflash,ngram-mod` is structurally allowed. The +# window below is the fleet's tuned one (n_min/n_max 48/64 match long echoes, +# which is the only traffic where n-grams pay). Measured against DFlash2 on +# 2026-09-16 -- see the runbook for the verdict before enabling it. +SPEC_NGRAM_N_MATCH=24 +SPEC_NGRAM_N_MIN=48 +SPEC_NGRAM_N_MAX=64 + # Text only: the server must not autoload a vision tower. Vision stays on # frank-small (Dankclaw, RTX 3090), which is the fleet's image endpoint. NO_MMPROJ=1 diff --git a/scripts/serve-frank-medium.sh b/scripts/serve-frank-medium.sh index 3b01d10..13c066c 100755 --- a/scripts/serve-frank-medium.sh +++ b/scripts/serve-frank-medium.sh @@ -87,6 +87,9 @@ fi : "${SPEC_DRAFT_TYPE_V:=f16}" : "${SPEC_DRAFT_POLL:=1}" : "${SPEC_DRAFT_POLL_BATCH:=1}" +: "${SPEC_NGRAM_N_MATCH:=24}" +: "${SPEC_NGRAM_N_MIN:=48}" +: "${SPEC_NGRAM_N_MAX:=64}" NO_MMPROJ="${NO_MMPROJ:-1}" @@ -129,6 +132,16 @@ if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then if [[ "$SPEC_TYPE" == *dspark* || ( "$SPEC_DRAFT_P_MIN" != "0.0" && "$SPEC_DRAFT_P_MIN" != "0" ) ]]; then SPEC_ARGS+=(--spec-draft-n-min "$SPEC_DRAFT_N_MIN" --spec-draft-p-min "$SPEC_DRAFT_P_MIN") fi + # ngram-mod is draftless: it needs no draft context and its window is its + # own (--spec-ngram-mod-*), so it can ride in the chain beside a model + # drafter. It also costs no VRAM -- a CPU hash table. + if [[ "$SPEC_TYPE" == *ngram* ]]; then + SPEC_ARGS+=( + --spec-ngram-mod-n-match "$SPEC_NGRAM_N_MATCH" + --spec-ngram-mod-n-min "$SPEC_NGRAM_N_MIN" + --spec-ngram-mod-n-max "$SPEC_NGRAM_N_MAX" + ) + fi if [[ -n "$SPEC_DRAFT_DEVICE" ]]; then SPEC_ARGS+=(--spec-draft-device "$SPEC_DRAFT_DEVICE") fi