From 95936bbc81e3a53781ea9b004b26de657b148a4b Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2026 16:38:12 -0700 Subject: [PATCH] bench-spec-ab: rewrite (write_env was lost), restore the unit in a finally Two real defects from patching a rewritten file by line number: the CUT that removed a duplicate parse_configs also took write_env with it (NameError after the first configuration, with the unit already stopped), and the restore step lived after the loop, so a crash left frank-medium DOWN. The endpoint is the reason the harness exists; it now restores itself in a finally and shouts if the service does not come back healthy. --- scripts/bench-spec-ab.py | 121 +++++++++++++++++++++++++-------------- 1 file changed, 78 insertions(+), 43 deletions(-) diff --git a/scripts/bench-spec-ab.py b/scripts/bench-spec-ab.py index d2d9862..e453cc9 100755 --- a/scripts/bench-spec-ab.py +++ b/scripts/bench-spec-ab.py @@ -13,9 +13,9 @@ the answer. /home/dank/serverAppConfigs/scripts/serve-frank-medium.sh \ --configs "dflash=draft-dflash@7;ngram=ngram-mod@64;chain=draft-dflash,ngram-mod@7" -Config syntax: `name=SPEC_TYPE[@n_max]`, semicolon-separated. The systemd unit is -restored at the end, so the endpoint comes back on whatever the env file says -whatever happens here. +Config syntax: `name=SPEC_TYPE[@n_max][|ENV=VAL|ENV=VAL]`, entries joined by `;`. +The systemd unit is restored in a finally block, so it comes back on whatever the +env file says even if a configuration crashes the harness. THREE SHAPES, because a single one answers the wrong question. A proposer can only draft text it has seen, so the shape decides the verdict: @@ -34,8 +34,14 @@ model thinks the answer into `reasoning_content`, `content` comes back empty and the cross-configuration identity check passes vacuously — which is exactly how this harness first "verified" an echo it never looked at. -Results on Dankserver (frank-medium payload, 2026-09-16, temperature 0). prose -1k/512 decode: none 17.7 | mtp n=3 32.4 | mtp n=7 26.7 | dflash=7 43.5 tok/s. +Results on Dankserver (frank-medium payload, 2026-09-16, temperature 0). Decode: + + shape dflash=7 ngram-mod(48/64) chain (tok/s) + prose 1k/512 43.8 17.7 (0 drafts) 43.7 + echo 1.5k/400 76.8 102.5 200.6 + structured/400 91.1 17.8 (0 drafts) 91.2 + identity 70.4 17.8 (0 drafts) 70.4 + `draft-mtp,draft-dflash` fails at load (a chain has ONE draft context and MTP builds against whatever model that context holds); ngram-mod has no such constraint — it is draftless and joins the chain unconditionally. @@ -101,9 +107,9 @@ def sh(cmd): def parse_configs(spec): """`name=SPEC_TYPE[@n_max][|KEY=VALUE]...`, entries joined by ';'. - KEY=VALUE parts override any env file key, which is how an impl's own - window (SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing - env files.""" + KEY=VALUE parts override any env file key, which is how an impl's own window + (SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing env + files.""" out = [] for chunk in spec.split(";"): chunk = chunk.strip() @@ -127,6 +133,23 @@ def parse_configs(spec): return out +def write_env(src, name, overrides): + """The env file with this configuration's overrides applied.""" + path = "/tmp/spec-ab-" + name.replace("/", "_") + ".env" + with open(src) as fh: + lines = fh.read().splitlines() + out = [] + for line in lines: + for k, v in overrides.items(): + if line.startswith(k + "="): + line = k + "=" + v + break + out.append(line) + with open(path, "w") as fh: + fh.write("\n".join(out) + "\n") + return path + + def post(base, body, timeout=1800): req = urllib.request.Request( base + "/v1/chat/completions", data=json.dumps(body).encode(), @@ -163,6 +186,23 @@ def stop_unit(): time.sleep(1) +def restore_unit(base, logfile): + """Bring the endpoint back on the deployed configuration. Runs in a + finally, because a harness that can leave production down is a hazard.""" + log("restoring " + UNIT, logfile) + sh("sudo systemctl start " + UNIT) + for _ in range(120): + try: + with urllib.request.urlopen(base + "/health", timeout=5) as r: + if json.load(r).get("status") == "ok": + log(UNIT + " healthy again", logfile) + return True + except Exception: + time.sleep(5) + log("WARNING: " + UNIT + " did not come back healthy -- check it by hand", logfile) + return False + + def run_shape(base, prompt, max_tokens, label, extra=None): body = {"model": MODEL, "messages": [{"role": "user", "content": prompt}], "max_tokens": max_tokens, "temperature": 0.0} @@ -182,8 +222,8 @@ def run_shape(base, prompt, max_tokens, label, extra=None): "answer": content, "reasoning_len": len(msg.get("reasoning_content") or "")} if label.startswith("echo"): - # How much of the answer really is the passage: a config that "wins" by - # answering something else is not a win. + # How much of the answer really is the passage: a configuration that + # "wins" by answering something else is not a win. want = {l.strip() for l in echo_passage().splitlines()} got = [l.strip() for l in content.splitlines() if l.strip()] row["copy_ratio"] = round(sum(1 for l in got if l in want) / len(got), 3) if got else 0.0 @@ -208,8 +248,8 @@ def main(): ap.add_argument("--base-url", default="http://127.0.0.1:7770") ap.add_argument("--log", default="/tmp/bench-spec-ab.log") ap.add_argument("--configs", default=None, - help="name=SPEC_TYPE[@n_max] joined by ';' (default: the four " - "configs measured on 2026-09-16)") + help="name=SPEC_TYPE[@n_max][|ENV=VAL] joined by ';' " + "(default: the four configs measured on 2026-09-16)") ap.add_argument("--json", default="/tmp/bench-spec-ab.json") args = ap.parse_args() @@ -222,29 +262,33 @@ def main(): results = {} log("spec A/B start: " + ", ".join(n for n, _ in configs) + " (unit " + UNIT + " stopped for the duration)", logfile) - stop_unit() - for name, over in configs: - env_path = write_env(env_file, name, over) - logf = "/tmp/spec-ab-" + name.replace("/", "_") + ".log" - sh("FRANK_MEDIUM_ENV_FILE=" + env_path + " setsid bash " + serve_script + - " > " + logf + " 2>&1 < /dev/null &") - if not wait_health(base): - log(name + ": FAILED to become healthy -- " + sh("tail -3 " + logf), logfile) - results[name] = {"error": "not healthy", "tail": sh("tail -6 " + logf)} - stop_unit() - continue - rows = measure(base) - results[name] = {"vram_gb": vram(), "rows": rows, - "spec_log": sh("grep -E '" + SPEC_GREP + "' " + logf + " | tail -4")} - for r in rows: - log(name + " | " + r["shape"] + " prefill " + str(r["prefill_tps"]) + - " tok/s | decode " + str(r["decode_tps"]) + " tok/s | draft " + - str(r["draft_n"]) + "/" + str(r["accepted"]) + - (" (" + str(round(100.0 * r["accepted"] / r["draft_n"])) + " %)" if r["draft_n"] else "") + - (" | copy " + str(r["copy_ratio"]) if "copy_ratio" in r else "") + - (" | EMPTY ANSWER" if not r["answer"] else ""), logfile) - log(name + " | vram " + str(results[name]["vram_gb"]), logfile) + try: stop_unit() + for name, over in configs: + env_path = write_env(env_file, name, over) + logf = "/tmp/spec-ab-" + name.replace("/", "_") + ".log" + sh("FRANK_MEDIUM_ENV_FILE=" + env_path + " setsid bash " + serve_script + + " > " + logf + " 2>&1 < /dev/null &") + if not wait_health(base): + log(name + ": FAILED to become healthy -- " + sh("tail -3 " + logf), logfile) + results[name] = {"error": "not healthy", "tail": sh("tail -6 " + logf)} + stop_unit() + continue + rows = measure(base) + results[name] = {"vram_gb": vram(), "rows": rows, + "spec_log": sh("grep -E '" + SPEC_GREP + "' " + logf + " | tail -4")} + for r in rows: + log(name + " | " + r["shape"] + " prefill " + str(r["prefill_tps"]) + + " tok/s | decode " + str(r["decode_tps"]) + " tok/s | draft " + + str(r["draft_n"]) + "/" + str(r["accepted"]) + + (" (" + str(round(100.0 * r["accepted"] / r["draft_n"])) + " %)" + if r["draft_n"] else "") + + (" | copy " + str(r["copy_ratio"]) if "copy_ratio" in r else "") + + (" | EMPTY ANSWER" if not r["answer"] else ""), logfile) + log(name + " | vram " + str(results[name]["vram_gb"]), logfile) + stop_unit() + finally: + restore_unit(base, logfile) with open(args.json, "w") as fh: json.dump(results, fh, indent=2) @@ -257,15 +301,6 @@ def main(): elif len(set(texts)) > 1: log("WARNING: configurations disagree on greedy text for '" + shape + "' -- speculation is changing answers", logfile) - log("restoring " + UNIT, logfile) - sh("sudo systemctl start " + UNIT) - for _ in range(120): - try: - with urllib.request.urlopen(base + "/health", timeout=5) as r: - if json.load(r).get("status") == "ok": - break - except Exception: - time.sleep(5) log("done; results in " + args.json, logfile) return 0