bench-spec-ab: rewrite (write_env was lost), restore the unit in a finally

Two real defects from patching a rewritten file by line number: the CUT that
removed a duplicate parse_configs also took write_env with it (NameError after
the first configuration, with the unit already stopped), and the restore step
lived after the loop, so a crash left frank-medium DOWN. The endpoint is the
reason the harness exists; it now restores itself in a finally and shouts if the
service does not come back healthy.

scripts/bench-spec-ab.py

@@ -13,9 +13,9 @@ the answer.
                      /home/dank/serverAppConfigs/scripts/serve-frank-medium.sh \
                      --configs "dflash=draft-dflash@7;ngram=ngram-mod@64;chain=draft-dflash,ngram-mod@7"
 
-Config syntax: `name=SPEC_TYPE[@n_max]`, semicolon-separated. The systemd unit is
+Config syntax: `name=SPEC_TYPE[@n_max][|ENV=VAL|ENV=VAL]`, entries joined by `;`.
-restored at the end, so the endpoint comes back on whatever the env file says
+The systemd unit is restored in a finally block, so it comes back on whatever the
-whatever happens here.
+env file says even if a configuration crashes the harness.
 
 THREE SHAPES, because a single one answers the wrong question. A proposer can
 only draft text it has seen, so the shape decides the verdict:
@@ -34,8 +34,14 @@ model thinks the answer into `reasoning_content`, `content` comes back empty and
 the cross-configuration identity check passes vacuously — which is exactly how
 this harness first "verified" an echo it never looked at.
 
-Results on Dankserver (frank-medium payload, 2026-09-16, temperature 0). prose
+Results on Dankserver (frank-medium payload, 2026-09-16, temperature 0). Decode:
-1k/512 decode: none 17.7 | mtp n=3 32.4 | mtp n=7 26.7 | dflash=7 43.5 tok/s.
+
+  shape            dflash=7   ngram-mod(48/64)   chain        (tok/s)
+  prose 1k/512        43.8        17.7 (0 drafts)   43.7
+  echo 1.5k/400       76.8       102.5             200.6
+  structured/400      91.1        17.8 (0 drafts)   91.2
+  identity            70.4        17.8 (0 drafts)   70.4
+
 `draft-mtp,draft-dflash` fails at load (a chain has ONE draft context and MTP
 builds against whatever model that context holds); ngram-mod has no such
 constraint — it is draftless and joins the chain unconditionally.
@@ -101,9 +107,9 @@ def sh(cmd):
 def parse_configs(spec):
     """`name=SPEC_TYPE[@n_max][|KEY=VALUE]...`, entries joined by ';'.
 
-    KEY=VALUE parts override any env file key, which is how an impl's own
+    KEY=VALUE parts override any env file key, which is how an impl's own window
-    window (SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing
+    (SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing env
-    env files."""
+    files."""
     out = []
     for chunk in spec.split(";"):
         chunk = chunk.strip()
@@ -127,6 +133,23 @@ def parse_configs(spec):
     return out
 
 
+def write_env(src, name, overrides):
+    """The env file with this configuration's overrides applied."""
+    path = "/tmp/spec-ab-" + name.replace("/", "_") + ".env"
+    with open(src) as fh:
+        lines = fh.read().splitlines()
+    out = []
+    for line in lines:
+        for k, v in overrides.items():
+            if line.startswith(k + "="):
+                line = k + "=" + v
+                break
+        out.append(line)
+    with open(path, "w") as fh:
+        fh.write("\n".join(out) + "\n")
+    return path
+
+
 def post(base, body, timeout=1800):
     req = urllib.request.Request(
         base + "/v1/chat/completions", data=json.dumps(body).encode(),
@@ -163,6 +186,23 @@ def stop_unit():
         time.sleep(1)
 
 
+def restore_unit(base, logfile):
+    """Bring the endpoint back on the deployed configuration. Runs in a
+    finally, because a harness that can leave production down is a hazard."""
+    log("restoring " + UNIT, logfile)
+    sh("sudo systemctl start " + UNIT)
+    for _ in range(120):
+        try:
+            with urllib.request.urlopen(base + "/health", timeout=5) as r:
+                if json.load(r).get("status") == "ok":
+                    log(UNIT + " healthy again", logfile)
+                    return True
+        except Exception:
+            time.sleep(5)
+    log("WARNING: " + UNIT + " did not come back healthy -- check it by hand", logfile)
+    return False
+
+
 def run_shape(base, prompt, max_tokens, label, extra=None):
     body = {"model": MODEL, "messages": [{"role": "user", "content": prompt}],
             "max_tokens": max_tokens, "temperature": 0.0}
@@ -182,8 +222,8 @@ def run_shape(base, prompt, max_tokens, label, extra=None):
            "answer": content,
            "reasoning_len": len(msg.get("reasoning_content") or "")}
     if label.startswith("echo"):
-        # How much of the answer really is the passage: a config that "wins" by
+        # How much of the answer really is the passage: a configuration that
-        # answering something else is not a win.
+        # "wins" by answering something else is not a win.
         want = {l.strip() for l in echo_passage().splitlines()}
         got = [l.strip() for l in content.splitlines() if l.strip()]
         row["copy_ratio"] = round(sum(1 for l in got if l in want) / len(got), 3) if got else 0.0
@@ -208,8 +248,8 @@ def main():
     ap.add_argument("--base-url", default="http://127.0.0.1:7770")
     ap.add_argument("--log", default="/tmp/bench-spec-ab.log")
     ap.add_argument("--configs", default=None,
-                    help="name=SPEC_TYPE[@n_max] joined by ';' (default: the four "
+                    help="name=SPEC_TYPE[@n_max][|ENV=VAL] joined by ';' "
-                         "configs measured on 2026-09-16)")
+                         "(default: the four configs measured on 2026-09-16)")
     ap.add_argument("--json", default="/tmp/bench-spec-ab.json")
     args = ap.parse_args()
 
@@ -222,6 +262,7 @@ def main():
     results = {}
     log("spec A/B start: " + ", ".join(n for n, _ in configs) +
         " (unit " + UNIT + " stopped for the duration)", logfile)
+    try:
         stop_unit()
         for name, over in configs:
             env_path = write_env(env_file, name, over)
@@ -240,11 +281,14 @@ def main():
                 log(name + " | " + r["shape"] + " prefill " + str(r["prefill_tps"]) +
                     " tok/s | decode " + str(r["decode_tps"]) + " tok/s | draft " +
                     str(r["draft_n"]) + "/" + str(r["accepted"]) +
-                (" (" + str(round(100.0 * r["accepted"] / r["draft_n"])) + " %)" if r["draft_n"] else "") +
+                    (" (" + str(round(100.0 * r["accepted"] / r["draft_n"])) + " %)"
+                     if r["draft_n"] else "") +
                     (" | copy " + str(r["copy_ratio"]) if "copy_ratio" in r else "") +
                     (" | EMPTY ANSWER" if not r["answer"] else ""), logfile)
             log(name + " | vram " + str(results[name]["vram_gb"]), logfile)
             stop_unit()
+    finally:
+        restore_unit(base, logfile)
 
     with open(args.json, "w") as fh:
         json.dump(results, fh, indent=2)
@@ -257,15 +301,6 @@ def main():
         elif len(set(texts)) > 1:
             log("WARNING: configurations disagree on greedy text for '" + shape +
                 "' -- speculation is changing answers", logfile)
-    log("restoring " + UNIT, logfile)
-    sh("sudo systemctl start " + UNIT)
-    for _ in range(120):
-        try:
-            with urllib.request.urlopen(base + "/health", timeout=5) as r:
-                if json.load(r).get("status") == "ok":
-                    break
-        except Exception:
-            time.sleep(5)
     log("done; results in " + args.json, logfile)
     return 0