bench-spec-ab: rewrite (write_env was lost), restore the unit in a finally
Two real defects from patching a rewritten file by line number: the CUT that removed a duplicate parse_configs also took write_env with it (NameError after the first configuration, with the unit already stopped), and the restore step lived after the loop, so a crash left frank-medium DOWN. The endpoint is the reason the harness exists; it now restores itself in a finally and shouts if the service does not come back healthy.
This commit is contained in:
Frank
+78
-43
@@ -13,9 +13,9 @@ the answer.
|
||||
/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh \
|
||||
--configs "dflash=draft-dflash@7;ngram=ngram-mod@64;chain=draft-dflash,ngram-mod@7"
|
||||
|
||||
Config syntax: `name=SPEC_TYPE[@n_max]`, semicolon-separated. The systemd unit is
|
||||
restored at the end, so the endpoint comes back on whatever the env file says
|
||||
whatever happens here.
|
||||
Config syntax: `name=SPEC_TYPE[@n_max][|ENV=VAL|ENV=VAL]`, entries joined by `;`.
|
||||
The systemd unit is restored in a finally block, so it comes back on whatever the
|
||||
env file says even if a configuration crashes the harness.
|
||||
|
||||
THREE SHAPES, because a single one answers the wrong question. A proposer can
|
||||
only draft text it has seen, so the shape decides the verdict:
|
||||
@@ -34,8 +34,14 @@ model thinks the answer into `reasoning_content`, `content` comes back empty and
|
||||
the cross-configuration identity check passes vacuously — which is exactly how
|
||||
this harness first "verified" an echo it never looked at.
|
||||
|
||||
Results on Dankserver (frank-medium payload, 2026-09-16, temperature 0). prose
|
||||
1k/512 decode: none 17.7 | mtp n=3 32.4 | mtp n=7 26.7 | dflash=7 43.5 tok/s.
|
||||
Results on Dankserver (frank-medium payload, 2026-09-16, temperature 0). Decode:
|
||||
|
||||
shape dflash=7 ngram-mod(48/64) chain (tok/s)
|
||||
prose 1k/512 43.8 17.7 (0 drafts) 43.7
|
||||
echo 1.5k/400 76.8 102.5 200.6
|
||||
structured/400 91.1 17.8 (0 drafts) 91.2
|
||||
identity 70.4 17.8 (0 drafts) 70.4
|
||||
|
||||
`draft-mtp,draft-dflash` fails at load (a chain has ONE draft context and MTP
|
||||
builds against whatever model that context holds); ngram-mod has no such
|
||||
constraint — it is draftless and joins the chain unconditionally.
|
||||
@@ -101,9 +107,9 @@ def sh(cmd):
|
||||
def parse_configs(spec):
|
||||
"""`name=SPEC_TYPE[@n_max][|KEY=VALUE]...`, entries joined by ';'.
|
||||
|
||||
KEY=VALUE parts override any env file key, which is how an impl's own
|
||||
window (SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing
|
||||
env files."""
|
||||
KEY=VALUE parts override any env file key, which is how an impl's own window
|
||||
(SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing env
|
||||
files."""
|
||||
out = []
|
||||
for chunk in spec.split(";"):
|
||||
chunk = chunk.strip()
|
||||
@@ -127,6 +133,23 @@ def parse_configs(spec):
|
||||
return out
|
||||
|
||||
|
||||
def write_env(src, name, overrides):
|
||||
"""The env file with this configuration's overrides applied."""
|
||||
path = "/tmp/spec-ab-" + name.replace("/", "_") + ".env"
|
||||
with open(src) as fh:
|
||||
lines = fh.read().splitlines()
|
||||
out = []
|
||||
for line in lines:
|
||||
for k, v in overrides.items():
|
||||
if line.startswith(k + "="):
|
||||
line = k + "=" + v
|
||||
break
|
||||
out.append(line)
|
||||
with open(path, "w") as fh:
|
||||
fh.write("\n".join(out) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def post(base, body, timeout=1800):
|
||||
req = urllib.request.Request(
|
||||
base + "/v1/chat/completions", data=json.dumps(body).encode(),
|
||||
@@ -163,6 +186,23 @@ def stop_unit():
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def restore_unit(base, logfile):
|
||||
"""Bring the endpoint back on the deployed configuration. Runs in a
|
||||
finally, because a harness that can leave production down is a hazard."""
|
||||
log("restoring " + UNIT, logfile)
|
||||
sh("sudo systemctl start " + UNIT)
|
||||
for _ in range(120):
|
||||
try:
|
||||
with urllib.request.urlopen(base + "/health", timeout=5) as r:
|
||||
if json.load(r).get("status") == "ok":
|
||||
log(UNIT + " healthy again", logfile)
|
||||
return True
|
||||
except Exception:
|
||||
time.sleep(5)
|
||||
log("WARNING: " + UNIT + " did not come back healthy -- check it by hand", logfile)
|
||||
return False
|
||||
|
||||
|
||||
def run_shape(base, prompt, max_tokens, label, extra=None):
|
||||
body = {"model": MODEL, "messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens, "temperature": 0.0}
|
||||
@@ -182,8 +222,8 @@ def run_shape(base, prompt, max_tokens, label, extra=None):
|
||||
"answer": content,
|
||||
"reasoning_len": len(msg.get("reasoning_content") or "")}
|
||||
if label.startswith("echo"):
|
||||
# How much of the answer really is the passage: a config that "wins" by
|
||||
# answering something else is not a win.
|
||||
# How much of the answer really is the passage: a configuration that
|
||||
# "wins" by answering something else is not a win.
|
||||
want = {l.strip() for l in echo_passage().splitlines()}
|
||||
got = [l.strip() for l in content.splitlines() if l.strip()]
|
||||
row["copy_ratio"] = round(sum(1 for l in got if l in want) / len(got), 3) if got else 0.0
|
||||
@@ -208,8 +248,8 @@ def main():
|
||||
ap.add_argument("--base-url", default="http://127.0.0.1:7770")
|
||||
ap.add_argument("--log", default="/tmp/bench-spec-ab.log")
|
||||
ap.add_argument("--configs", default=None,
|
||||
help="name=SPEC_TYPE[@n_max] joined by ';' (default: the four "
|
||||
"configs measured on 2026-09-16)")
|
||||
help="name=SPEC_TYPE[@n_max][|ENV=VAL] joined by ';' "
|
||||
"(default: the four configs measured on 2026-09-16)")
|
||||
ap.add_argument("--json", default="/tmp/bench-spec-ab.json")
|
||||
args = ap.parse_args()
|
||||
|
||||
@@ -222,29 +262,33 @@ def main():
|
||||
results = {}
|
||||
log("spec A/B start: " + ", ".join(n for n, _ in configs) +
|
||||
" (unit " + UNIT + " stopped for the duration)", logfile)
|
||||
stop_unit()
|
||||
for name, over in configs:
|
||||
env_path = write_env(env_file, name, over)
|
||||
logf = "/tmp/spec-ab-" + name.replace("/", "_") + ".log"
|
||||
sh("FRANK_MEDIUM_ENV_FILE=" + env_path + " setsid bash " + serve_script +
|
||||
" > " + logf + " 2>&1 < /dev/null &")
|
||||
if not wait_health(base):
|
||||
log(name + ": FAILED to become healthy -- " + sh("tail -3 " + logf), logfile)
|
||||
results[name] = {"error": "not healthy", "tail": sh("tail -6 " + logf)}
|
||||
stop_unit()
|
||||
continue
|
||||
rows = measure(base)
|
||||
results[name] = {"vram_gb": vram(), "rows": rows,
|
||||
"spec_log": sh("grep -E '" + SPEC_GREP + "' " + logf + " | tail -4")}
|
||||
for r in rows:
|
||||
log(name + " | " + r["shape"] + " prefill " + str(r["prefill_tps"]) +
|
||||
" tok/s | decode " + str(r["decode_tps"]) + " tok/s | draft " +
|
||||
str(r["draft_n"]) + "/" + str(r["accepted"]) +
|
||||
(" (" + str(round(100.0 * r["accepted"] / r["draft_n"])) + " %)" if r["draft_n"] else "") +
|
||||
(" | copy " + str(r["copy_ratio"]) if "copy_ratio" in r else "") +
|
||||
(" | EMPTY ANSWER" if not r["answer"] else ""), logfile)
|
||||
log(name + " | vram " + str(results[name]["vram_gb"]), logfile)
|
||||
try:
|
||||
stop_unit()
|
||||
for name, over in configs:
|
||||
env_path = write_env(env_file, name, over)
|
||||
logf = "/tmp/spec-ab-" + name.replace("/", "_") + ".log"
|
||||
sh("FRANK_MEDIUM_ENV_FILE=" + env_path + " setsid bash " + serve_script +
|
||||
" > " + logf + " 2>&1 < /dev/null &")
|
||||
if not wait_health(base):
|
||||
log(name + ": FAILED to become healthy -- " + sh("tail -3 " + logf), logfile)
|
||||
results[name] = {"error": "not healthy", "tail": sh("tail -6 " + logf)}
|
||||
stop_unit()
|
||||
continue
|
||||
rows = measure(base)
|
||||
results[name] = {"vram_gb": vram(), "rows": rows,
|
||||
"spec_log": sh("grep -E '" + SPEC_GREP + "' " + logf + " | tail -4")}
|
||||
for r in rows:
|
||||
log(name + " | " + r["shape"] + " prefill " + str(r["prefill_tps"]) +
|
||||
" tok/s | decode " + str(r["decode_tps"]) + " tok/s | draft " +
|
||||
str(r["draft_n"]) + "/" + str(r["accepted"]) +
|
||||
(" (" + str(round(100.0 * r["accepted"] / r["draft_n"])) + " %)"
|
||||
if r["draft_n"] else "") +
|
||||
(" | copy " + str(r["copy_ratio"]) if "copy_ratio" in r else "") +
|
||||
(" | EMPTY ANSWER" if not r["answer"] else ""), logfile)
|
||||
log(name + " | vram " + str(results[name]["vram_gb"]), logfile)
|
||||
stop_unit()
|
||||
finally:
|
||||
restore_unit(base, logfile)
|
||||
|
||||
with open(args.json, "w") as fh:
|
||||
json.dump(results, fh, indent=2)
|
||||
@@ -257,15 +301,6 @@ def main():
|
||||
elif len(set(texts)) > 1:
|
||||
log("WARNING: configurations disagree on greedy text for '" + shape +
|
||||
"' -- speculation is changing answers", logfile)
|
||||
log("restoring " + UNIT, logfile)
|
||||
sh("sudo systemctl start " + UNIT)
|
||||
for _ in range(120):
|
||||
try:
|
||||
with urllib.request.urlopen(base + "/health", timeout=5) as r:
|
||||
if json.load(r).get("status") == "ok":
|
||||
break
|
||||
except Exception:
|
||||
time.sleep(5)
|
||||
log("done; results in " + args.json, logfile)
|
||||
return 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user