bench-spec-ab: sweep an impl's own window, not just the spec type

ngram-mod's match window (n_min/n_max) decides which traffic it can see at all
-- the fleet's 48/64 preset is tuned for long verbatim echoes and finds nothing
in a repetitive-but-short output. Config entries now take |KEY=VALUE overrides,
so the window can be swept in the same run as the configurations it applies to.
This commit is contained in:
Frank
2026-09-16 16:31:48 -07:00
parent e767810c9a
commit 9c59513cf5
+13 -19
View File
@@ -96,41 +96,35 @@ def log(msg, logfile):
def sh(cmd):
return subprocess.run(cmd, shell=True, capture_output=True, text=True).stdout.strip()
def parse_configs(spec):
"""`name=SPEC_TYPE[@n_max][|KEY=VALUE]...`, entries joined by ';'.
KEY=VALUE parts override any env file key, which is how an impl's own
window (SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing
env files."""
out = []
for chunk in spec.split(";"):
chunk = chunk.strip()
if not chunk:
continue
name, _, rest = chunk.partition("=")
over = {}
if "|" in rest:
rest, _, extras = rest.partition("|")
for item in extras.split("|"):
if "=" in item:
k, _, v = item.partition("=")
over[k.strip()] = v.strip()
n_max = None
if "@" in rest:
rest, _, n_max = rest.partition("@")
over = {"SPEC_TYPE": rest.strip()}
over["SPEC_TYPE"] = rest.strip()
if n_max:
over["SPEC_DRAFT_N_MAX"] = n_max.strip()
out.append((name.strip(), over))
return out
def write_env(src, name, overrides):
path = "/tmp/spec-ab-" + name.replace("/", "_") + ".env"
with open(src) as fh:
lines = fh.read().splitlines()
out = []
for line in lines:
for k, v in overrides.items():
if line.startswith(k + "="):
line = k + "=" + v
break
out.append(line)
with open(path, "w") as fh:
fh.write("\n".join(out) + "\n")
return path
def post(base, body, timeout=1800):
req = urllib.request.Request(
base + "/v1/chat/completions", data=json.dumps(body).encode(),