diff --git a/scripts/bench-spec-ab.py b/scripts/bench-spec-ab.py index 73294eb..cf2ae9c 100755 --- a/scripts/bench-spec-ab.py +++ b/scripts/bench-spec-ab.py @@ -96,41 +96,35 @@ def log(msg, logfile): def sh(cmd): return subprocess.run(cmd, shell=True, capture_output=True, text=True).stdout.strip() - - def parse_configs(spec): + """`name=SPEC_TYPE[@n_max][|KEY=VALUE]...`, entries joined by ';'. + + KEY=VALUE parts override any env file key, which is how an impl's own + window (SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing + env files.""" out = [] for chunk in spec.split(";"): chunk = chunk.strip() if not chunk: continue name, _, rest = chunk.partition("=") + over = {} + if "|" in rest: + rest, _, extras = rest.partition("|") + for item in extras.split("|"): + if "=" in item: + k, _, v = item.partition("=") + over[k.strip()] = v.strip() n_max = None if "@" in rest: rest, _, n_max = rest.partition("@") - over = {"SPEC_TYPE": rest.strip()} + over["SPEC_TYPE"] = rest.strip() if n_max: over["SPEC_DRAFT_N_MAX"] = n_max.strip() out.append((name.strip(), over)) return out -def write_env(src, name, overrides): - path = "/tmp/spec-ab-" + name.replace("/", "_") + ".env" - with open(src) as fh: - lines = fh.read().splitlines() - out = [] - for line in lines: - for k, v in overrides.items(): - if line.startswith(k + "="): - line = k + "=" + v - break - out.append(line) - with open(path, "w") as fh: - fh.write("\n".join(out) + "\n") - return path - - def post(base, body, timeout=1800): req = urllib.request.Request( base + "/v1/chat/completions", data=json.dumps(body).encode(),