bench-spec-ab: sweep an impl's own window, not just the spec type
ngram-mod's match window (n_min/n_max) decides which traffic it can see at all -- the fleet's 48/64 preset is tuned for long verbatim echoes and finds nothing in a repetitive-but-short output. Config entries now take |KEY=VALUE overrides, so the window can be swept in the same run as the configurations it applies to.
This commit is contained in:
+13
-19
@@ -96,41 +96,35 @@ def log(msg, logfile):
|
||||
|
||||
def sh(cmd):
|
||||
return subprocess.run(cmd, shell=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def parse_configs(spec):
|
||||
"""`name=SPEC_TYPE[@n_max][|KEY=VALUE]...`, entries joined by ';'.
|
||||
|
||||
KEY=VALUE parts override any env file key, which is how an impl's own
|
||||
window (SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing
|
||||
env files."""
|
||||
out = []
|
||||
for chunk in spec.split(";"):
|
||||
chunk = chunk.strip()
|
||||
if not chunk:
|
||||
continue
|
||||
name, _, rest = chunk.partition("=")
|
||||
over = {}
|
||||
if "|" in rest:
|
||||
rest, _, extras = rest.partition("|")
|
||||
for item in extras.split("|"):
|
||||
if "=" in item:
|
||||
k, _, v = item.partition("=")
|
||||
over[k.strip()] = v.strip()
|
||||
n_max = None
|
||||
if "@" in rest:
|
||||
rest, _, n_max = rest.partition("@")
|
||||
over = {"SPEC_TYPE": rest.strip()}
|
||||
over["SPEC_TYPE"] = rest.strip()
|
||||
if n_max:
|
||||
over["SPEC_DRAFT_N_MAX"] = n_max.strip()
|
||||
out.append((name.strip(), over))
|
||||
return out
|
||||
|
||||
|
||||
def write_env(src, name, overrides):
|
||||
path = "/tmp/spec-ab-" + name.replace("/", "_") + ".env"
|
||||
with open(src) as fh:
|
||||
lines = fh.read().splitlines()
|
||||
out = []
|
||||
for line in lines:
|
||||
for k, v in overrides.items():
|
||||
if line.startswith(k + "="):
|
||||
line = k + "=" + v
|
||||
break
|
||||
out.append(line)
|
||||
with open(path, "w") as fh:
|
||||
fh.write("\n".join(out) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def post(base, body, timeout=1800):
|
||||
req = urllib.request.Request(
|
||||
base + "/v1/chat/completions", data=json.dumps(body).encode(),
|
||||
|
||||
Reference in New Issue
Block a user