bench-spec-ab: sweep an impl's own window, not just the spec type
ngram-mod's match window (n_min/n_max) decides which traffic it can see at all -- the fleet's 48/64 preset is tuned for long verbatim echoes and finds nothing in a repetitive-but-short output. Config entries now take |KEY=VALUE overrides, so the window can be swept in the same run as the configurations it applies to.
This commit is contained in:
1 parent
e767810c9a
commit
9c59513cf5
1 file changed
+13
-19
+13
-19
@@ -96,41 +96,35 @@ def log(msg, logfile):
|
|||||||
|
|
||||||
def sh(cmd):
|
def sh(cmd):
|
||||||
return subprocess.run(cmd, shell=True, capture_output=True, text=True).stdout.strip()
|
return subprocess.run(cmd, shell=True, capture_output=True, text=True).stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
def parse_configs(spec):
|
def parse_configs(spec):
|
||||||
|
"""`name=SPEC_TYPE[@n_max][|KEY=VALUE]...`, entries joined by ';'.
|
||||||
|
|
||||||
|
KEY=VALUE parts override any env file key, which is how an impl's own
|
||||||
|
window (SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing
|
||||||
|
env files."""
|
||||||
out = []
|
out = []
|
||||||
for chunk in spec.split(";"):
|
for chunk in spec.split(";"):
|
||||||
chunk = chunk.strip()
|
chunk = chunk.strip()
|
||||||
if not chunk:
|
if not chunk:
|
||||||
continue
|
continue
|
||||||
name, _, rest = chunk.partition("=")
|
name, _, rest = chunk.partition("=")
|
||||||
|
over = {}
|
||||||
|
if "|" in rest:
|
||||||
|
rest, _, extras = rest.partition("|")
|
||||||
|
for item in extras.split("|"):
|
||||||
|
if "=" in item:
|
||||||
|
k, _, v = item.partition("=")
|
||||||
|
over[k.strip()] = v.strip()
|
||||||
n_max = None
|
n_max = None
|
||||||
if "@" in rest:
|
if "@" in rest:
|
||||||
rest, _, n_max = rest.partition("@")
|
rest, _, n_max = rest.partition("@")
|
||||||
over = {"SPEC_TYPE": rest.strip()}
|
over["SPEC_TYPE"] = rest.strip()
|
||||||
if n_max:
|
if n_max:
|
||||||
over["SPEC_DRAFT_N_MAX"] = n_max.strip()
|
over["SPEC_DRAFT_N_MAX"] = n_max.strip()
|
||||||
out.append((name.strip(), over))
|
out.append((name.strip(), over))
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def write_env(src, name, overrides):
|
|
||||||
path = "/tmp/spec-ab-" + name.replace("/", "_") + ".env"
|
|
||||||
with open(src) as fh:
|
|
||||||
lines = fh.read().splitlines()
|
|
||||||
out = []
|
|
||||||
for line in lines:
|
|
||||||
for k, v in overrides.items():
|
|
||||||
if line.startswith(k + "="):
|
|
||||||
line = k + "=" + v
|
|
||||||
break
|
|
||||||
out.append(line)
|
|
||||||
with open(path, "w") as fh:
|
|
||||||
fh.write("\n".join(out) + "\n")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def post(base, body, timeout=1800):
|
def post(base, body, timeout=1800):
|
||||||
req = urllib.request.Request(
|
req = urllib.request.Request(
|
||||||
base + "/v1/chat/completions", data=json.dumps(body).encode(),
|
base + "/v1/chat/completions", data=json.dumps(body).encode(),
|
||||||
|
|||||||
Reference in new issue
Block a user