71 lines
2.8 KiB
Python
71 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Analyze the spec-matrix TSV: per-config median gen t/s, acceptance,
|
|
sorted, with the top configs and per-spec-type bests."""
|
|
import sys, statistics, collections
|
|
|
|
path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/spec_matrix_results.tsv"
|
|
rows = []
|
|
with open(path) as f:
|
|
header = f.readline().strip().split("\t")
|
|
for line in f:
|
|
p = line.rstrip("\n").split("\t")
|
|
if len(p) < 9:
|
|
continue
|
|
spec, n_max, p_min, run, tps, pps, acc, dn, stop = p[:9]
|
|
try:
|
|
tps = float(tps)
|
|
except ValueError:
|
|
continue
|
|
rows.append(dict(spec=spec, n_max=n_max, p_min=p_min, run=run,
|
|
tps=tps, pps=float(pps or 0), acc=acc, dn=dn, stop=stop))
|
|
|
|
by_cfg = collections.defaultdict(list)
|
|
for r in rows:
|
|
by_cfg[(r["spec"], r["n_max"], r["p_min"])].append(r)
|
|
|
|
print("%-22s %5s %5s %8s %8s %10s" % ("spec", "n_max", "p_min", "med_tps", "med_pps", "accept"))
|
|
summary = []
|
|
for (spec, n_max, p_min), runs in by_cfg.items():
|
|
tps_list = [r["tps"] for r in runs]
|
|
med = statistics.median(tps_list)
|
|
pps_list = [r["pps"] for r in runs]
|
|
accs = [r["acc"] for r in runs if r["acc"] not in ("", "None")]
|
|
acc_s = ""
|
|
if accs:
|
|
try:
|
|
nums = [float(a) for a in accs]
|
|
acc_s = "%.3f" % (sum(nums)/len(nums))
|
|
except ValueError:
|
|
pass
|
|
summary.append((med, spec, n_max, p_min,
|
|
statistics.median(pps_list), acc_s, len(runs)))
|
|
|
|
summary.sort(reverse=True)
|
|
for med, spec, n_max, p_min, pps, acc, n in summary:
|
|
print("%-22s %5s %5s %8.2f %8.2f %10s" % (spec, n_max, p_min, med, pps, acc))
|
|
|
|
print("\n=== TOP 12 CONFIGS ===")
|
|
for med, spec, n_max, p_min, pps, acc, n in summary[:12]:
|
|
print(" %5.2f t/s spec=%-22s n_max=%2s p_min=%s" % (med, spec, n_max, p_min))
|
|
|
|
print("\n=== BEST PER SPEC TYPE ===")
|
|
best = {}
|
|
for med, spec, n_max, p_min, pps, acc, n in summary:
|
|
key = spec
|
|
if key not in best:
|
|
best[key] = (med, n_max, p_min, pps, acc)
|
|
for k, v in best.items():
|
|
print(" %-22s %5.2f t/s n_max=%s p_min=%s (pps=%s)" % (k, v[0], v[1], v[2], v[3]))
|
|
|
|
print("\n=== p_min 0.8 vs off (per spec/n_max, med delta) ===")
|
|
from itertools import product
|
|
for spec in sorted(set(r["spec"] for r in rows)):
|
|
print(" spec:", spec)
|
|
for n_max in sorted(set(int(r["n_max"]) for r in rows if r["spec"] == spec)):
|
|
on = [r["tps"] for r in rows if r["spec"] == spec and r["n_max"] == str(n_max) and r["p_min"] == "0.8"]
|
|
off = [r["tps"] for r in rows if r["spec"] == spec and r["n_max"] == str(n_max) and r["p_min"] == "none"]
|
|
if on and off:
|
|
d = statistics.median(on) - statistics.median(off)
|
|
print(" n_max=%-2s on=%.2f off=%.2f delta=%+.2f" % (n_max,
|
|
statistics.median(on), statistics.median(off), d))
|