Reorganize: llm/old -> old (retired PP/NPU-era scripts, reference only)

old/scripts/analyze-spec-matrix.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Analyze the spec-matrix TSV: per-config median gen t/s, acceptance,
+sorted, with the top configs and per-spec-type bests."""
+import sys, statistics, collections
+
+path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/spec_matrix_results.tsv"
+rows = []
+with open(path) as f:
+    header = f.readline().strip().split("\t")
+    for line in f:
+        p = line.rstrip("\n").split("\t")
+        if len(p) < 9:
+            continue
+        spec, n_max, p_min, run, tps, pps, acc, dn, stop = p[:9]
+        try:
+            tps = float(tps)
+        except ValueError:
+            continue
+        rows.append(dict(spec=spec, n_max=n_max, p_min=p_min, run=run,
+                         tps=tps, pps=float(pps or 0), acc=acc, dn=dn, stop=stop))
+
+by_cfg = collections.defaultdict(list)
+for r in rows:
+    by_cfg[(r["spec"], r["n_max"], r["p_min"])].append(r)
+
+print("%-22s %5s %5s %8s %8s %10s" % ("spec", "n_max", "p_min", "med_tps", "med_pps", "accept"))
+summary = []
+for (spec, n_max, p_min), runs in by_cfg.items():
+    tps_list = [r["tps"] for r in runs]
+    med = statistics.median(tps_list)
+    pps_list = [r["pps"] for r in runs]
+    accs = [r["acc"] for r in runs if r["acc"] not in ("", "None")]
+    acc_s = ""
+    if accs:
+        try:
+            nums = [float(a) for a in accs]
+            acc_s = "%.3f" % (sum(nums)/len(nums))
+        except ValueError:
+            pass
+    summary.append((med, spec, n_max, p_min,
+                    statistics.median(pps_list), acc_s, len(runs)))
+
+summary.sort(reverse=True)
+for med, spec, n_max, p_min, pps, acc, n in summary:
+    print("%-22s %5s %5s %8.2f %8.2f %10s" % (spec, n_max, p_min, med, pps, acc))
+
+print("\n=== TOP 12 CONFIGS ===")
+for med, spec, n_max, p_min, pps, acc, n in summary[:12]:
+    print("  %5.2f t/s  spec=%-22s n_max=%2s p_min=%s" % (med, spec, n_max, p_min))
+
+print("\n=== BEST PER SPEC TYPE ===")
+best = {}
+for med, spec, n_max, p_min, pps, acc, n in summary:
+    key = spec
+    if key not in best:
+        best[key] = (med, n_max, p_min, pps, acc)
+for k, v in best.items():
+    print("  %-22s %5.2f t/s n_max=%s p_min=%s (pps=%s)" % (k, v[0], v[1], v[2], v[3]))
+
+print("\n=== p_min 0.8 vs off (per spec/n_max, med delta) ===")
+from itertools import product
+for spec in sorted(set(r["spec"] for r in rows)):
+    print("  spec:", spec)
+    for n_max in sorted(set(int(r["n_max"]) for r in rows if r["spec"] == spec)):
+        on = [r["tps"] for r in rows if r["spec"] == spec and r["n_max"] == str(n_max) and r["p_min"] == "0.8"]
+        off = [r["tps"] for r in rows if r["spec"] == spec and r["n_max"] == str(n_max) and r["p_min"] == "none"]
+        if on and off:
+            d = statistics.median(on) - statistics.median(off)
+            print("    n_max=%-2s  on=%.2f off=%.2f delta=%+.2f" % (n_max,
+                  statistics.median(on), statistics.median(off), d))