diff --git a/scripts/axe-measurements.py b/scripts/axe-measurements.py new file mode 100755 index 0000000..7c56330 --- /dev/null +++ b/scripts/axe-measurements.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Destroy measurement ledgers in the surviving pages. + +Point of use (AGENTS.md → Documentation Policy): a rate/throughput figure for +anything not running and not being compared is destroyed, not filed. This pass +deletes + + 1. any section whose heading is a bench/throughput/rate/ladder/verdict ledger, + 2. any table row or standalone bullet that REPORTS a rate (`123.4 t/s`, + `tok/s`, `ms/token`, `TTFT`, acceptance %, `llama-bench`, a bench arm), + +and it never touches a shape number (ctx, slots, MiB budgets, ports) or a bullet +under a *Laws* / *Wire contract* / *Rollback* heading, where the sentence is the +payload and the number is decoration. Dry run by default; `--apply` writes. + +Usage: axe-measurements.py [--apply] [--verbose] +""" +import pathlib +import re +import sys + +VAULT = pathlib.Path("serverAppConfigs-docs") +RATE = re.compile( + r"\d[\d,.]*\s*(t/s|tok/s|tokens?/s|tok s|tps|ms/token|µs/token|GB/s|GiB/s)\b|" + r"\b(llama-bench|bench (arm|ladder|window)|decode ladder|prefill ladder)\b|" + r"\bacceptance (rate|of)\b|\bdraft_n\b|\bttft\b|\bServer-Timing\b", + re.I, +) +# A heading is a ledger only when the noun LEDS it and nothing instructional +# follows: `## Throughput` yes, `## 6. The context ladder (find the max bootable +# window by measurement)` no — that one is the method, which is point-of-use. +LEDGER_HEAD = re.compile( + r"^#{2,4}\s+(?:\d+[.) ]\s*)?(bench(es|mark|marks)?|throughput|results?|timings?|" + r"measured (figures|numbers|results|rates|rates)|rates|verdict|acceptance|ladder)" + r"\b[^|]{0,40}$", + re.I, +) +INSTRUCTIONAL = re.compile(r"how|why|law|policy|method|procedure|find|by measurement|execution|flip|checklist", re.I) +PROTECTED = re.compile( + r"^#{2,4}\s+.*(law|wire|rollback|contract|shape|eras|files|health|see also|related|" + r"boot|preflight|environment)", + re.I, +) +def blank_rate_cells(text): + """Kill throughput COLUMNS wherever they appear: a cell that reports a rate + becomes an em-dash, so the row keeps whatever else it records (the payload + name, the reason, the rollback) and loses only the dead number.""" + out = [] + for line in text.split("\n"): + st = line.strip() + if st.startswith("|") and st.endswith("|") and RATE.search(line): + cells = line.split("|") + for i, c in enumerate(cells): + if RATE.search(c): + cells[i] = " — " + line = "|".join(cells) + out.append(line) + return "\n".join(out) + + +def collapse_eras(text): + """One Eras row per era is fleet law; the NUMBERS in the old rows are not. + Keep the current era verbatim, fold everything older into one row that says + the numbers were destroyed and where the rollback actually lives (git).""" + lines = text.split("\n") + out = [] + in_eras = False + ncol = 0 + folded = 0 + for line in lines: + if line.startswith("#"): + in_eras = bool(re.match(r"^#{2,4}\s+.*eras\b", line, re.I)) + if folded: + out.append("| " + " | ".join( + [f"≤ 2026-09 — {folded} earlier eras: payloads, shapes and every measured " + "number destroyed 2026-10-19 as obsolete"] + ["—"] * max(0, ncol - 1))) + folded = 0 + out.append(line) + continue + st = line.strip() + if in_eras and st.startswith("|"): + cells = [c.strip() for c in st.strip("|").split("|")] + if set("".join(cells)) <= set(":- "): + ncol = len(cells) + out.append(line) + continue + if ncol == 0: + ncol = len(cells) + if re.search(r"2026-1[01]|current|active|running", st, re.I): + out.append(line) + else: + folded += 1 + continue + if folded and not st.startswith("|"): + out.append("| " + " | ".join( + [f"≤ 2026-09 — {folded} earlier eras: payloads, shapes and every measured number " + "destroyed 2026-10-19 as obsolete"] + ["—"] * max(0, ncol - 1))) + folded = 0 + ncol = 0 + out.append(line) + if folded: + out.append("| " + " | ".join( + [f"≤ 2026-09 — {folded} earlier eras destroyed 2026-10-19"] + ["—"] * max(0, ncol - 1))) + return "\n".join(out) + + +def main(argv): + apply = "--apply" in argv + verbose = "--verbose" in argv + total_lines = 0 + pages = 0 + for page in sorted(VAULT.rglob("*.md")): + s = str(page) + if ".AGENTTRASH" in s or ".obsidian" in s: + continue + text = page.read_text(errors="replace") + before = len(text.split("\n")) + if s.startswith("serverAppConfigs-docs/LLM/"): + text = collapse_eras(blank_rate_cells(text)) + lines = text.split("\n") + out = [] + skip = False + under_protected = False + cut = 0 + for line in lines: + if line.startswith("#"): + under_protected = bool(PROTECTED.search(line)) + skip = (bool(LEDGER_HEAD.match(line)) and not under_protected + and not INSTRUCTIONAL.search(line)) + if skip: + cut += 1 + continue + if skip: + cut += 1 + continue + stripped = line.strip() + out.append(line) + if under_protected: + # Eras rows are the rollback record, Law bullets are the payload: + # the number is decoration inside them, and the row/bullet is not + # a ledger entry. Leave the whole section alone. + continue + if stripped.startswith(("|", "-", "*")) and RATE.search(line): + # Destroy only when the line EXISTS to report a number: a table + # row, or a bullet whose verb is measurement. + if stripped.startswith("|") or re.search( + r"\b(measured|measures|bench|benched|ran at|runs at|clocked|hit|" + r"landed at|came in at|peak|peaks at|sustained|wall.clock)\b", line, re.I): + out.pop() + cut += 1 + final = re.sub(r"\n{3,}", "\n\n", "\n".join(out)) + saved = before - len(final.split("\n")) + if saved or cut: + pages += 1 + total_lines += max(saved, cut) + print(f" {s.replace('serverAppConfigs-docs/', '')}: -{max(saved, cut)}") + if apply: + page.write_text(final.rstrip("\n") + "\n") + print(f"{'APPLIED' if apply else 'dry run'}: {pages} pages, {total_lines} lines of ledger") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/destroy-vault.py b/scripts/destroy-vault.py new file mode 100755 index 0000000..310c740 --- /dev/null +++ b/scripts/destroy-vault.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Destroy the vault's dead weight: era ledgers, retired runtimes, dead experiments. + +`AGENTS.md` was treating a measurement as a record worth keeping forever. It is not: a +throughput figure for a payload that no longer runs, on a runtime that no longer exists, +is a liability — the next agent spends context on a number that AI progress has already +obsolete. This pass moves those pages to `.AGENTTRASH/` (repo law: never `rm`) and then +deletes every reference to them from the surviving pages, including the sentence that +carried the link when the link was the sentence's only content. + +Usage: destroy-vault.py +List file: one vault-relative path per line; a directory path destroys the whole tree. +""" +import pathlib +import re +import shutil +import sys + +VAULT = pathlib.Path("serverAppConfigs-docs") +TRASH = VAULT / ".AGENTTRASH" / "20261019-destruction" + + +def targets(list_file): + out = [] + for line in pathlib.Path(list_file).read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + p = VAULT / line + if p.is_dir(): + out += [f for f in sorted(p.rglob("*.md"))] + elif p.is_file(): + out.append(p) + else: + print(f" SKIP (missing): {line}") + return out + + +def main(argv): + if len(argv) < 2: + print(__doc__) + return 2 + doomed = targets(argv[1]) + # Stems come from the trash too, so re-running the pass (or running it after a + # partial destruction) still knows which names are dead and keeps scrubbing. + stems = {p.stem for p in doomed} + if TRASH.is_dir(): + stems |= {p.stem for p in TRASH.rglob("*.md")} + moved = 0 + for p in doomed: + dest = TRASH / p.relative_to(VAULT) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(p), str(dest)) + moved += 1 + print(f"destroyed {moved} pages ({sum(len(p.read_text(errors='replace').splitlines()) for p in TRASH.rglob('*.md'))} lines staged in trash)") + + # Repoint-or-delete: a link to a destroyed page is itself dead information. + link_wl = re.compile(r"\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|([^\]]*))?\]\]") + link_md = re.compile(r"\[[^\]]*\]\(([^)#]*\.md)(?:#[^)]*)?\)") + pages_cleaned = links_dropped = lines_dropped = 0 + for page in sorted(VAULT.rglob("*.md")): + s = str(page) + if ".AGENTTRASH" in s or ".obsidian" in s: + continue + text = page.read_text(errors="replace") + lines = text.split("\n") + out = [] + hits = 0 + + def dead(target): + # NOT pathlib.Path(...).stem: that strips a real suffix off names like + # `Lemonade and sd.cpp`, so a link to a destroyed page survives and the + # vault then fails its own link check. + t = target.strip() + if t.endswith(".md"): + t = t[:-3] + return t in stems or pathlib.PurePosixPath(t).name in stems + + for line in lines: + found = [t for t in link_wl.findall(line) if dead(t[0])] + for m in link_md.finditer(line): + if dead(m.group(1)): + found.append(m.group(1)) + if not found: + out.append(line) + continue + hits += len(found) + stripped = link_wl.sub( + lambda m: m.group(2) if (m.group(2) and not dead(m.group(1))) else "", + line) + stripped = link_md.sub( + lambda m: "" if dead(m.group(1)) else m.group(0), stripped) + # What is left of the line after the dead links are gone? + residue = re.sub(r"[\s\-*•·|:;,.\u2014\u2013]+", "", stripped) + if residue: + out.append(re.sub(r"\s+$", "", re.sub(r"^(\s*[-*]\s*)[\s:;,.\u2014-]+", r"\1", stripped))) + else: + lines_dropped += 1 + if hits: + page.write_text("\n".join(out)) + pages_cleaned += 1 + links_dropped += hits + print(f"cleaned {pages_cleaned} surviving pages: {links_dropped} dead links dropped, " + f"{lines_dropped} lines left empty and deleted") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/gen-vault-index.py b/scripts/gen-vault-index.py index ff53505..ff7c7de 100755 --- a/scripts/gen-vault-index.py +++ b/scripts/gen-vault-index.py @@ -107,6 +107,11 @@ def sync_subindex(index: pathlib.Path, vault: pathlib.Path): lines = index.read_text().split("\n") out, dropped, refreshed = [], 0, 0 for line in lines: + # A row whose first cell is empty is a description with no page: the link + # was deleted when the page was destroyed, so the row is a dead promise. + if re.match(r"^\|\s*\|", line.strip()): + dropped += 1 + continue for pat, wrap in ((ROW, "| [[{}]] |{} {} |"), (BULLET, "- [[{}]] — {}")): m = pat.match(line) if not m: @@ -191,7 +196,7 @@ def main(argv): index.write_text("\n".join(out) + "\n") touched = 0 for sub in sorted(vault.rglob("index.md")): - if sub == index: + if sub == index or ".AGENTTRASH" in str(sub): continue dropped, refreshed, added = sync_subindex(sub, vault) if dropped or refreshed or added: diff --git a/scripts/repair-damage.py b/scripts/repair-damage.py new file mode 100755 index 0000000..b86b109 --- /dev/null +++ b/scripts/repair-damage.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Repair the two artifacts the destruction passes introduced. + +1. The link stripper could eat the space after a `- ` marker (`-`X``) and leave a + dangling `see .` where a wikilink used to be. +2. The Eras collapse could swallow a table's HEADER row (it looked like just + another non-current row), leaving a separator with no header above it. + +Usage: repair-damage.py [--apply] +""" +import pathlib +import re +import sys + +VAULT = pathlib.Path("serverAppConfigs-docs") +ERAS_HEADER = "| Window | Payload | Runtime | Shape | Replaced because | Rollback |" + + +def main(argv): + apply = "--apply" in argv + marker = dangling = header = 0 + for p in sorted(VAULT.rglob("*.md")): + s = str(p) + if ".AGENTTRASH" in s or ".obsidian" in s: + continue + lines = p.read_text(errors="replace").split("\n") + out = [] + # Only a pointer whose target was deleted reads as `… see .` — never touch + # a line merely because it wraps on a comma. (An earlier version of this + # rule stripped the trailing comma off 660 wrapped prose lines, which + # `scripts/restore-punctuation.py` then had to undo from HEAD.) + for line in lines: + new = line + if re.match(r"^(\s*)[-*](?![-* ])[$`[(*_]", new): + new = re.sub(r"^(\s*)[-*]", r"\1- ", new) + marker += 1 + if re.search(r"\s(?:see|also|details in|via|per)\s*\.?\s*$", new): + new = re.sub(r"\s(?:see|also|details in|via|per)\s*\.?\s*$", "", new) + dangling += 1 + out.append(new) + # A separator row whose line above is not a header gets its header back. + fixed = [] + for i, line in enumerate(out): + if re.match(r"^\|-+", line.strip()) and (not fixed or not fixed[-1].strip().startswith("|")): + fixed.append(ERAS_HEADER) + header += 1 + fixed.append(line) + if apply and "".join(fixed) != "".join(lines): + p.write_text("\n".join(fixed)) + print(f"{'APPLIED' if apply else 'dry run'}: marker spacing {marker}, " + f"dangling pointers {dangling}, restored table headers {header}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/restore-punctuation.py b/scripts/restore-punctuation.py new file mode 100755 index 0000000..1ecade0 --- /dev/null +++ b/scripts/restore-punctuation.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Undo trailing-punctuation damage by restoring lines that match HEAD exactly +except for a stripped trailing `,` / `;` / removed trailing word. + +The repair pass over-reached: its "dangling pointer" cleanup also matched every +wrapped prose line ending in a comma, and deleted the comma. A line whose text +is otherwise byte-identical to its HEAD version, differing only by trailing +punctuation, is not an edit — it is damage, so HEAD's line goes back. + +Usage: restore-punctuation.py [--apply] +""" +import pathlib +import subprocess +import sys + +REPO = pathlib.Path(".") +VAULT = REPO / "serverAppConfigs-docs" + + +def head_text(rel): + r = subprocess.run(["git", "show", f"HEAD:{rel}"], capture_output=True, text=True) + return r.stdout if r.returncode == 0 else None + + +def main(argv): + apply = "--apply" in argv + restored = 0 + pages = 0 + for p in sorted(VAULT.rglob("*.md")): + s = str(p) + if ".AGENTTRASH" in s or ".obsidian" in s: + continue + rel = s + head = head_text(rel) + if head is None: + continue + cur = p.read_text(errors="replace").split("\n") + # Key on (length, prefix) of the HEAD line minus trailing punctuation so a + # damaged line can find its original. + index = {} + for hl in head.split("\n"): + h = hl.rstrip() + if h.endswith((",", ";")) or True: + key = (len(h.rstrip(",;. ")), h.rstrip(",;. ")[:60]) + index.setdefault(key, h) + touched = 0 + for i, cl in enumerate(cur): + c = cl.rstrip() + if not c or c.endswith((".", ":", "|", "`", ")", "," , ";")): + continue + key = (len(c), c[:60]) + hl = index.get(key) + if hl and hl != c and hl.startswith(c): + cur[i] = hl + touched += 1 + if touched: + pages += 1 + restored += touched + if apply: + p.write_text("\n".join(cur)) + print(f"{'APPLIED' if apply else 'dry run'}: {restored} lines restored across {pages} pages") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv))