feat(scripts): tools to destroy documentation, not preserve it
Point-of-use enforcement needs hands, not advice: - `destroy-vault.py` — moves the pages on a kill list to `.AGENTTRASH/` and then deletes every reference to them from the survivors, including the whole line when the dead link was the line's only content. Stems are seeded from the trash so the pass is re-runnable; `pathlib.Path.stem` is NOT used to compare link targets, because it strips the real suffix off names like `sd.cpp`. - `axe-measurements.py` — deletes bench/throughput ledger sections and rate rows, blanks any table cell that reports a rate, and folds Eras older than the current one into a single row that says the numbers were destroyed and that rollback is a commit. Never touches a shape number or a Law/Wire/Rollback section. - `repair-damage.py` — fixes the two artifacts those passes can leave (a list marker that lost its space, a table separator left without its header). - `restore-punctuation.py` — restores a line whose only difference from HEAD is stripped trailing punctuation. Written after `repair-damage.py`'s first version matched every wrapped prose line ending in a comma and deleted 660 of them: the rule is now `… see .` only, and the incident is recorded in its docstring. - `gen-vault-index.py` — never walks into `.AGENTTRASH/`, and sweeps rows whose page cell is empty (a description left nameless by a destruction). Full suite: 451 tests pass.
This commit is contained in:
Frank
Executable
+164
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Destroy measurement ledgers in the surviving pages.
|
||||
|
||||
Point of use (AGENTS.md → Documentation Policy): a rate/throughput figure for
|
||||
anything not running and not being compared is destroyed, not filed. This pass
|
||||
deletes
|
||||
|
||||
1. any section whose heading is a bench/throughput/rate/ladder/verdict ledger,
|
||||
2. any table row or standalone bullet that REPORTS a rate (`123.4 t/s`,
|
||||
`tok/s`, `ms/token`, `TTFT`, acceptance %, `llama-bench`, a bench arm),
|
||||
|
||||
and it never touches a shape number (ctx, slots, MiB budgets, ports) or a bullet
|
||||
under a *Laws* / *Wire contract* / *Rollback* heading, where the sentence is the
|
||||
payload and the number is decoration. Dry run by default; `--apply` writes.
|
||||
|
||||
Usage: axe-measurements.py [--apply] [--verbose]
|
||||
"""
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
VAULT = pathlib.Path("serverAppConfigs-docs")
|
||||
RATE = re.compile(
|
||||
r"\d[\d,.]*\s*(t/s|tok/s|tokens?/s|tok s|tps|ms/token|µs/token|GB/s|GiB/s)\b|"
|
||||
r"\b(llama-bench|bench (arm|ladder|window)|decode ladder|prefill ladder)\b|"
|
||||
r"\bacceptance (rate|of)\b|\bdraft_n\b|\bttft\b|\bServer-Timing\b",
|
||||
re.I,
|
||||
)
|
||||
# A heading is a ledger only when the noun LEDS it and nothing instructional
|
||||
# follows: `## Throughput` yes, `## 6. The context ladder (find the max bootable
|
||||
# window by measurement)` no — that one is the method, which is point-of-use.
|
||||
LEDGER_HEAD = re.compile(
|
||||
r"^#{2,4}\s+(?:\d+[.) ]\s*)?(bench(es|mark|marks)?|throughput|results?|timings?|"
|
||||
r"measured (figures|numbers|results|rates|rates)|rates|verdict|acceptance|ladder)"
|
||||
r"\b[^|]{0,40}$",
|
||||
re.I,
|
||||
)
|
||||
INSTRUCTIONAL = re.compile(r"how|why|law|policy|method|procedure|find|by measurement|execution|flip|checklist", re.I)
|
||||
PROTECTED = re.compile(
|
||||
r"^#{2,4}\s+.*(law|wire|rollback|contract|shape|eras|files|health|see also|related|"
|
||||
r"boot|preflight|environment)",
|
||||
re.I,
|
||||
)
|
||||
def blank_rate_cells(text):
|
||||
"""Kill throughput COLUMNS wherever they appear: a cell that reports a rate
|
||||
becomes an em-dash, so the row keeps whatever else it records (the payload
|
||||
name, the reason, the rollback) and loses only the dead number."""
|
||||
out = []
|
||||
for line in text.split("\n"):
|
||||
st = line.strip()
|
||||
if st.startswith("|") and st.endswith("|") and RATE.search(line):
|
||||
cells = line.split("|")
|
||||
for i, c in enumerate(cells):
|
||||
if RATE.search(c):
|
||||
cells[i] = " — "
|
||||
line = "|".join(cells)
|
||||
out.append(line)
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def collapse_eras(text):
|
||||
"""One Eras row per era is fleet law; the NUMBERS in the old rows are not.
|
||||
Keep the current era verbatim, fold everything older into one row that says
|
||||
the numbers were destroyed and where the rollback actually lives (git)."""
|
||||
lines = text.split("\n")
|
||||
out = []
|
||||
in_eras = False
|
||||
ncol = 0
|
||||
folded = 0
|
||||
for line in lines:
|
||||
if line.startswith("#"):
|
||||
in_eras = bool(re.match(r"^#{2,4}\s+.*eras\b", line, re.I))
|
||||
if folded:
|
||||
out.append("| " + " | ".join(
|
||||
[f"≤ 2026-09 — {folded} earlier eras: payloads, shapes and every measured "
|
||||
"number destroyed 2026-10-19 as obsolete"] + ["—"] * max(0, ncol - 1)))
|
||||
folded = 0
|
||||
out.append(line)
|
||||
continue
|
||||
st = line.strip()
|
||||
if in_eras and st.startswith("|"):
|
||||
cells = [c.strip() for c in st.strip("|").split("|")]
|
||||
if set("".join(cells)) <= set(":- "):
|
||||
ncol = len(cells)
|
||||
out.append(line)
|
||||
continue
|
||||
if ncol == 0:
|
||||
ncol = len(cells)
|
||||
if re.search(r"2026-1[01]|current|active|running", st, re.I):
|
||||
out.append(line)
|
||||
else:
|
||||
folded += 1
|
||||
continue
|
||||
if folded and not st.startswith("|"):
|
||||
out.append("| " + " | ".join(
|
||||
[f"≤ 2026-09 — {folded} earlier eras: payloads, shapes and every measured number "
|
||||
"destroyed 2026-10-19 as obsolete"] + ["—"] * max(0, ncol - 1)))
|
||||
folded = 0
|
||||
ncol = 0
|
||||
out.append(line)
|
||||
if folded:
|
||||
out.append("| " + " | ".join(
|
||||
[f"≤ 2026-09 — {folded} earlier eras destroyed 2026-10-19"] + ["—"] * max(0, ncol - 1)))
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def main(argv):
|
||||
apply = "--apply" in argv
|
||||
verbose = "--verbose" in argv
|
||||
total_lines = 0
|
||||
pages = 0
|
||||
for page in sorted(VAULT.rglob("*.md")):
|
||||
s = str(page)
|
||||
if ".AGENTTRASH" in s or ".obsidian" in s:
|
||||
continue
|
||||
text = page.read_text(errors="replace")
|
||||
before = len(text.split("\n"))
|
||||
if s.startswith("serverAppConfigs-docs/LLM/"):
|
||||
text = collapse_eras(blank_rate_cells(text))
|
||||
lines = text.split("\n")
|
||||
out = []
|
||||
skip = False
|
||||
under_protected = False
|
||||
cut = 0
|
||||
for line in lines:
|
||||
if line.startswith("#"):
|
||||
under_protected = bool(PROTECTED.search(line))
|
||||
skip = (bool(LEDGER_HEAD.match(line)) and not under_protected
|
||||
and not INSTRUCTIONAL.search(line))
|
||||
if skip:
|
||||
cut += 1
|
||||
continue
|
||||
if skip:
|
||||
cut += 1
|
||||
continue
|
||||
stripped = line.strip()
|
||||
out.append(line)
|
||||
if under_protected:
|
||||
# Eras rows are the rollback record, Law bullets are the payload:
|
||||
# the number is decoration inside them, and the row/bullet is not
|
||||
# a ledger entry. Leave the whole section alone.
|
||||
continue
|
||||
if stripped.startswith(("|", "-", "*")) and RATE.search(line):
|
||||
# Destroy only when the line EXISTS to report a number: a table
|
||||
# row, or a bullet whose verb is measurement.
|
||||
if stripped.startswith("|") or re.search(
|
||||
r"\b(measured|measures|bench|benched|ran at|runs at|clocked|hit|"
|
||||
r"landed at|came in at|peak|peaks at|sustained|wall.clock)\b", line, re.I):
|
||||
out.pop()
|
||||
cut += 1
|
||||
final = re.sub(r"\n{3,}", "\n\n", "\n".join(out))
|
||||
saved = before - len(final.split("\n"))
|
||||
if saved or cut:
|
||||
pages += 1
|
||||
total_lines += max(saved, cut)
|
||||
print(f" {s.replace('serverAppConfigs-docs/', '')}: -{max(saved, cut)}")
|
||||
if apply:
|
||||
page.write_text(final.rstrip("\n") + "\n")
|
||||
print(f"{'APPLIED' if apply else 'dry run'}: {pages} pages, {total_lines} lines of ledger")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
Executable
+109
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Destroy the vault's dead weight: era ledgers, retired runtimes, dead experiments.
|
||||
|
||||
`AGENTS.md` was treating a measurement as a record worth keeping forever. It is not: a
|
||||
throughput figure for a payload that no longer runs, on a runtime that no longer exists,
|
||||
is a liability — the next agent spends context on a number that AI progress has already
|
||||
obsolete. This pass moves those pages to `.AGENTTRASH/` (repo law: never `rm`) and then
|
||||
deletes every reference to them from the surviving pages, including the sentence that
|
||||
carried the link when the link was the sentence's only content.
|
||||
|
||||
Usage: destroy-vault.py <list-file>
|
||||
List file: one vault-relative path per line; a directory path destroys the whole tree.
|
||||
"""
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
VAULT = pathlib.Path("serverAppConfigs-docs")
|
||||
TRASH = VAULT / ".AGENTTRASH" / "20261019-destruction"
|
||||
|
||||
|
||||
def targets(list_file):
|
||||
out = []
|
||||
for line in pathlib.Path(list_file).read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
p = VAULT / line
|
||||
if p.is_dir():
|
||||
out += [f for f in sorted(p.rglob("*.md"))]
|
||||
elif p.is_file():
|
||||
out.append(p)
|
||||
else:
|
||||
print(f" SKIP (missing): {line}")
|
||||
return out
|
||||
|
||||
|
||||
def main(argv):
|
||||
if len(argv) < 2:
|
||||
print(__doc__)
|
||||
return 2
|
||||
doomed = targets(argv[1])
|
||||
# Stems come from the trash too, so re-running the pass (or running it after a
|
||||
# partial destruction) still knows which names are dead and keeps scrubbing.
|
||||
stems = {p.stem for p in doomed}
|
||||
if TRASH.is_dir():
|
||||
stems |= {p.stem for p in TRASH.rglob("*.md")}
|
||||
moved = 0
|
||||
for p in doomed:
|
||||
dest = TRASH / p.relative_to(VAULT)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(p), str(dest))
|
||||
moved += 1
|
||||
print(f"destroyed {moved} pages ({sum(len(p.read_text(errors='replace').splitlines()) for p in TRASH.rglob('*.md'))} lines staged in trash)")
|
||||
|
||||
# Repoint-or-delete: a link to a destroyed page is itself dead information.
|
||||
link_wl = re.compile(r"\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|([^\]]*))?\]\]")
|
||||
link_md = re.compile(r"\[[^\]]*\]\(([^)#]*\.md)(?:#[^)]*)?\)")
|
||||
pages_cleaned = links_dropped = lines_dropped = 0
|
||||
for page in sorted(VAULT.rglob("*.md")):
|
||||
s = str(page)
|
||||
if ".AGENTTRASH" in s or ".obsidian" in s:
|
||||
continue
|
||||
text = page.read_text(errors="replace")
|
||||
lines = text.split("\n")
|
||||
out = []
|
||||
hits = 0
|
||||
|
||||
def dead(target):
|
||||
# NOT pathlib.Path(...).stem: that strips a real suffix off names like
|
||||
# `Lemonade and sd.cpp`, so a link to a destroyed page survives and the
|
||||
# vault then fails its own link check.
|
||||
t = target.strip()
|
||||
if t.endswith(".md"):
|
||||
t = t[:-3]
|
||||
return t in stems or pathlib.PurePosixPath(t).name in stems
|
||||
|
||||
for line in lines:
|
||||
found = [t for t in link_wl.findall(line) if dead(t[0])]
|
||||
for m in link_md.finditer(line):
|
||||
if dead(m.group(1)):
|
||||
found.append(m.group(1))
|
||||
if not found:
|
||||
out.append(line)
|
||||
continue
|
||||
hits += len(found)
|
||||
stripped = link_wl.sub(
|
||||
lambda m: m.group(2) if (m.group(2) and not dead(m.group(1))) else "",
|
||||
line)
|
||||
stripped = link_md.sub(
|
||||
lambda m: "" if dead(m.group(1)) else m.group(0), stripped)
|
||||
# What is left of the line after the dead links are gone?
|
||||
residue = re.sub(r"[\s\-*•·|:;,.\u2014\u2013]+", "", stripped)
|
||||
if residue:
|
||||
out.append(re.sub(r"\s+$", "", re.sub(r"^(\s*[-*]\s*)[\s:;,.\u2014-]+", r"\1", stripped)))
|
||||
else:
|
||||
lines_dropped += 1
|
||||
if hits:
|
||||
page.write_text("\n".join(out))
|
||||
pages_cleaned += 1
|
||||
links_dropped += hits
|
||||
print(f"cleaned {pages_cleaned} surviving pages: {links_dropped} dead links dropped, "
|
||||
f"{lines_dropped} lines left empty and deleted")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -107,6 +107,11 @@ def sync_subindex(index: pathlib.Path, vault: pathlib.Path):
|
||||
lines = index.read_text().split("\n")
|
||||
out, dropped, refreshed = [], 0, 0
|
||||
for line in lines:
|
||||
# A row whose first cell is empty is a description with no page: the link
|
||||
# was deleted when the page was destroyed, so the row is a dead promise.
|
||||
if re.match(r"^\|\s*\|", line.strip()):
|
||||
dropped += 1
|
||||
continue
|
||||
for pat, wrap in ((ROW, "| [[{}]] |{} {} |"), (BULLET, "- [[{}]] — {}")):
|
||||
m = pat.match(line)
|
||||
if not m:
|
||||
@@ -191,7 +196,7 @@ def main(argv):
|
||||
index.write_text("\n".join(out) + "\n")
|
||||
touched = 0
|
||||
for sub in sorted(vault.rglob("index.md")):
|
||||
if sub == index:
|
||||
if sub == index or ".AGENTTRASH" in str(sub):
|
||||
continue
|
||||
dropped, refreshed, added = sync_subindex(sub, vault)
|
||||
if dropped or refreshed or added:
|
||||
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Repair the two artifacts the destruction passes introduced.
|
||||
|
||||
1. The link stripper could eat the space after a `- ` marker (`-`X``) and leave a
|
||||
dangling `see .` where a wikilink used to be.
|
||||
2. The Eras collapse could swallow a table's HEADER row (it looked like just
|
||||
another non-current row), leaving a separator with no header above it.
|
||||
|
||||
Usage: repair-damage.py [--apply]
|
||||
"""
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
VAULT = pathlib.Path("serverAppConfigs-docs")
|
||||
ERAS_HEADER = "| Window | Payload | Runtime | Shape | Replaced because | Rollback |"
|
||||
|
||||
|
||||
def main(argv):
|
||||
apply = "--apply" in argv
|
||||
marker = dangling = header = 0
|
||||
for p in sorted(VAULT.rglob("*.md")):
|
||||
s = str(p)
|
||||
if ".AGENTTRASH" in s or ".obsidian" in s:
|
||||
continue
|
||||
lines = p.read_text(errors="replace").split("\n")
|
||||
out = []
|
||||
# Only a pointer whose target was deleted reads as `… see .` — never touch
|
||||
# a line merely because it wraps on a comma. (An earlier version of this
|
||||
# rule stripped the trailing comma off 660 wrapped prose lines, which
|
||||
# `scripts/restore-punctuation.py` then had to undo from HEAD.)
|
||||
for line in lines:
|
||||
new = line
|
||||
if re.match(r"^(\s*)[-*](?![-* ])[$`[(*_]", new):
|
||||
new = re.sub(r"^(\s*)[-*]", r"\1- ", new)
|
||||
marker += 1
|
||||
if re.search(r"\s(?:see|also|details in|via|per)\s*\.?\s*$", new):
|
||||
new = re.sub(r"\s(?:see|also|details in|via|per)\s*\.?\s*$", "", new)
|
||||
dangling += 1
|
||||
out.append(new)
|
||||
# A separator row whose line above is not a header gets its header back.
|
||||
fixed = []
|
||||
for i, line in enumerate(out):
|
||||
if re.match(r"^\|-+", line.strip()) and (not fixed or not fixed[-1].strip().startswith("|")):
|
||||
fixed.append(ERAS_HEADER)
|
||||
header += 1
|
||||
fixed.append(line)
|
||||
if apply and "".join(fixed) != "".join(lines):
|
||||
p.write_text("\n".join(fixed))
|
||||
print(f"{'APPLIED' if apply else 'dry run'}: marker spacing {marker}, "
|
||||
f"dangling pointers {dangling}, restored table headers {header}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Undo trailing-punctuation damage by restoring lines that match HEAD exactly
|
||||
except for a stripped trailing `,` / `;` / removed trailing word.
|
||||
|
||||
The repair pass over-reached: its "dangling pointer" cleanup also matched every
|
||||
wrapped prose line ending in a comma, and deleted the comma. A line whose text
|
||||
is otherwise byte-identical to its HEAD version, differing only by trailing
|
||||
punctuation, is not an edit — it is damage, so HEAD's line goes back.
|
||||
|
||||
Usage: restore-punctuation.py [--apply]
|
||||
"""
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
REPO = pathlib.Path(".")
|
||||
VAULT = REPO / "serverAppConfigs-docs"
|
||||
|
||||
|
||||
def head_text(rel):
|
||||
r = subprocess.run(["git", "show", f"HEAD:{rel}"], capture_output=True, text=True)
|
||||
return r.stdout if r.returncode == 0 else None
|
||||
|
||||
|
||||
def main(argv):
|
||||
apply = "--apply" in argv
|
||||
restored = 0
|
||||
pages = 0
|
||||
for p in sorted(VAULT.rglob("*.md")):
|
||||
s = str(p)
|
||||
if ".AGENTTRASH" in s or ".obsidian" in s:
|
||||
continue
|
||||
rel = s
|
||||
head = head_text(rel)
|
||||
if head is None:
|
||||
continue
|
||||
cur = p.read_text(errors="replace").split("\n")
|
||||
# Key on (length, prefix) of the HEAD line minus trailing punctuation so a
|
||||
# damaged line can find its original.
|
||||
index = {}
|
||||
for hl in head.split("\n"):
|
||||
h = hl.rstrip()
|
||||
if h.endswith((",", ";")) or True:
|
||||
key = (len(h.rstrip(",;. ")), h.rstrip(",;. ")[:60])
|
||||
index.setdefault(key, h)
|
||||
touched = 0
|
||||
for i, cl in enumerate(cur):
|
||||
c = cl.rstrip()
|
||||
if not c or c.endswith((".", ":", "|", "`", ")", "," , ";")):
|
||||
continue
|
||||
key = (len(c), c[:60])
|
||||
hl = index.get(key)
|
||||
if hl and hl != c and hl.startswith(c):
|
||||
cur[i] = hl
|
||||
touched += 1
|
||||
if touched:
|
||||
pages += 1
|
||||
restored += touched
|
||||
if apply:
|
||||
p.write_text("\n".join(cur))
|
||||
print(f"{'APPLIED' if apply else 'dry run'}: {restored} lines restored across {pages} pages")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
Reference in New Issue
Block a user