Frank
52dbe18486
`scripts/port-census.py` built its corpus from the new directory, so each page under test was held by itself: every era page reported 0 orphan lines. With the corpus excluding the file under test, each candidate holds 14-63 sole-record lines, so no era page is padding and none was trashed. Deletions of restatement only: - Multi-Node Runtimes.md (-257): restated launch material owned by Qwen3.8 PP Deployment.md; index rows and two Related bullets repointed. - log.md: 66 headings folded into 35 dated entries (one entry per date, newest first), titled headings kept as `date — Title` per the spec's wording, and 78 lines of transient deploy chatter dropped where every fact token in the line is held by an owner page (preserved in .AGENTTRASH/20261019-log-transient/). - Authentic: the 5-page split reverted (-93); the parent carries `okf-exempt: length` and the three stub pages stay deleted. check-okf.py accepts `## YYYY-MM-DD — Title`; port-census.py excludes the file under test and gained a docstring note about the trap. One-off compaction tool moved to ~/.AGENTTRASH. Full suite: 451 tests pass, 0 OKF violations, 155 pages, 0 unresolved links.
101 lines
3.6 KiB
Python
Executable File
101 lines
3.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Port census: prove a merge/collapse lost no fact.
|
|
|
|
For every line that existed in the OLD set and not in the NEW set, check that
|
|
each of its significant tokens (numbers, long identifiers, paths) still occurs
|
|
somewhere in the NEW corpus. A line whose every token survives is a reworded
|
|
line; a line with an unheld token is a candidate fact loss and gets printed.
|
|
|
|
port-census.py <new-glob-dir> <old-file> [<old-file> ...]
|
|
|
|
Old files may be `git-rev:path` (read from git) or a path (e.g. a trash copy).
|
|
"""
|
|
import pathlib
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
def toks(line):
|
|
out = []
|
|
for n in re.findall(r"\d[\d,]*(?:\.\d+)?", line):
|
|
if len(n.replace(",", "").split(".")[0]) >= 3 or (
|
|
"." in n and len(n.split(".")[-1]) >= 2 and len(n.replace(",", "")) >= 4
|
|
):
|
|
out.append(n)
|
|
# Identifiers, flags, paths and backticked spans are the facts worth
|
|
# tracking; ordinary prose words are not — they reword on every rewrite.
|
|
# A hyphen alone is not evidence of an identifier (`long-standing`), so a
|
|
# dashed word counts only with two hyphens (`frank-tts-engine`) or with a
|
|
# digit, dot, slash, colon or `--` inside it.
|
|
for w in re.findall(r"[A-Za-z][\w./:-]{7,}", line):
|
|
core = w.strip(".,;:!?\"')]")
|
|
if (core.startswith("--") or core.isupper()
|
|
or re.search(r"[\d_/:.]", core) or core.count("-") >= 2):
|
|
out.append(core)
|
|
out += [s.strip() for s in re.findall(r"`([^`]+)`", line) if len(s) > 3]
|
|
return out
|
|
|
|
|
|
def held(tok, corpus):
|
|
t = tok.replace(",", "")
|
|
alt = rf"\b{re.escape(t)}\b"
|
|
for c in corpus:
|
|
if tok in c or t in c:
|
|
return True
|
|
if re.search(alt, c.replace(",", "")):
|
|
return True
|
|
return False
|
|
|
|
|
|
def main(argv):
|
|
new_dir = pathlib.Path(argv[1])
|
|
|
|
def corpus_excluding(exclude):
|
|
"""Every vault page except the one under test.
|
|
|
|
The corpus is what must still hold a fact AFTER the file is gone, so a
|
|
file that lives inside the scanned directory has to be removed from it —
|
|
otherwise every token is "held" by the file itself and the census proves
|
|
nothing (it reports 0 losses for every page, which is the bug this
|
|
guards).
|
|
"""
|
|
out = []
|
|
for p in new_dir.rglob("*.md"):
|
|
s = str(p)
|
|
if ".AGENTTRASH" in s or ".obsidian" in s:
|
|
continue
|
|
if exclude is not None and p.resolve() == exclude:
|
|
continue
|
|
out.append(p.read_text(errors="replace"))
|
|
return out
|
|
|
|
lost = checked = 0
|
|
for spec in argv[2:]:
|
|
if spec.startswith("git-"):
|
|
ref, path = spec[4:].split(":", 1)
|
|
old = subprocess.run(["git", "show", f"{ref}:{path}"],
|
|
capture_output=True, text=True).stdout
|
|
label = f"{ref}:{path}"
|
|
exclude = None
|
|
else:
|
|
old = pathlib.Path(spec).read_text(errors="replace")
|
|
label = spec
|
|
exclude = pathlib.Path(spec).resolve()
|
|
blob = "\n".join(corpus_excluding(exclude))
|
|
for line in old.splitlines():
|
|
s = line.strip().strip("->*|# ").strip()
|
|
if len(s) < 30 or s.startswith("```"):
|
|
continue
|
|
checked += 1
|
|
missing = [t for t in toks(s) if not held(t, [blob])]
|
|
if missing:
|
|
lost += 1
|
|
print(f" {label}\n L: {s[:150]}\n unheld: {', '.join(missing[:6])}")
|
|
print(f"{checked} old lines checked, {lost} with unheld tokens")
|
|
return 1 if lost else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv))
|