Frank
483fd843b7
A hyphen or a trailing period inside a word was enough to make it a tracked token, so `long-standing`, `deployed:` and `cleaned.` were reported as lost facts while real identifiers hid behind them. Require a digit, dot, slash, colon, `--`, two hyphens (`frank-tts-engine`) or ALL-CAPS in the stripped word. The census of the log rewrite went from 979 noisy lines to a set whose unheld tokens are all fact-shaped.
85 lines
3.0 KiB
Python
Executable File
85 lines
3.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Port census: prove a merge/collapse lost no fact.
|
|
|
|
For every line that existed in the OLD set and not in the NEW set, check that
|
|
each of its significant tokens (numbers, long identifiers, paths) still occurs
|
|
somewhere in the NEW corpus. A line whose every token survives is a reworded
|
|
line; a line with an unheld token is a candidate fact loss and gets printed.
|
|
|
|
port-census.py <new-glob-dir> <old-file> [<old-file> ...]
|
|
|
|
Old files may be `git-rev:path` (read from git) or a path (e.g. a trash copy).
|
|
"""
|
|
import pathlib
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
def toks(line):
|
|
out = []
|
|
for n in re.findall(r"\d[\d,]*(?:\.\d+)?", line):
|
|
if len(n.replace(",", "").split(".")[0]) >= 3 or (
|
|
"." in n and len(n.split(".")[-1]) >= 2 and len(n.replace(",", "")) >= 4
|
|
):
|
|
out.append(n)
|
|
# Identifiers, flags, paths and backticked spans are the facts worth
|
|
# tracking; ordinary prose words are not — they reword on every rewrite.
|
|
# A hyphen alone is not evidence of an identifier (`long-standing`), so a
|
|
# dashed word counts only with two hyphens (`frank-tts-engine`) or with a
|
|
# digit, dot, slash, colon or `--` inside it.
|
|
for w in re.findall(r"[A-Za-z][\w./:-]{7,}", line):
|
|
core = w.strip(".,;:!?\"')]")
|
|
if (core.startswith("--") or core.isupper()
|
|
or re.search(r"[\d_/:.]", core) or core.count("-") >= 2):
|
|
out.append(core)
|
|
out += [s.strip() for s in re.findall(r"`([^`]+)`", line) if len(s) > 3]
|
|
return out
|
|
|
|
|
|
def held(tok, corpus):
|
|
t = tok.replace(",", "")
|
|
alt = rf"\b{re.escape(t)}\b"
|
|
for c in corpus:
|
|
if tok in c or t in c:
|
|
return True
|
|
if re.search(alt, c.replace(",", "")):
|
|
return True
|
|
return False
|
|
|
|
|
|
def main(argv):
|
|
new_dir = pathlib.Path(argv[1])
|
|
corpus = [
|
|
p.read_text(errors="replace")
|
|
for p in new_dir.rglob("*.md")
|
|
if ".AGENTTRASH" not in str(p) and ".obsidian" not in str(p)
|
|
]
|
|
blob = "\n".join(corpus)
|
|
lost = 0
|
|
checked = 0
|
|
for spec in argv[2:]:
|
|
if spec.startswith("git-"):
|
|
ref, path = spec[4:].split(":", 1)
|
|
old = subprocess.run(["git", "show", f"{ref}:{path}"],
|
|
capture_output=True, text=True).stdout
|
|
label = f"{ref}:{path}"
|
|
else:
|
|
old = pathlib.Path(spec).read_text(errors="replace")
|
|
label = spec
|
|
for line in old.splitlines():
|
|
s = line.strip().strip("->*|# ").strip()
|
|
if len(s) < 30 or s.startswith("```"):
|
|
continue
|
|
checked += 1
|
|
missing = [t for t in toks(s) if not held(t, [blob])]
|
|
if missing:
|
|
lost += 1
|
|
print(f" {label}\n L: {s[:150]}\n unheld: {', '.join(missing[:6])}")
|
|
print(f"{checked} old lines checked, {lost} with unheld tokens")
|
|
return 1 if lost else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv))
|