#!/usr/bin/env python3
"""Port census: prove a merge/collapse lost no fact.

For every line that existed in the OLD set and not in the NEW set, check that
each of its significant tokens (numbers, long identifiers, paths) still occurs
somewhere in the NEW corpus. A line whose every token survives is a reworded
line; a line with an unheld token is a candidate fact loss and gets printed.

  port-census.py <new-glob-dir> <old-file> [<old-file> ...]

Old files may be `git-rev:path` (read from git) or a path (e.g. a trash copy).
"""
import pathlib
import re
import subprocess
import sys


def toks(line):
    out = []
    for n in re.findall(r"\d[\d,]*(?:\.\d+)?", line):
        if len(n.replace(",", "").split(".")[0]) >= 3 or (
            "." in n and len(n.split(".")[-1]) >= 2 and len(n.replace(",", "")) >= 4
        ):
            out.append(n)
    # Identifiers, flags, paths and backticked spans are the facts worth
    # tracking; ordinary prose words are not — they reword on every rewrite.
    # A hyphen alone is not evidence of an identifier (`long-standing`), so a
    # dashed word counts only with two hyphens (`frank-tts-engine`) or with a
    # digit, dot, slash, colon or `--` inside it.
    for w in re.findall(r"[A-Za-z][\w./:-]{7,}", line):
        core = w.strip(".,;:!?\"')]")
        if (core.startswith("--") or core.isupper()
                or re.search(r"[\d_/:.]", core) or core.count("-") >= 2):
            out.append(core)
    out += [s.strip() for s in re.findall(r"`([^`]+)`", line) if len(s) > 3]
    return out


def held(tok, corpus):
    t = tok.replace(",", "")
    alt = rf"\b{re.escape(t)}\b"
    for c in corpus:
        if tok in c or t in c:
            return True
        if re.search(alt, c.replace(",", "")):
            return True
    return False


def main(argv):
    new_dir = pathlib.Path(argv[1])
    corpus = [
        p.read_text(errors="replace")
        for p in new_dir.rglob("*.md")
        if ".AGENTTRASH" not in str(p) and ".obsidian" not in str(p)
    ]
    blob = "\n".join(corpus)
    lost = 0
    checked = 0
    for spec in argv[2:]:
        if spec.startswith("git-"):
            ref, path = spec[4:].split(":", 1)
            old = subprocess.run(["git", "show", f"{ref}:{path}"],
                                 capture_output=True, text=True).stdout
            label = f"{ref}:{path}"
        else:
            old = pathlib.Path(spec).read_text(errors="replace")
            label = spec
        for line in old.splitlines():
            s = line.strip().strip("->*|# ").strip()
            if len(s) < 30 or s.startswith("```"):
                continue
            checked += 1
            missing = [t for t in toks(s) if not held(t, [blob])]
            if missing:
                lost += 1
                print(f"  {label}\n    L: {s[:150]}\n    unheld: {', '.join(missing[:6])}")
    print(f"{checked} old lines checked, {lost} with unheld tokens")
    return 1 if lost else 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))