#!/usr/bin/env python3
"""Port census: prove a merge/collapse lost no fact.

For every line that existed in the OLD set and not in the NEW set, check that
each of its significant tokens (numbers, long identifiers, paths) still occurs
somewhere in the NEW corpus. A line whose every token survives is a reworded
line; a line with an unheld token is a candidate fact loss and gets printed.

  port-census.py <new-glob-dir> <old-file> [<old-file> ...]

Old files may be `git-rev:path` (read from git) or a path (e.g. a trash copy).
"""
import pathlib
import re
import subprocess
import sys


def toks(line):
    out = []
    for n in re.findall(r"\d[\d,]*(?:\.\d+)?", line):
        if len(n.replace(",", "").split(".")[0]) >= 3 or (
            "." in n and len(n.split(".")[-1]) >= 2 and len(n.replace(",", "")) >= 4
        ):
            out.append(n)
    # Identifiers, flags, paths and backticked spans are the facts worth
    # tracking; ordinary prose words are not — they reword on every rewrite.
    # A hyphen alone is not evidence of an identifier (`long-standing`), so a
    # dashed word counts only with two hyphens (`frank-tts-engine`) or with a
    # digit, dot, slash, colon or `--` inside it.
    for w in re.findall(r"[A-Za-z][\w./:-]{7,}", line):
        core = w.strip(".,;:!?\"')]")
        if (core.startswith("--") or core.isupper()
                or re.search(r"[\d_/:.]", core) or core.count("-") >= 2):
            out.append(core)
    out += [s.strip() for s in re.findall(r"`([^`]+)`", line) if len(s) > 3]
    return out


def held(tok, corpus):
    t = tok.replace(",", "")
    alt = rf"\b{re.escape(t)}\b"
    for c in corpus:
        if tok in c or t in c:
            return True
        if re.search(alt, c.replace(",", "")):
            return True
    return False


def main(argv):
    new_dir = pathlib.Path(argv[1])

    def corpus_excluding(exclude):
        """Every vault page except the one under test.

        The corpus is what must still hold a fact AFTER the file is gone, so a
        file that lives inside the scanned directory has to be removed from it —
        otherwise every token is "held" by the file itself and the census proves
        nothing (it reports 0 losses for every page, which is the bug this
        guards).
        """
        out = []
        for p in new_dir.rglob("*.md"):
            s = str(p)
            if ".AGENTTRASH" in s or ".obsidian" in s:
                continue
            if exclude is not None and p.resolve() == exclude:
                continue
            out.append(p.read_text(errors="replace"))
        return out

    lost = checked = 0
    for spec in argv[2:]:
        if spec.startswith("git-"):
            ref, path = spec[4:].split(":", 1)
            old = subprocess.run(["git", "show", f"{ref}:{path}"],
                                 capture_output=True, text=True).stdout
            label = f"{ref}:{path}"
            exclude = None
        else:
            old = pathlib.Path(spec).read_text(errors="replace")
            label = spec
            exclude = pathlib.Path(spec).resolve()
        blob = "\n".join(corpus_excluding(exclude))
        for line in old.splitlines():
            s = line.strip().strip("->*|# ").strip()
            if len(s) < 30 or s.startswith("```"):
                continue
            checked += 1
            missing = [t for t in toks(s) if not held(t, [blob])]
            if missing:
                lost += 1
                print(f"  {label}\n    L: {s[:150]}\n    unheld: {', '.join(missing[:6])}")
    print(f"{checked} old lines checked, {lost} with unheld tokens")
    return 1 if lost else 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))