#!/usr/bin/env python3
"""media-ab-verifydups.py — confirm byte-duplicates among size-group survivors.

Stage 2 of the 2026-11-04 Audiobooks dedupe. Stage 1 (partial blake2b of the
first+last 4 MiB, done by the campaign's hashcand.py) over-claims: two files
of equal size and equal head+tail are CANDIDATES, not copies. This tool takes
the candidate files from --partial (rows: size, hash, path), groups them by
(size, partial-hash), full-hashes only members of groups with 2+ members, and
writes verdicts: one line per file whose full hash ties it to at least one
other file.

Output rows: full-hash <TAB> size <TAB> path   (sorted by hash) — a hash that
appears once is not written at all; singletons are not duplicates.

Read-only. Never moves, never deletes. Run it after the partial pass finishes
(pgrep it first; two hashers on one output file have raced before).
"""
import argparse
import hashlib
import os
from collections import defaultdict


def full_hash(path, chunk=1 << 20):
    h = hashlib.blake2b(digest_size=32)
    with open(path, "rb") as fh:
        for block in iter(lambda: fh.read(chunk), b""):
            h.update(block)
    return h.hexdigest()


def main():
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--partial", required=True, help="partial-hashes.tsv")
    ap.add_argument("--out", required=True, help="full-hash verdicts TSV")
    ap.add_argument("--progress-every", type=int, default=50)
    args = ap.parse_args()

    groups = defaultdict(list)
    with open(args.partial) as fh:
        for line in fh:
            f = line.rstrip("\n").split("\t")
            if len(f) != 3 or f[0] == "ERR":
                continue
            groups[(int(f[0]), f[1])].append(f[2])
    survivors = sorted({p for g in groups.values() if len(g) > 1 for p in g})
    print(f"survivor files to full-hash: {len(survivors)}")

    done = set()
    if os.path.exists(args.out):
        with open(args.out) as fh:
            for line in fh:
                done.add(line.rstrip("\n").split("\t", 2)[2])
    full = {}
    for line in open(args.out) if os.path.exists(args.out) else []:
        h, s, p = line.rstrip("\n").split("\t", 2)
        full[p] = h

    todo = [p for p in survivors if p not in done]
    out = open(args.out, "a")
    for i, p in enumerate(todo):
        try:
            full[p] = full_hash(p)
        except OSError as e:
            print(f"SKIP {p}: {e}")
            continue
        if i % args.progress_every == args.progress_every - 1:
            out.flush()
    out.close()

    # Rewrite the whole verdict file keeping only TRUE duplicates (2+ fulls).
    by_hash = defaultdict(list)
    for p in survivors:
        if p in full:
            by_hash[full[p]].append(p)
    with open(args.out, "w") as out:
        for h, ps in sorted(by_hash.items()):
            if len(ps) > 1:
                for p in sorted(ps):
                    out.write(f"{h}\t{os.path.getsize(p)}\t{p}\n")
    dups = {h: ps for h, ps in by_hash.items() if len(ps) > 1}
    n = sum(len(ps) - 1 for ps in dups.values())
    waste = sum((len(ps) - 1) * os.path.getsize(ps[0]) for ps in dups.values())
    print(f"true dup sets: {len(dups)}  redundant files: {n}  "
          f"waste: {waste / 2**30:.1f} GiB")


if __name__ == "__main__":
    main()