dank
fab17129e2
media-ab-reorg.py: planner for the ABS layout {Author}/{Series}/{Seq -
Year - Title {Narrator}} — claims one book folder per DB item (tracks
of duplicate recordings must not interleave), gates any book folder
that would nest above another (ABS's swallow defect), routes comic and
ebook-only items to intake, holds dedup losers under the campaign dir.
media-ab-verifydups.py: full-blake2b confirmation of size+head/tail
collision survivors. media-ebook-dedupe.py: dedupe the eBooks pile
against the Calibre library (identity key title-core+first-surname,
OPF metadata authority) and emit the calibredb ingest list.
abs-post-reorg.py: server-side ABS driver (cover export, DB-row wipe —
never hard=1 — scan, status) with credentials out of argv.
90 lines
3.2 KiB
Python
Executable File
90 lines
3.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""media-ab-verifydups.py — confirm byte-duplicates among size-group survivors.
|
|
|
|
Stage 2 of the 2026-11-04 Audiobooks dedupe. Stage 1 (partial blake2b of the
|
|
first+last 4 MiB, done by the campaign's hashcand.py) over-claims: two files
|
|
of equal size and equal head+tail are CANDIDATES, not copies. This tool takes
|
|
the candidate files from --partial (rows: size, hash, path), groups them by
|
|
(size, partial-hash), full-hashes only members of groups with 2+ members, and
|
|
writes verdicts: one line per file whose full hash ties it to at least one
|
|
other file.
|
|
|
|
Output rows: full-hash <TAB> size <TAB> path (sorted by hash) — a hash that
|
|
appears once is not written at all; singletons are not duplicates.
|
|
|
|
Read-only. Never moves, never deletes. Run it after the partial pass finishes
|
|
(pgrep it first; two hashers on one output file have raced before).
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
from collections import defaultdict
|
|
|
|
|
|
def full_hash(path, chunk=1 << 20):
|
|
h = hashlib.blake2b(digest_size=32)
|
|
with open(path, "rb") as fh:
|
|
for block in iter(lambda: fh.read(chunk), b""):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--partial", required=True, help="partial-hashes.tsv")
|
|
ap.add_argument("--out", required=True, help="full-hash verdicts TSV")
|
|
ap.add_argument("--progress-every", type=int, default=50)
|
|
args = ap.parse_args()
|
|
|
|
groups = defaultdict(list)
|
|
with open(args.partial) as fh:
|
|
for line in fh:
|
|
f = line.rstrip("\n").split("\t")
|
|
if len(f) != 3 or f[0] == "ERR":
|
|
continue
|
|
groups[(int(f[0]), f[1])].append(f[2])
|
|
survivors = sorted({p for g in groups.values() if len(g) > 1 for p in g})
|
|
print(f"survivor files to full-hash: {len(survivors)}")
|
|
|
|
done = set()
|
|
if os.path.exists(args.out):
|
|
with open(args.out) as fh:
|
|
for line in fh:
|
|
done.add(line.rstrip("\n").split("\t", 2)[2])
|
|
full = {}
|
|
for line in open(args.out) if os.path.exists(args.out) else []:
|
|
h, s, p = line.rstrip("\n").split("\t", 2)
|
|
full[p] = h
|
|
|
|
todo = [p for p in survivors if p not in done]
|
|
out = open(args.out, "a")
|
|
for i, p in enumerate(todo):
|
|
try:
|
|
full[p] = full_hash(p)
|
|
except OSError as e:
|
|
print(f"SKIP {p}: {e}")
|
|
continue
|
|
if i % args.progress_every == args.progress_every - 1:
|
|
out.flush()
|
|
out.close()
|
|
|
|
# Rewrite the whole verdict file keeping only TRUE duplicates (2+ fulls).
|
|
by_hash = defaultdict(list)
|
|
for p in survivors:
|
|
if p in full:
|
|
by_hash[full[p]].append(p)
|
|
with open(args.out, "w") as out:
|
|
for h, ps in sorted(by_hash.items()):
|
|
if len(ps) > 1:
|
|
for p in sorted(ps):
|
|
out.write(f"{h}\t{os.path.getsize(p)}\t{p}\n")
|
|
dups = {h: ps for h, ps in by_hash.items() if len(ps) > 1}
|
|
n = sum(len(ps) - 1 for ps in dups.values())
|
|
waste = sum((len(ps) - 1) * os.path.getsize(ps[0]) for ps in dups.values())
|
|
print(f"true dup sets: {len(dups)} redundant files: {n} "
|
|
f"waste: {waste / 2**30:.1f} GiB")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|