#!/usr/bin/env python3 """media-ebook-dedupe.py — dedupe the eBooks pile against the Calibre library. 2026-11-04 campaign, ebook side. The Calibre library (/mnt/entropy/CalibreBooks, 6,289 books) is DB-owned and healthy; /mnt/entropy/eBooks is a raw pile whose best books already live in the library, several of them twice in the pile. Audiobooks' orphaned ebooks/comics land in the pile first (media-ab-reorg's intake), so this campaign runs after the audiobook move. Inputs --meta ebook-meta.tsv: JSON lines {p, sz, title, authors, ident} for epub (embedded OPF) and mobi/azw3 (Palm header title); the producer is the campaign's opfcensus.py. --enum enum TSV (size, ext, path) for every tree. --calibre calibre-books.tsv: JSON lines [id, title, authors, series, path, pubdate, formats] dumped from metadata.db. --dups full blake2b verdicts (hash, size, path) — only confirmed byte-identities remove a file. Optional: without it, identity rules alone run and every hash-dependent pair stays a KEEP. Outputs --plan media-apply plan (--root /mnt/entropy): hash-duplicates of a file already in CalibreBooks -> HOLD; byte-identical files inside the pile keep the best format of one book once; non-book junk (mp4/mp3/images) -> PARK. --ingest one JSON line per surviving BOOK (a set of format files of one identity): {paths, title, authors, source}. The ingest runner feeds these to calibredb inside the calibre-web image. --exceptions every identity pair the file-level evidence could not settle. Match law: identity is (core title, first author surname). Calibre's "Dinniman| Matt" and an OPF "Matt Dinniman" share surname "dinniman". Title core drops the leading article and all non-alphanumerics. Identity alone NEVER deletes against the pile — it only deletes when the same identity sits in the Calibre library with the same format (the pile copy adds nothing), or when a byte-hash confirms the pair. Format preference when the pile holds one book in several copies: epub > azw3 > mobi > pdf > fb2 > lit > rtf > txt. """ import argparse import json import os import re import unicodedata from collections import defaultdict BOOK_EXT = {"epub", "azw3", "mobi", "pdf", "fb2", "lit", "rtf", "txt", "cbr", "cbz"} FORMAT_RANK = {"epub": 0, "azw3": 1, "mobi": 2, "pdf": 3, "fb2": 4, "cbr": 5, "cbz": 5, "lit": 6, "rtf": 7, "txt": 8} ARTICLE = re.compile(r"^(the|a|an)\s+", re.I) def ascii_fold(s): return "".join(c for c in unicodedata.normalize("NFKD", s or "") if not unicodedata.combining(c)) def title_core(title): t = ascii_fold(title).lower() t = ARTICLE.sub("", t) t = re.sub(r"\(.*?\)", " ", t) # (1998), (Unabridged) — noise t = re.sub(r"[^a-z0-9 ]", " ", t) t = re.sub(r"\b(1[5-9]|20)\d\d\b", " ", t) # years return " ".join(t.split())[:120] def surname(author): a = ascii_fold(author).strip() a = re.split(r"[;,]", a)[0] if "," in a else " ".join(a.split()[-1:]) return re.sub(r"[^a-z]", "", a.lower()) def first_author(authors): parts = [p.strip() for p in re.split(r"[;&]", authors or "") if p.strip()] return parts[0] if parts else "" def stem_title(path): s = os.path.splitext(os.path.basename(path))[0] return ascii_fold(s) class Campaign: def __init__(self, args): self.args = args self.root = args.root.rstrip("/") self.pile = self.root + "/eBooks" self.lib = self.root + "/CalibreBooks" self.hold = args.hold self.park = args.park self.meta = {} # path -> meta dict self.pile_files = [] # (sz, ext, path) self.cal_by_path = {} # file path under CalibreBooks -> calibre row self.cal_identity = {} # (tcore, surname) -> calibre row self.drop = {} # path -> reason self.exc = defaultdict(list) self.moves = [] self.dirs_pile_book = defaultdict(list) # book dir (pile) -> files # ------------------------------------------------------------- inputs def load(self): for line in open(self.args.meta): r = json.loads(line) self.meta[r["p"]] = r by_hash = defaultdict(list) if self.args.dups: for line in open(self.args.dups): h, sz, p = line.rstrip("\n").split("\t", 2) by_hash[h].append(p) self.dup_sets = [ps for ps in by_hash.values() if len(ps) > 1] for line in open(self.args.enum): sz, ext, p = line.rstrip("\n").split("\t", 2) ext = ext.lstrip(".").lower() if p.startswith(self.pile + "/") and p not in self.park: self.pile_files.append((int(sz), ext, p)) for line in open(self.args.calibre): row = json.loads(line) self.cal_identity[(title_core(row[1]), surname(first_author(row[2])))] = row def identity(self, p): m = self.meta.get(p) if m and m.get("title"): t = title_core(m["title"]) a = surname(first_author(m.get("authors", ""))) if t: return (t, a) return (title_core(stem_title(p)), "") # ------------------------------------------------------------- rules def run(self): pile_paths = {p for _, _, p in self.pile_files} # R1/R2: confirmed byte-duplicates for ps in self.dup_sets: in_lib = [p for p in ps if p.startswith(self.lib + "/")] in_pile = [p for p in ps if p.startswith(self.pile + "/")] if in_lib: for p in in_pile: self.drop[p] = "hash-duplicate of CalibreBooks file" if len(in_pile) > 1: keep = min(in_pile, key=lambda p: (FORMAT_RANK.get( os.path.splitext(p)[1].lstrip(".").lower(), 9), len(p), p)) for p in in_pile: if p != keep: self.drop[p] = "hash-duplicate inside the pile" # R3/R4: identity against the library (same format only deletes) cal_files = defaultdict(set) # identity -> formats in the library for key, row in self.cal_identity.items(): cal_files[key] = set(f for f in row[6].split("; ") if f) for sz, ext, p in self.pile_files: if p in self.drop or ext not in BOOK_EXT: continue key = self.identity(p) if key[0] and key in self.cal_identity: if ext in cal_files[key]: self.drop[p] = f"same identity+format already in Calibre ({key[0][:40]})" else: self.exc["pile holds a format the library lacks (add-format on ingest)"].append( f"{p}\t{key[0][:50]}\t{self.cal_identity[key][0]}") # R5: surviving files grouped by (identity, parent-ish book dir) books = defaultdict(list) for sz, ext, p in self.pile_files: if p in self.drop: continue if ext not in BOOK_EXT: self.moves.append((p, os.path.join(self.park, os.path.relpath(p, self.root)), "non-book")) continue m = self.meta.get(p, {}) if m.get("err") and ext in ("epub",): self.exc["unreadable container (park for a human)"].append(p) self.moves.append((p, os.path.join(self.park, os.path.relpath(p, self.root)), "broken-book")) continue key = self.identity(p) books[key].append(p) for key, ps in books.items(): first = min(ps, key=lambda p: (FORMAT_RANK.get( os.path.splitext(p)[1].lstrip(".").lower(), 9), len(p), p)) m = self.meta.get(first, {}) self.ingest = { "key": list(key), "paths": sorted(ps), "title": m.get("title") or stem_title(first), "authors": m.get("authors") or "", "ident": m.get("ident") or "", } self.ingest_list.append(self.ingest) for p, why in self.drop.items(): self.moves.append((p, os.path.join(self.hold, os.path.relpath(p, self.root)), "dup")) def write(self): dirs = {os.path.dirname(d) for _, d, _ in self.moves} all_dirs = set() for d in dirs: while d and d != self.root: all_dirs.add(d) d = os.path.dirname(d) with open(self.args.plan, "w") as fh: fh.write("action\tsrc\tdst\tkind\n") for d in sorted(all_dirs): fh.write(f"mkdir\t\t{d}\tcampaign\n") for src, dst, kind in sorted(self.moves): fh.write(f"move\t{src}\t{dst}\t{kind}\n") with open(self.args.ingest, "w") as fh: for b in self.ingest_list: fh.write(json.dumps(b, ensure_ascii=False) + "\n") with open(self.args.exceptions, "w") as fh: for why in sorted(self.exc): fh.write(f"# {why}\n") for line in sorted(set(self.exc[why])): fh.write(line + "\n") fh.write("\n") ndrop = sum(1 for _, _, k in self.moves if k == "dup") print(f"pile books to ingest: {len(self.ingest_list)} " f"files dropped as duplicates: {ndrop} " f"non-book/broken moved out: {sum(1 for _, _, k in self.moves if k in ('non-book', 'broken-book'))}") def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--root", default="/mnt/entropy") ap.add_argument("--meta", required=True) ap.add_argument("--enum", required=True) ap.add_argument("--calibre", required=True) ap.add_argument("--dups", default=None) ap.add_argument("--hold", default="/mnt/entropy/.media-campaign/dup-hold-ebooks-20261104") ap.add_argument("--park", default="/mnt/entropy/NeedsProcessing/_from-ebooks-20261104") ap.add_argument("--plan", required=True) ap.add_argument("--ingest", required=True) ap.add_argument("--exceptions", required=True) args = ap.parse_args() c = Campaign(args) c.ingest_list = [] c.load() c.run() c.write() if __name__ == "__main__": main()