dank
fab17129e2
media-ab-reorg.py: planner for the ABS layout {Author}/{Series}/{Seq -
Year - Title {Narrator}} — claims one book folder per DB item (tracks
of duplicate recordings must not interleave), gates any book folder
that would nest above another (ABS's swallow defect), routes comic and
ebook-only items to intake, holds dedup losers under the campaign dir.
media-ab-verifydups.py: full-blake2b confirmation of size+head/tail
collision survivors. media-ebook-dedupe.py: dedupe the eBooks pile
against the Calibre library (identity key title-core+first-surname,
OPF metadata authority) and emit the calibredb ingest list.
abs-post-reorg.py: server-side ABS driver (cover export, DB-row wipe —
never hard=1 — scan, status) with credentials out of argv.
240 lines
10 KiB
Python
Executable File
240 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""media-ebook-dedupe.py — dedupe the eBooks pile against the Calibre library.
|
|
|
|
2026-11-04 campaign, ebook side. The Calibre library (/mnt/entropy/CalibreBooks,
|
|
6,289 books) is DB-owned and healthy; /mnt/entropy/eBooks is a raw pile whose
|
|
best books already live in the library, several of them twice in the pile.
|
|
Audiobooks' orphaned ebooks/comics land in the pile first (media-ab-reorg's
|
|
intake), so this campaign runs after the audiobook move.
|
|
|
|
Inputs
|
|
--meta ebook-meta.tsv: JSON lines {p, sz, title, authors, ident} for
|
|
epub (embedded OPF) and mobi/azw3 (Palm header title); the
|
|
producer is the campaign's opfcensus.py.
|
|
--enum enum TSV (size, ext, path) for every tree.
|
|
--calibre calibre-books.tsv: JSON lines [id, title, authors, series, path,
|
|
pubdate, formats] dumped from metadata.db.
|
|
--dups full blake2b verdicts (hash, size, path) — only confirmed
|
|
byte-identities remove a file. Optional: without it, identity
|
|
rules alone run and every hash-dependent pair stays a KEEP.
|
|
|
|
Outputs
|
|
--plan media-apply plan (--root /mnt/entropy): hash-duplicates of a
|
|
file already in CalibreBooks -> HOLD; byte-identical files inside
|
|
the pile keep the best format of one book once; non-book junk
|
|
(mp4/mp3/images) -> PARK.
|
|
--ingest one JSON line per surviving BOOK (a set of format files of one
|
|
identity): {paths, title, authors, source}. The ingest runner
|
|
feeds these to calibredb inside the calibre-web image.
|
|
--exceptions every identity pair the file-level evidence could not settle.
|
|
|
|
Match law: identity is (core title, first author surname). Calibre's
|
|
"Dinniman| Matt" and an OPF "Matt Dinniman" share surname "dinniman". Title
|
|
core drops the leading article and all non-alphanumerics. Identity alone NEVER
|
|
deletes against the pile — it only deletes when the same identity sits in the
|
|
Calibre library with the same format (the pile copy adds nothing), or when a
|
|
byte-hash confirms the pair. Format preference when the pile holds one book in
|
|
several copies: epub > azw3 > mobi > pdf > fb2 > lit > rtf > txt.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import unicodedata
|
|
from collections import defaultdict
|
|
|
|
BOOK_EXT = {"epub", "azw3", "mobi", "pdf", "fb2", "lit", "rtf", "txt", "cbr", "cbz"}
|
|
FORMAT_RANK = {"epub": 0, "azw3": 1, "mobi": 2, "pdf": 3, "fb2": 4,
|
|
"cbr": 5, "cbz": 5, "lit": 6, "rtf": 7, "txt": 8}
|
|
ARTICLE = re.compile(r"^(the|a|an)\s+", re.I)
|
|
|
|
|
|
def ascii_fold(s):
|
|
return "".join(c for c in unicodedata.normalize("NFKD", s or "")
|
|
if not unicodedata.combining(c))
|
|
|
|
|
|
def title_core(title):
|
|
t = ascii_fold(title).lower()
|
|
t = ARTICLE.sub("", t)
|
|
t = re.sub(r"\(.*?\)", " ", t) # (1998), (Unabridged) — noise
|
|
t = re.sub(r"[^a-z0-9 ]", " ", t)
|
|
t = re.sub(r"\b(1[5-9]|20)\d\d\b", " ", t) # years
|
|
return " ".join(t.split())[:120]
|
|
|
|
|
|
def surname(author):
|
|
a = ascii_fold(author).strip()
|
|
a = re.split(r"[;,]", a)[0] if "," in a else " ".join(a.split()[-1:])
|
|
return re.sub(r"[^a-z]", "", a.lower())
|
|
|
|
|
|
def first_author(authors):
|
|
parts = [p.strip() for p in re.split(r"[;&]", authors or "") if p.strip()]
|
|
return parts[0] if parts else ""
|
|
|
|
|
|
def stem_title(path):
|
|
s = os.path.splitext(os.path.basename(path))[0]
|
|
return ascii_fold(s)
|
|
|
|
|
|
class Campaign:
|
|
def __init__(self, args):
|
|
self.args = args
|
|
self.root = args.root.rstrip("/")
|
|
self.pile = self.root + "/eBooks"
|
|
self.lib = self.root + "/CalibreBooks"
|
|
self.hold = args.hold
|
|
self.park = args.park
|
|
self.meta = {} # path -> meta dict
|
|
self.pile_files = [] # (sz, ext, path)
|
|
self.cal_by_path = {} # file path under CalibreBooks -> calibre row
|
|
self.cal_identity = {} # (tcore, surname) -> calibre row
|
|
self.drop = {} # path -> reason
|
|
self.exc = defaultdict(list)
|
|
self.moves = []
|
|
self.dirs_pile_book = defaultdict(list) # book dir (pile) -> files
|
|
|
|
# ------------------------------------------------------------- inputs
|
|
def load(self):
|
|
for line in open(self.args.meta):
|
|
r = json.loads(line)
|
|
self.meta[r["p"]] = r
|
|
by_hash = defaultdict(list)
|
|
if self.args.dups:
|
|
for line in open(self.args.dups):
|
|
h, sz, p = line.rstrip("\n").split("\t", 2)
|
|
by_hash[h].append(p)
|
|
self.dup_sets = [ps for ps in by_hash.values() if len(ps) > 1]
|
|
for line in open(self.args.enum):
|
|
sz, ext, p = line.rstrip("\n").split("\t", 2)
|
|
ext = ext.lstrip(".").lower()
|
|
if p.startswith(self.pile + "/") and p not in self.park:
|
|
self.pile_files.append((int(sz), ext, p))
|
|
for line in open(self.args.calibre):
|
|
row = json.loads(line)
|
|
self.cal_identity[(title_core(row[1]), surname(first_author(row[2])))] = row
|
|
|
|
def identity(self, p):
|
|
m = self.meta.get(p)
|
|
if m and m.get("title"):
|
|
t = title_core(m["title"])
|
|
a = surname(first_author(m.get("authors", "")))
|
|
if t:
|
|
return (t, a)
|
|
return (title_core(stem_title(p)), "")
|
|
|
|
# ------------------------------------------------------------- rules
|
|
def run(self):
|
|
pile_paths = {p for _, _, p in self.pile_files}
|
|
# R1/R2: confirmed byte-duplicates
|
|
for ps in self.dup_sets:
|
|
in_lib = [p for p in ps if p.startswith(self.lib + "/")]
|
|
in_pile = [p for p in ps if p.startswith(self.pile + "/")]
|
|
if in_lib:
|
|
for p in in_pile:
|
|
self.drop[p] = "hash-duplicate of CalibreBooks file"
|
|
if len(in_pile) > 1:
|
|
keep = min(in_pile, key=lambda p: (FORMAT_RANK.get(
|
|
os.path.splitext(p)[1].lstrip(".").lower(), 9), len(p), p))
|
|
for p in in_pile:
|
|
if p != keep:
|
|
self.drop[p] = "hash-duplicate inside the pile"
|
|
# R3/R4: identity against the library (same format only deletes)
|
|
cal_files = defaultdict(set) # identity -> formats in the library
|
|
for key, row in self.cal_identity.items():
|
|
cal_files[key] = set(f for f in row[6].split("; ") if f)
|
|
for sz, ext, p in self.pile_files:
|
|
if p in self.drop or ext not in BOOK_EXT:
|
|
continue
|
|
key = self.identity(p)
|
|
if key[0] and key in self.cal_identity:
|
|
if ext in cal_files[key]:
|
|
self.drop[p] = f"same identity+format already in Calibre ({key[0][:40]})"
|
|
else:
|
|
self.exc["pile holds a format the library lacks (add-format on ingest)"].append(
|
|
f"{p}\t{key[0][:50]}\t{self.cal_identity[key][0]}")
|
|
# R5: surviving files grouped by (identity, parent-ish book dir)
|
|
books = defaultdict(list)
|
|
for sz, ext, p in self.pile_files:
|
|
if p in self.drop:
|
|
continue
|
|
if ext not in BOOK_EXT:
|
|
self.moves.append((p, os.path.join(self.park, os.path.relpath(p, self.root)),
|
|
"non-book"))
|
|
continue
|
|
m = self.meta.get(p, {})
|
|
if m.get("err") and ext in ("epub",):
|
|
self.exc["unreadable container (park for a human)"].append(p)
|
|
self.moves.append((p, os.path.join(self.park, os.path.relpath(p, self.root)),
|
|
"broken-book"))
|
|
continue
|
|
key = self.identity(p)
|
|
books[key].append(p)
|
|
for key, ps in books.items():
|
|
first = min(ps, key=lambda p: (FORMAT_RANK.get(
|
|
os.path.splitext(p)[1].lstrip(".").lower(), 9), len(p), p))
|
|
m = self.meta.get(first, {})
|
|
self.ingest = {
|
|
"key": list(key),
|
|
"paths": sorted(ps),
|
|
"title": m.get("title") or stem_title(first),
|
|
"authors": m.get("authors") or "",
|
|
"ident": m.get("ident") or "",
|
|
}
|
|
self.ingest_list.append(self.ingest)
|
|
for p, why in self.drop.items():
|
|
self.moves.append((p, os.path.join(self.hold, os.path.relpath(p, self.root)), "dup"))
|
|
|
|
def write(self):
|
|
dirs = {os.path.dirname(d) for _, d, _ in self.moves}
|
|
all_dirs = set()
|
|
for d in dirs:
|
|
while d and d != self.root:
|
|
all_dirs.add(d)
|
|
d = os.path.dirname(d)
|
|
with open(self.args.plan, "w") as fh:
|
|
fh.write("action\tsrc\tdst\tkind\n")
|
|
for d in sorted(all_dirs):
|
|
fh.write(f"mkdir\t\t{d}\tcampaign\n")
|
|
for src, dst, kind in sorted(self.moves):
|
|
fh.write(f"move\t{src}\t{dst}\t{kind}\n")
|
|
with open(self.args.ingest, "w") as fh:
|
|
for b in self.ingest_list:
|
|
fh.write(json.dumps(b, ensure_ascii=False) + "\n")
|
|
with open(self.args.exceptions, "w") as fh:
|
|
for why in sorted(self.exc):
|
|
fh.write(f"# {why}\n")
|
|
for line in sorted(set(self.exc[why])):
|
|
fh.write(line + "\n")
|
|
fh.write("\n")
|
|
ndrop = sum(1 for _, _, k in self.moves if k == "dup")
|
|
print(f"pile books to ingest: {len(self.ingest_list)} "
|
|
f"files dropped as duplicates: {ndrop} "
|
|
f"non-book/broken moved out: {sum(1 for _, _, k in self.moves if k in ('non-book', 'broken-book'))}")
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--root", default="/mnt/entropy")
|
|
ap.add_argument("--meta", required=True)
|
|
ap.add_argument("--enum", required=True)
|
|
ap.add_argument("--calibre", required=True)
|
|
ap.add_argument("--dups", default=None)
|
|
ap.add_argument("--hold", default="/mnt/entropy/.media-campaign/dup-hold-ebooks-20261104")
|
|
ap.add_argument("--park", default="/mnt/entropy/NeedsProcessing/_from-ebooks-20261104")
|
|
ap.add_argument("--plan", required=True)
|
|
ap.add_argument("--ingest", required=True)
|
|
ap.add_argument("--exceptions", required=True)
|
|
args = ap.parse_args()
|
|
c = Campaign(args)
|
|
c.ingest_list = []
|
|
c.load()
|
|
c.run()
|
|
c.write()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|