dank
6bf70c6a64
Plex's artist row for 272 chapters is a chapter number the ripper put in the artist tag (0.4, 31, 05), which would have filed them under a shelf called 0.4. A narrator is a person, so it must contain a letter; the same test is deliberately not applied to a book title, which may honestly be 1984.
426 lines
20 KiB
Python
Executable File
426 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
r"""media-audiobooks.py — a plan that re-files the audiobook library into one folder per book.
|
|
|
|
media-audiobooks.py --root DIR --enum TSV --rows TSV [--out TSV] [--exceptions FILE]
|
|
|
|
Plex reads this library and Jellyfin does not: its music scanner has no audiobook
|
|
concept, and the tree is indexed entirely by Plex, which matched 58,975 of the
|
|
62,879 files here into its Artist/Album/Track shape. So the authority is
|
|
`scripts/media-census.py audiobooks` — the BOOK (album row), its NARRATOR (that
|
|
book's single artist row) and its CHAPTERS (track rows) — and never the folder a
|
|
release pack made. The plan is:
|
|
|
|
Audiobooks/<Narrator>/<Book> (<year>)/<the file's own name>.<ext>
|
|
... the same folder, " (2)" on the name, when two chapters
|
|
of one book carry the same name
|
|
|
|
What that buys is the one property both media servers' book scanners assume: a
|
|
folder holds exactly one book. Today 212 top-level folders are a mix of
|
|
author-named shelves and collection packs, and the books inside the packs are
|
|
split across `… [Part 1]`, `… [Part 2]` folders three or four deep.
|
|
|
|
Three laws here are the opposite of what a music shelf does, and each is forced
|
|
by what the rows actually hold:
|
|
|
|
* A chapter is NEVER renumbered. 7,556 of the 10,485 books contain at least one
|
|
chapter whose index Plex does not know, so a number written into a name would
|
|
be invented — and it would buy nothing, because both servers order playback by
|
|
the embedded track tag, not by the file name. A wrong number in a name is worse
|
|
than none: it is the number a human later trusts. A file already carrying its
|
|
own number keeps it, untouched.
|
|
* The shelf is the NARRATOR, not the author. Every book has exactly one artist row
|
|
and it is the narrator; the author survives only inside 10% of the album titles
|
|
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving the
|
|
author would mean parsing titles for one book in ten and inventing it for the
|
|
rest. One rule that always resolves beats a shelf that is half author and half
|
|
narrator; the author stays visible in the book folder's own name, which is the
|
|
title Plex shows.
|
|
* Nothing is guessed at. A chapter row with no book or no narrator parks — a
|
|
tag-scan placeholder (`[Unknown Album]`) names neither, since unrelated books
|
|
share it and filing on one merges them, and so does a chapter number the ripper
|
|
mistagged as the artist (`0.4`). An audio file Plex never matched parks, a
|
|
video container parks, and a loose file in a folder holding two books parks.
|
|
|
|
Nothing is deleted and nothing is overwritten. The output is a list of rename(2)
|
|
intents for `scripts/media-apply.py`, whose preflight refuses any target that is
|
|
already claimed and whose journal is the rollback.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import os
|
|
import re
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
|
|
# What a chapter file can be. .m4b is the audiobook container; the rest are the
|
|
# audio a pack ships when it was ripped as music.
|
|
AUDIO_EXT = {".mp3", ".m4a", ".m4b", ".ogg", ".oga", ".opus", ".wav", ".flac",
|
|
".wma", ".aiff", ".aif", ".m4v", ".aac", ".mka"}
|
|
# Never an audiobook, whatever the library it sits in says: Plex reads these as
|
|
# video, and renaming one into an audio extension would claim it is a chapter.
|
|
VIDEO_EXT = {".mkv", ".mp4", ".webm", ".avi", ".mov", ".mpg", ".mpeg", ".ts",
|
|
".m2ts", ".vob", ".ifo", ".wmv", ".iso", ".img", ".bdmv", ".ogm",
|
|
".divx"}
|
|
|
|
# Tag-scan placeholders. These strings are not identities: 1,106 chapters from
|
|
# 14 unrelated top-level folders carry the album `[Unknown Album]`, so filing on
|
|
# them would merge books that have nothing to do with each other — the exact
|
|
# shape this relayout exists to make impossible. Such a chapter parks with the
|
|
# path it has today and the fix belongs in its tags. `Various Artists` is NOT a
|
|
# placeholder: that is what a multi-cast read carries, and the one book here
|
|
# with 1,189 chapters under it is genuinely one book.
|
|
PLACEHOLDER_BOOK = {"[unknown album]", "unknown album", "no title", "untitled",
|
|
"[untitled]", "unknown title", "n/a", "-"}
|
|
PLACEHOLDER_NARRATOR = {"[unknown artist]", "unknown artist", "unknown author",
|
|
"<unknown>", "[unknown]", "unknown"}
|
|
|
|
|
|
def identity(value, placeholders, needs_letter=False):
|
|
"""The tag value Plex matched, when it names something; '' when it does not.
|
|
|
|
A trailing parenthesized group is stripped for the test only, because rippers
|
|
glue timestamps to the placeholder (`Unknown album (03/03/2015 21:39:55)`).
|
|
|
|
`needs_letter` is for a NARRATOR alone: a shelf is a person's name, and 272
|
|
chapters here carry a chapter number in the artist tag (`0.4`, `31`). The same
|
|
test must never judge a BOOK title, which may honestly be `1984`.
|
|
"""
|
|
v = (value or "").strip()
|
|
bare = re.sub(r"\s*\([^()]*\)\s*$", "", v).strip().casefold()
|
|
if bare in placeholders or (needs_letter and not any(c.isalpha() for c in v)):
|
|
return ""
|
|
return v
|
|
|
|
|
|
# The characters Plex documents as breaking playback. They are dropped, not
|
|
# substituted: a replacement character is a guess at what the title meant.
|
|
BAD_CHARS = set('<>:"/\\|?*')
|
|
TRASH_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir"
|
|
r"|lost\+found)(/|$)")
|
|
|
|
ARGS = None
|
|
HELD = {} # path -> (bytes, ext)
|
|
ROWS = {} # path -> the chapter row Plex matched
|
|
PLACED = {} # chapter path -> the destination the plan gave it
|
|
CLAIMED = set() # every destination path the plan has handed out
|
|
MOVES, KEEPS = [], []
|
|
PARKED = defaultdict(list) # reason -> [path]
|
|
NOTES = defaultdict(list) # section -> [line]
|
|
AUDIO_BY_DIR = defaultdict(set) # source dir -> chapter files a row names
|
|
CHILD_OF = defaultdict(set) # dir -> the dirs directly below it holding chapters
|
|
FOLDERS_SEEN = {} # (dir, depth) -> book folders placed below it
|
|
SUBTREE_DEPTH = 2 # how far below a folder its chapters may sit
|
|
BOOKS = {} # (narrator, book, year) -> the book group
|
|
|
|
AMBIGUOUS = object()
|
|
|
|
|
|
def dominant(values):
|
|
"""The spelling the rows mostly agree on: the most frequent value, ties broken
|
|
by sort so the answer never depends on the order rows arrived in."""
|
|
counts = Counter(values)
|
|
return min(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0]
|
|
|
|
|
|
def rel(path):
|
|
"""The path as seen from the library root."""
|
|
return os.path.relpath(path, ARGS.root)
|
|
|
|
|
|
def safe(name):
|
|
"""A name every client can carry: no reserved character, nothing unprintable,
|
|
no trailing dot or space, and short enough to rename."""
|
|
out = "".join(c for c in (name or "")
|
|
if c not in BAD_CHARS and (c.isprintable() or c == " "))
|
|
out = re.sub(r"\s+", " ", out).strip().rstrip(". ")
|
|
while out and len(out.encode("utf-8", "ignore")) > 200:
|
|
out = out[:-4].rstrip()
|
|
return out
|
|
|
|
|
|
def assign(src, folder, name, ext, kind, note):
|
|
"""Book one file into `folder` under `name`, renamed if that exact destination
|
|
is already taken — by another file in the plan, or by a file standing there
|
|
today. Two files claiming one path is the fault that costs a re-run of a 60k
|
|
plan, and so is a plan that only works if the renames happen in the right
|
|
order: the executor refuses any target that is occupied, so an occupied path
|
|
is never a destination, even when its occupant is scheduled to leave.
|
|
A file already standing where the contract puts it is a keep, not a move."""
|
|
stem = safe(name) or "file"
|
|
dst = os.path.join(folder, stem + ext)
|
|
if dst == src:
|
|
CLAIMED.add(dst.lower()) # it is standing on its own target
|
|
KEEPS.append((src, dst, kind))
|
|
return dst
|
|
# A file already in this folder under this name — or under the numbered
|
|
# variant the collision rule gives it — keeps the name it has. Renumbering it
|
|
# every run would turn the plan into a shuffle instead of a fixed point.
|
|
base = os.path.splitext(os.path.basename(src))[0]
|
|
if os.path.dirname(src) == folder and (base == stem
|
|
or base.startswith(stem + " (")):
|
|
CLAIMED.add(src.lower())
|
|
KEEPS.append((src, src, kind))
|
|
return src
|
|
n = 1
|
|
while dst.lower() in CLAIMED:
|
|
n += 1
|
|
dst = os.path.join(folder, f"{stem} ({n}){ext}")
|
|
if n > 1:
|
|
NOTES[note].append(rel(dst))
|
|
CLAIMED.add(dst.lower())
|
|
MOVES.append((src, dst, kind))
|
|
return dst
|
|
|
|
|
|
def park(path, reason):
|
|
"""Hold a file where it is, under the park folder, keeping its relative path."""
|
|
PARKED[reason].append(path)
|
|
dst = os.path.join(ARGS.park_dir, rel(path))
|
|
if dst != path:
|
|
CLAIMED.add(dst.lower())
|
|
MOVES.append((path, dst, "park"))
|
|
|
|
|
|
def under_park(path):
|
|
"""True for a file already parked: that pile is for a human to review, and
|
|
re-parking it every run would only push it deeper."""
|
|
return path.startswith(ARGS.park_dir + os.sep)
|
|
|
|
|
|
# ------------------------------------------------------------------------ books
|
|
def group_books():
|
|
"""Every matched chapter into one group per (narrator, book, year).
|
|
|
|
The key is the identity Plex asserts in its own database, not the folder the
|
|
file sits in: one book arrives as four part folders inside a collection pack,
|
|
and two copies of one book in one pack are one book to Plex, whose album item
|
|
IS that narrator, title and year. The year is part of the key so a 1965
|
|
recording and a 2020 recording that share a title stay two books.
|
|
"""
|
|
for path, r in sorted(ROWS.items()):
|
|
if path not in HELD or under_park(path) or TRASH_DIR.search(path):
|
|
continue
|
|
d = os.path.dirname(path)
|
|
AUDIO_BY_DIR[d].add(path)
|
|
CHILD_OF[os.path.dirname(d)].add(d) # so a cover above Part 1/Part 2 sees both
|
|
narrator = identity(r["narrator"], PLACEHOLDER_NARRATOR, needs_letter=True)
|
|
book = identity(r["book"], PLACEHOLDER_BOOK)
|
|
if not book:
|
|
park(path, "a chapter row that names no book — Plex's album title is a "
|
|
"placeholder like [Unknown Album], so filing on it would "
|
|
"merge unrelated books onto one shelf")
|
|
continue
|
|
if not narrator:
|
|
park(path, "a chapter row that names no narrator — the artist row is a "
|
|
"placeholder like [Unknown Artist] or a chapter number a "
|
|
"ripper mistagged as the artist, so no shelf owns it")
|
|
continue
|
|
key = (narrator.casefold(), book.casefold(), (r["year"] or "").strip())
|
|
g = BOOKS.setdefault(key, {"narrators": [], "books": [], "year": key[2],
|
|
"members": [], "indexed": 0})
|
|
g["members"].append(path)
|
|
g["narrators"].append(narrator)
|
|
g["books"].append(book)
|
|
if str(r["track"] or "").isdigit():
|
|
g["indexed"] += 1
|
|
|
|
|
|
def book_folder(g):
|
|
"""The shelf Plex's own rows name: <narrator>/<book> (<year>). None when a
|
|
name is made of characters no path can hold — inventing a shelf is filing it
|
|
wrong, so those chapters park instead."""
|
|
narrator = safe(dominant(g["narrators"]))
|
|
book = safe(dominant(g["books"]))
|
|
if not narrator or not book:
|
|
return None
|
|
return os.path.join(ARGS.root, narrator,
|
|
f"{book} ({g['year']})" if g["year"] else book)
|
|
|
|
|
|
def place_books():
|
|
"""One folder per book; every chapter keeps the name it has today."""
|
|
for key in sorted(BOOKS):
|
|
g = BOOKS[key]
|
|
folder = book_folder(g)
|
|
if folder is None:
|
|
for path in sorted(g["members"]):
|
|
park(path, "a book or narrator Plex spells with characters no "
|
|
"folder name can hold")
|
|
continue
|
|
for path in sorted(g["members"]):
|
|
ext = HELD[path][1]
|
|
base = os.path.splitext(os.path.basename(path))[0]
|
|
# The name the ripper wrote, unchanged: the track tag inside the file
|
|
# is what orders playback, and 72% of the books here have chapters no
|
|
# row numbers, so any number added here would be invented.
|
|
PLACED[path] = assign(path, folder, base, ext, "chapter",
|
|
"two chapters of one book carried the same name")
|
|
|
|
|
|
# --------------------------------------------------------------------- sidecars
|
|
def folders_below(d, depth):
|
|
"""The book folders the chapters in `d`, or within `depth` directories below
|
|
it, were placed in. Memoised, because every loose file in a big collection
|
|
folder asks the same question about it."""
|
|
if (d, depth) not in FOLDERS_SEEN:
|
|
seen = {os.path.dirname(PLACED[a]) for a in AUDIO_BY_DIR.get(d, ())
|
|
if a in PLACED}
|
|
if depth:
|
|
for child in CHILD_OF.get(d, ()):
|
|
seen |= folders_below(child, depth - 1)
|
|
FOLDERS_SEEN[(d, depth)] = seen
|
|
return FOLDERS_SEEN[(d, depth)]
|
|
|
|
|
|
def owner_folder(d):
|
|
"""The book folder that owns this directory — the first place up the tree
|
|
where chapters sit, in the directory itself or within SUBTREE_DEPTH levels
|
|
below it, which is how a pack whose parts are `… [Part 1]`/`… [Part 2]` owns
|
|
the cover art lying above them. None when no chapter is near enough to say,
|
|
AMBIGUOUS when the chapters there belong to more than one book."""
|
|
while d.startswith(ARGS.root + os.sep) and d != ARGS.root:
|
|
folders = folders_below(d, SUBTREE_DEPTH)
|
|
if folders:
|
|
return next(iter(folders)) if len(folders) == 1 else AMBIGUOUS
|
|
d = os.path.dirname(d)
|
|
return None
|
|
|
|
|
|
def place_other():
|
|
"""Everything that is not a chapter Plex matched: covers, booklets, EPUBs,
|
|
logs, checksums, playlists — and the clutter that is none of those."""
|
|
for path, (_size, ext) in sorted(HELD.items()):
|
|
if path in ROWS or under_park(path) or TRASH_DIR.search(path):
|
|
continue
|
|
d = os.path.dirname(path)
|
|
base = os.path.splitext(os.path.basename(path))[0]
|
|
if ext in VIDEO_EXT:
|
|
park(path, "a video container inside the audiobook library: Plex reads "
|
|
"it as video, and renaming it would claim it is a chapter")
|
|
elif ext in AUDIO_EXT:
|
|
park(path, "an audio file no chapter row names — it is not in the "
|
|
"library Plex matched, so nothing says which book it is")
|
|
else:
|
|
owner = owner_folder(d)
|
|
if owner is AMBIGUOUS:
|
|
park(path, "a file whose folder holds more than one book, so "
|
|
"nothing says which one it belongs to")
|
|
elif owner:
|
|
assign(path, owner, base, ext,
|
|
"playlist" if ext in (".m3u", ".m3u8") else "sidecar",
|
|
"one book folder would have carried two files of the same"
|
|
" sidecar name")
|
|
else:
|
|
park(path, "not an audiobook file, and no book's folder lies with"
|
|
" it to carry it")
|
|
|
|
|
|
# ---------------------------------------------------------------------- outputs
|
|
def write_plan():
|
|
"""mkdir rows for every destination folder and its whole chain, then the moves.
|
|
Parents first: the executor creates directories in the order they are listed."""
|
|
chain = set()
|
|
for _s, dst, _k in MOVES:
|
|
d = os.path.dirname(dst)
|
|
while d and d != ARGS.root and d != os.path.dirname(d):
|
|
chain.add(d)
|
|
d = os.path.dirname(d)
|
|
dirs = sorted(chain, key=lambda p: (p.count("/"), p))
|
|
with open(ARGS.out, "w") as fh:
|
|
fh.write("action\tsrc\tdst\tkind\n")
|
|
for d in dirs:
|
|
fh.write(f"mkdir\t\t{d}\tdir\n")
|
|
for src, dst, kind in sorted(MOVES, key=lambda m: (m[1], m[0])):
|
|
fh.write(f"move\t{src}\t{dst}\t{kind}\n")
|
|
for src, dst, kind in sorted(KEEPS, key=lambda m: (m[1], m[0])):
|
|
fh.write(f"keep\t{src}\t{dst}\t{kind}\n")
|
|
parked = [m for m in MOVES if m[2] == "park"]
|
|
sized = sum(HELD[s][0] for s, _d, _k in MOVES if s in HELD)
|
|
print(f"plan: {len(MOVES) - len(parked)} moves, {len(parked)} parked"
|
|
f" ({sized / 2**40:.2f} TiB touched), {len(KEEPS)} already placed,"
|
|
f" {len(dirs)} folders -> {ARGS.out}")
|
|
|
|
|
|
def write_exceptions():
|
|
with open(ARGS.exceptions, "w") as fh:
|
|
fh.write(f"Audiobook relayout exceptions for {ARGS.root}\n\n")
|
|
fh.write(f"Everything below is parked under {ARGS.park_dir}, keeping the\n"
|
|
"relative path it has today. Nothing is deleted and nothing is\n"
|
|
"overwritten: a parked file is one rename from wherever the plan is\n"
|
|
"pointed next.\n\n")
|
|
for reason in sorted(PARKED, key=lambda r: -len(PARKED[r])):
|
|
rs = PARKED[reason]
|
|
fh.write(f"{len(rs):>6} files {reason}\n")
|
|
for p in sorted(rs, key=rel)[:400]:
|
|
fh.write(f" {rel(p)}\n")
|
|
if len(rs) > 400:
|
|
fh.write(f" … and {len(rs) - 400} more\n")
|
|
fh.write("\n")
|
|
for section in sorted(NOTES, key=lambda s: -len(NOTES[s])):
|
|
lines = sorted(set(NOTES[section]))
|
|
fh.write(f"{len(lines):>6} {section}\n")
|
|
for line in lines[:200]:
|
|
fh.write(f" {line}\n")
|
|
if len(lines) > 200:
|
|
fh.write(f" … and {len(lines) - 200} more\n")
|
|
fh.write("\n")
|
|
unindexed = sorted(f"{safe(dominant(g['narrators']))}/{safe(dominant(g['books']))}"
|
|
for g in BOOKS.values()
|
|
if 0 < g["indexed"] < len(g["members"]))
|
|
if unindexed:
|
|
fh.write(f"{len(unindexed):>6} books have chapters Plex does not number. Their\n")
|
|
fh.write(" files keep the names they carry: playback order comes from the\n"
|
|
" track tag inside each file, and a number written here would be\n"
|
|
" invented for the chapters with no index. Fix it in the tags, not\n"
|
|
" in the names — Plex re-reads tags on a scan and never names files.\n\n")
|
|
fh.write(" Playlists move with their book and their contents are never\n"
|
|
" rewritten: the paths inside an .m3u go stale.\n")
|
|
print(f"exceptions -> {ARGS.exceptions}")
|
|
|
|
|
|
def main():
|
|
global ARGS
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--root", default=os.environ.get("MEDIA_AUDIOBOOKS_ROOT",
|
|
"/mnt/entropy/Audiobooks"))
|
|
ap.add_argument("--enum", default="/tmp/audiobooks-enum.tsv",
|
|
help="the enumerate TSV (bytes, ext, path)")
|
|
ap.add_argument("--rows", default="/tmp/plex-audiobooks.tsv",
|
|
help="the chapter rows scripts/media-census.py audiobooks wrote")
|
|
ap.add_argument("--out", default="/tmp/audiobooks-plan.tsv")
|
|
ap.add_argument("--exceptions", default="/tmp/audiobooks-exceptions.txt")
|
|
ap.add_argument("--park", default="_unsorted",
|
|
help="the folder inside the root that holds what cannot be filed")
|
|
ARGS = ap.parse_args()
|
|
ARGS.root = os.path.normpath(ARGS.root)
|
|
ARGS.park_dir = os.path.join(ARGS.root, ARGS.park)
|
|
read_inputs()
|
|
# Every path a file occupies today is claimed before a single destination is
|
|
# handed out: the executor refuses an occupied target, so a plan that aimed at
|
|
# one would only fail after the renames that were supposed to clear it.
|
|
CLAIMED.update(p.lower() for p in HELD)
|
|
group_books()
|
|
place_books()
|
|
place_other()
|
|
write_plan()
|
|
write_exceptions()
|
|
|
|
|
|
def read_inputs():
|
|
with open(ARGS.enum) as fh:
|
|
for line in fh:
|
|
parts = line.rstrip("\n").split("\t")
|
|
if len(parts) >= 3:
|
|
HELD[parts[2]] = (int(parts[0]), parts[1].lower())
|
|
with open(ARGS.rows, newline="") as fh:
|
|
for row in csv.DictReader(fh, delimiter="\t"):
|
|
if row.get("file"):
|
|
ROWS[row["file"]] = row
|
|
gone = [p for p in ROWS if p not in HELD]
|
|
if gone:
|
|
NOTES["a chapter row whose file the census no longer sees"].extend(gone)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|