Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5dbbfcb8a0 | ||
|
|
02a568fb97 | ||
|
|
1e1527d101 | ||
|
|
9eb85ea1f9 | ||
|
|
96944f0b17 | ||
|
|
33f97cd3b3 | ||
|
|
254fb41fd0 | ||
|
|
3613f69593 | ||
|
|
6bf70c6a64 | ||
|
|
f1b8b60817 | ||
|
|
4ae2bde9ab | ||
|
|
ceb5617a42 | ||
|
|
a7ceda95fd | ||
|
|
a32361ee5e | ||
|
|
ce6b3b1d3b | ||
|
|
382e3eb0b8 | ||
|
|
54a393edab | ||
|
|
a5459b0b1e | ||
|
|
b03931f7ce | ||
|
|
82f6ec55c2 | ||
|
|
1a1dbac268 |
No files matched your search
@@ -851,6 +851,6 @@ history.
|
||||
| Benchmarking (only when ordered) | **Stop and mask the service first**, bench, unmask, restart, verify health. |
|
||||
| Deleting a file | **Move it to trash** — `.AGENTTRASH/` (in-repo) or `~/.AGENTTRASH/YYYYMMDD-topic`. Never `rm`. |
|
||||
| Relayout the movie library | **Census → plan → read the exceptions → `scripts/media-apply.py`** — renames only, journal before every move; never hand-rename, never copy. |
|
||||
| Making media metadata visible to both media servers | **Bind `/var/lib/media-meta/view`** — the overlay `scripts/media-view.sh` installs (array below, NVMe above). Never write the array; never delete through the view. |
|
||||
| Making media metadata visible to both media servers | **Bind `/var/lib/media-meta/view`** — the overlay `scripts/media-view.sh` installs (array below, NVMe above). Never write the array; never delete through the view. **Not yet deployed on Dankserver: the live quadlets still bind the array read-only — see [[Media Metadata View]].** |
|
||||
| Tests exist | **Run them** — verify tests pass after every change. Do not commit failing tests. |
|
||||
| Real secrets | **Never commit** — host `.env` files; `.env.example` holds placeholders only. |
|
||||
+25
-5
@@ -32,6 +32,12 @@ cross a filesystem (rename cannot do it and a silent copy at 24 TB would not).
|
||||
--force relaxes only the guards that are opinion (journal already rolled back,
|
||||
paths outside --root); it never allows overwriting a file.
|
||||
|
||||
--metadata-view names the overlay the media servers read their library through
|
||||
(default /var/lib/media-meta/view). The sidecar follow-up printed after an apply
|
||||
or rollback is conditioned on that path being a mountpoint: where no view is
|
||||
mounted the servers bind the array itself, the sidecars moved with the media,
|
||||
and a reap order would point at a directory that is not there.
|
||||
|
||||
The journal defaults to `media-relayout.jsonl` BESIDE the library root, never
|
||||
inside it: a log inside the library is a file Jellyfin will index.
|
||||
"""
|
||||
@@ -114,13 +120,24 @@ def read_plan(path, root, force):
|
||||
return mkdirs, keeps, moves
|
||||
|
||||
|
||||
# A relayout moves bytes on the raw array; the sidecars the media servers wrote
|
||||
# live in the SSD overlay and stay at the old coordinates. Naming the follow-up
|
||||
# here is the point of use for that fact — see the Media Metadata View page.
|
||||
# The media servers may read the library through the SSD overlay view
|
||||
# (scripts/media-view.sh). A relayout moves bytes on the raw array, so sidecars
|
||||
# the servers wrote through that view stay at the old coordinates — naming the
|
||||
# follow-up here is the point of use for that fact, but only where the view
|
||||
# exists: where it is not mounted the servers bind the array itself, the
|
||||
# sidecars moved with the media, and a reap order points at a directory that
|
||||
# is not there. See the Media Metadata View page for the host's state.
|
||||
METADATA_VIEW = "/var/lib/media-meta/view"
|
||||
METADATA_HINT = ("note: sidecars written through the metadata view did not follow"
|
||||
" these moves — next: scripts/media-view.sh reap")
|
||||
|
||||
|
||||
def metadata_note(args):
|
||||
"""Print the sidecar follow-up only when the overlay is really mounted."""
|
||||
if os.path.ismount(args.metadata_view):
|
||||
print(METADATA_HINT)
|
||||
|
||||
|
||||
def do_apply(args, root, journal, mkdirs, keeps, moves):
|
||||
"""Preflight everything, then folders, then renames. Never a partial start."""
|
||||
problems = []
|
||||
@@ -202,7 +219,7 @@ def do_apply(args, root, journal, mkdirs, keeps, moves):
|
||||
f" {dirs_new} folders created, {len(keeps)} files stayed put")
|
||||
print(f"journal: {journal.path} — undo with: media-apply.py rollback"
|
||||
f" --journal {journal.path} --root {root} --apply")
|
||||
print(METADATA_HINT)
|
||||
metadata_note(args)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -386,7 +403,7 @@ def do_rollback(args, root, journal):
|
||||
t=datetime.datetime.now(datetime.timezone.utc).isoformat())
|
||||
print(f"rolled back: {undone} files back where the census found them,"
|
||||
f" {dirs_gone} empty folders removed, {recreated} swept folders recreated")
|
||||
print(METADATA_HINT)
|
||||
metadata_note(args)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -398,6 +415,9 @@ def main():
|
||||
ap.add_argument("--journal")
|
||||
ap.add_argument("--apply", action="store_true", help="actually change the library")
|
||||
ap.add_argument("--force", action="store_true", help="relax the opinion guards")
|
||||
ap.add_argument("--metadata-view", default=METADATA_VIEW,
|
||||
help="overlay the media servers read through; the sidecar"
|
||||
" note prints only when this path is a mountpoint")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = os.path.abspath(args.root)
|
||||
|
||||
Executable
+425
@@ -0,0 +1,425 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""media-audiobooks.py — a plan that re-files the audiobook library into one folder per book.
|
||||
|
||||
media-audiobooks.py --root DIR --enum TSV --rows TSV [--out TSV] [--exceptions FILE]
|
||||
|
||||
Plex reads this library and Jellyfin does not: its music scanner has no audiobook
|
||||
concept, and the tree is indexed entirely by Plex, which matched 58,975 of the
|
||||
62,879 files here into its Artist/Album/Track shape. So the authority is
|
||||
`scripts/media-census.py audiobooks` — the BOOK (album row), its NARRATOR (that
|
||||
book's single artist row) and its CHAPTERS (track rows) — and never the folder a
|
||||
release pack made. The plan is:
|
||||
|
||||
Audiobooks/<Narrator>/<Book> (<year>)/<the file's own name>.<ext>
|
||||
... the same folder, " (2)" on the name, when two chapters
|
||||
of one book carry the same name
|
||||
|
||||
What that buys is the one property both media servers' book scanners assume: a
|
||||
folder holds exactly one book. Today 212 top-level folders are a mix of
|
||||
author-named shelves and collection packs, and the books inside the packs are
|
||||
split across `… [Part 1]`, `… [Part 2]` folders three or four deep.
|
||||
|
||||
Three laws here are the opposite of what a music shelf does, and each is forced
|
||||
by what the rows actually hold:
|
||||
|
||||
* A chapter is NEVER renumbered. 7,556 of the 10,485 books contain at least one
|
||||
chapter whose index Plex does not know, so a number written into a name would
|
||||
be invented — and it would buy nothing, because both servers order playback by
|
||||
the embedded track tag, not by the file name. A wrong number in a name is worse
|
||||
than none: it is the number a human later trusts. A file already carrying its
|
||||
own number keeps it, untouched.
|
||||
* The shelf is the NARRATOR, not the author. Every book has exactly one artist row
|
||||
and it is the narrator; the author survives only inside 10% of the album titles
|
||||
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving the
|
||||
author would mean parsing titles for one book in ten and inventing it for the
|
||||
rest. One rule that always resolves beats a shelf that is half author and half
|
||||
narrator; the author stays visible in the book folder's own name, which is the
|
||||
title Plex shows.
|
||||
* Nothing is guessed at. A chapter row with no book or no narrator parks — a
|
||||
tag-scan placeholder (`[Unknown Album]`) names neither, since unrelated books
|
||||
share it and filing on one merges them, and so does a chapter number the ripper
|
||||
mistagged as the artist (`0.4`). An audio file Plex never matched parks, a
|
||||
video container parks, and a loose file in a folder holding two books parks.
|
||||
|
||||
Nothing is deleted and nothing is overwritten. The output is a list of rename(2)
|
||||
intents for `scripts/media-apply.py`, whose preflight refuses any target that is
|
||||
already claimed and whose journal is the rollback.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
# What a chapter file can be. .m4b is the audiobook container; the rest are the
|
||||
# audio a pack ships when it was ripped as music.
|
||||
AUDIO_EXT = {".mp3", ".m4a", ".m4b", ".ogg", ".oga", ".opus", ".wav", ".flac",
|
||||
".wma", ".aiff", ".aif", ".m4v", ".aac", ".mka"}
|
||||
# Never an audiobook, whatever the library it sits in says: Plex reads these as
|
||||
# video, and renaming one into an audio extension would claim it is a chapter.
|
||||
VIDEO_EXT = {".mkv", ".mp4", ".webm", ".avi", ".mov", ".mpg", ".mpeg", ".ts",
|
||||
".m2ts", ".vob", ".ifo", ".wmv", ".iso", ".img", ".bdmv", ".ogm",
|
||||
".divx"}
|
||||
|
||||
# Tag-scan placeholders. These strings are not identities: 1,106 chapters from
|
||||
# 14 unrelated top-level folders carry the album `[Unknown Album]`, so filing on
|
||||
# them would merge books that have nothing to do with each other — the exact
|
||||
# shape this relayout exists to make impossible. Such a chapter parks with the
|
||||
# path it has today and the fix belongs in its tags. `Various Artists` is NOT a
|
||||
# placeholder: that is what a multi-cast read carries, and the one book here
|
||||
# with 1,189 chapters under it is genuinely one book.
|
||||
PLACEHOLDER_BOOK = {"[unknown album]", "unknown album", "no title", "untitled",
|
||||
"[untitled]", "unknown title", "n/a", "-"}
|
||||
PLACEHOLDER_NARRATOR = {"[unknown artist]", "unknown artist", "unknown author",
|
||||
"<unknown>", "[unknown]", "unknown"}
|
||||
|
||||
|
||||
def identity(value, placeholders, needs_letter=False):
|
||||
"""The tag value Plex matched, when it names something; '' when it does not.
|
||||
|
||||
A trailing parenthesized group is stripped for the test only, because rippers
|
||||
glue timestamps to the placeholder (`Unknown album (03/03/2015 21:39:55)`).
|
||||
|
||||
`needs_letter` is for a NARRATOR alone: a shelf is a person's name, and 272
|
||||
chapters here carry a chapter number in the artist tag (`0.4`, `31`). The same
|
||||
test must never judge a BOOK title, which may honestly be `1984`.
|
||||
"""
|
||||
v = (value or "").strip()
|
||||
bare = re.sub(r"\s*\([^()]*\)\s*$", "", v).strip().casefold()
|
||||
if bare in placeholders or (needs_letter and not any(c.isalpha() for c in v)):
|
||||
return ""
|
||||
return v
|
||||
|
||||
|
||||
# The characters Plex documents as breaking playback. They are dropped, not
|
||||
# substituted: a replacement character is a guess at what the title meant.
|
||||
BAD_CHARS = set('<>:"/\\|?*')
|
||||
TRASH_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir"
|
||||
r"|lost\+found)(/|$)")
|
||||
|
||||
ARGS = None
|
||||
HELD = {} # path -> (bytes, ext)
|
||||
ROWS = {} # path -> the chapter row Plex matched
|
||||
PLACED = {} # chapter path -> the destination the plan gave it
|
||||
CLAIMED = set() # every destination path the plan has handed out
|
||||
MOVES, KEEPS = [], []
|
||||
PARKED = defaultdict(list) # reason -> [path]
|
||||
NOTES = defaultdict(list) # section -> [line]
|
||||
AUDIO_BY_DIR = defaultdict(set) # source dir -> chapter files a row names
|
||||
CHILD_OF = defaultdict(set) # dir -> the dirs directly below it holding chapters
|
||||
FOLDERS_SEEN = {} # (dir, depth) -> book folders placed below it
|
||||
SUBTREE_DEPTH = 2 # how far below a folder its chapters may sit
|
||||
BOOKS = {} # (narrator, book, year) -> the book group
|
||||
|
||||
AMBIGUOUS = object()
|
||||
|
||||
|
||||
def dominant(values):
|
||||
"""The spelling the rows mostly agree on: the most frequent value, ties broken
|
||||
by sort so the answer never depends on the order rows arrived in."""
|
||||
counts = Counter(values)
|
||||
return min(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0]
|
||||
|
||||
|
||||
def rel(path):
|
||||
"""The path as seen from the library root."""
|
||||
return os.path.relpath(path, ARGS.root)
|
||||
|
||||
|
||||
def safe(name):
|
||||
"""A name every client can carry: no reserved character, nothing unprintable,
|
||||
no trailing dot or space, and short enough to rename."""
|
||||
out = "".join(c for c in (name or "")
|
||||
if c not in BAD_CHARS and (c.isprintable() or c == " "))
|
||||
out = re.sub(r"\s+", " ", out).strip().rstrip(". ")
|
||||
while out and len(out.encode("utf-8", "ignore")) > 200:
|
||||
out = out[:-4].rstrip()
|
||||
return out
|
||||
|
||||
|
||||
def assign(src, folder, name, ext, kind, note):
|
||||
"""Book one file into `folder` under `name`, renamed if that exact destination
|
||||
is already taken — by another file in the plan, or by a file standing there
|
||||
today. Two files claiming one path is the fault that costs a re-run of a 60k
|
||||
plan, and so is a plan that only works if the renames happen in the right
|
||||
order: the executor refuses any target that is occupied, so an occupied path
|
||||
is never a destination, even when its occupant is scheduled to leave.
|
||||
A file already standing where the contract puts it is a keep, not a move."""
|
||||
stem = safe(name) or "file"
|
||||
dst = os.path.join(folder, stem + ext)
|
||||
if dst == src:
|
||||
CLAIMED.add(dst.lower()) # it is standing on its own target
|
||||
KEEPS.append((src, dst, kind))
|
||||
return dst
|
||||
# A file already in this folder under this name — or under the numbered
|
||||
# variant the collision rule gives it — keeps the name it has. Renumbering it
|
||||
# every run would turn the plan into a shuffle instead of a fixed point.
|
||||
base = os.path.splitext(os.path.basename(src))[0]
|
||||
if os.path.dirname(src) == folder and (base == stem
|
||||
or base.startswith(stem + " (")):
|
||||
CLAIMED.add(src.lower())
|
||||
KEEPS.append((src, src, kind))
|
||||
return src
|
||||
n = 1
|
||||
while dst.lower() in CLAIMED:
|
||||
n += 1
|
||||
dst = os.path.join(folder, f"{stem} ({n}){ext}")
|
||||
if n > 1:
|
||||
NOTES[note].append(rel(dst))
|
||||
CLAIMED.add(dst.lower())
|
||||
MOVES.append((src, dst, kind))
|
||||
return dst
|
||||
|
||||
|
||||
def park(path, reason):
|
||||
"""Hold a file where it is, under the park folder, keeping its relative path."""
|
||||
PARKED[reason].append(path)
|
||||
dst = os.path.join(ARGS.park_dir, rel(path))
|
||||
if dst != path:
|
||||
CLAIMED.add(dst.lower())
|
||||
MOVES.append((path, dst, "park"))
|
||||
|
||||
|
||||
def under_park(path):
|
||||
"""True for a file already parked: that pile is for a human to review, and
|
||||
re-parking it every run would only push it deeper."""
|
||||
return path.startswith(ARGS.park_dir + os.sep)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------ books
|
||||
def group_books():
|
||||
"""Every matched chapter into one group per (narrator, book, year).
|
||||
|
||||
The key is the identity Plex asserts in its own database, not the folder the
|
||||
file sits in: one book arrives as four part folders inside a collection pack,
|
||||
and two copies of one book in one pack are one book to Plex, whose album item
|
||||
IS that narrator, title and year. The year is part of the key so a 1965
|
||||
recording and a 2020 recording that share a title stay two books.
|
||||
"""
|
||||
for path, r in sorted(ROWS.items()):
|
||||
if path not in HELD or under_park(path) or TRASH_DIR.search(path):
|
||||
continue
|
||||
d = os.path.dirname(path)
|
||||
AUDIO_BY_DIR[d].add(path)
|
||||
CHILD_OF[os.path.dirname(d)].add(d) # so a cover above Part 1/Part 2 sees both
|
||||
narrator = identity(r["narrator"], PLACEHOLDER_NARRATOR, needs_letter=True)
|
||||
book = identity(r["book"], PLACEHOLDER_BOOK)
|
||||
if not book:
|
||||
park(path, "a chapter row that names no book — Plex's album title is a "
|
||||
"placeholder like [Unknown Album], so filing on it would "
|
||||
"merge unrelated books onto one shelf")
|
||||
continue
|
||||
if not narrator:
|
||||
park(path, "a chapter row that names no narrator — the artist row is a "
|
||||
"placeholder like [Unknown Artist] or a chapter number a "
|
||||
"ripper mistagged as the artist, so no shelf owns it")
|
||||
continue
|
||||
key = (narrator.casefold(), book.casefold(), (r["year"] or "").strip())
|
||||
g = BOOKS.setdefault(key, {"narrators": [], "books": [], "year": key[2],
|
||||
"members": [], "indexed": 0})
|
||||
g["members"].append(path)
|
||||
g["narrators"].append(narrator)
|
||||
g["books"].append(book)
|
||||
if str(r["track"] or "").isdigit():
|
||||
g["indexed"] += 1
|
||||
|
||||
|
||||
def book_folder(g):
|
||||
"""The shelf Plex's own rows name: <narrator>/<book> (<year>). None when a
|
||||
name is made of characters no path can hold — inventing a shelf is filing it
|
||||
wrong, so those chapters park instead."""
|
||||
narrator = safe(dominant(g["narrators"]))
|
||||
book = safe(dominant(g["books"]))
|
||||
if not narrator or not book:
|
||||
return None
|
||||
return os.path.join(ARGS.root, narrator,
|
||||
f"{book} ({g['year']})" if g["year"] else book)
|
||||
|
||||
|
||||
def place_books():
|
||||
"""One folder per book; every chapter keeps the name it has today."""
|
||||
for key in sorted(BOOKS):
|
||||
g = BOOKS[key]
|
||||
folder = book_folder(g)
|
||||
if folder is None:
|
||||
for path in sorted(g["members"]):
|
||||
park(path, "a book or narrator Plex spells with characters no "
|
||||
"folder name can hold")
|
||||
continue
|
||||
for path in sorted(g["members"]):
|
||||
ext = HELD[path][1]
|
||||
base = os.path.splitext(os.path.basename(path))[0]
|
||||
# The name the ripper wrote, unchanged: the track tag inside the file
|
||||
# is what orders playback, and 72% of the books here have chapters no
|
||||
# row numbers, so any number added here would be invented.
|
||||
PLACED[path] = assign(path, folder, base, ext, "chapter",
|
||||
"two chapters of one book carried the same name")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- sidecars
|
||||
def folders_below(d, depth):
|
||||
"""The book folders the chapters in `d`, or within `depth` directories below
|
||||
it, were placed in. Memoised, because every loose file in a big collection
|
||||
folder asks the same question about it."""
|
||||
if (d, depth) not in FOLDERS_SEEN:
|
||||
seen = {os.path.dirname(PLACED[a]) for a in AUDIO_BY_DIR.get(d, ())
|
||||
if a in PLACED}
|
||||
if depth:
|
||||
for child in CHILD_OF.get(d, ()):
|
||||
seen |= folders_below(child, depth - 1)
|
||||
FOLDERS_SEEN[(d, depth)] = seen
|
||||
return FOLDERS_SEEN[(d, depth)]
|
||||
|
||||
|
||||
def owner_folder(d):
|
||||
"""The book folder that owns this directory — the first place up the tree
|
||||
where chapters sit, in the directory itself or within SUBTREE_DEPTH levels
|
||||
below it, which is how a pack whose parts are `… [Part 1]`/`… [Part 2]` owns
|
||||
the cover art lying above them. None when no chapter is near enough to say,
|
||||
AMBIGUOUS when the chapters there belong to more than one book."""
|
||||
while d.startswith(ARGS.root + os.sep) and d != ARGS.root:
|
||||
folders = folders_below(d, SUBTREE_DEPTH)
|
||||
if folders:
|
||||
return next(iter(folders)) if len(folders) == 1 else AMBIGUOUS
|
||||
d = os.path.dirname(d)
|
||||
return None
|
||||
|
||||
|
||||
def place_other():
|
||||
"""Everything that is not a chapter Plex matched: covers, booklets, EPUBs,
|
||||
logs, checksums, playlists — and the clutter that is none of those."""
|
||||
for path, (_size, ext) in sorted(HELD.items()):
|
||||
if path in ROWS or under_park(path) or TRASH_DIR.search(path):
|
||||
continue
|
||||
d = os.path.dirname(path)
|
||||
base = os.path.splitext(os.path.basename(path))[0]
|
||||
if ext in VIDEO_EXT:
|
||||
park(path, "a video container inside the audiobook library: Plex reads "
|
||||
"it as video, and renaming it would claim it is a chapter")
|
||||
elif ext in AUDIO_EXT:
|
||||
park(path, "an audio file no chapter row names — it is not in the "
|
||||
"library Plex matched, so nothing says which book it is")
|
||||
else:
|
||||
owner = owner_folder(d)
|
||||
if owner is AMBIGUOUS:
|
||||
park(path, "a file whose folder holds more than one book, so "
|
||||
"nothing says which one it belongs to")
|
||||
elif owner:
|
||||
assign(path, owner, base, ext,
|
||||
"playlist" if ext in (".m3u", ".m3u8") else "sidecar",
|
||||
"one book folder would have carried two files of the same"
|
||||
" sidecar name")
|
||||
else:
|
||||
park(path, "not an audiobook file, and no book's folder lies with"
|
||||
" it to carry it")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- outputs
|
||||
def write_plan():
|
||||
"""mkdir rows for every destination folder and its whole chain, then the moves.
|
||||
Parents first: the executor creates directories in the order they are listed."""
|
||||
chain = set()
|
||||
for _s, dst, _k in MOVES:
|
||||
d = os.path.dirname(dst)
|
||||
while d and d != ARGS.root and d != os.path.dirname(d):
|
||||
chain.add(d)
|
||||
d = os.path.dirname(d)
|
||||
dirs = sorted(chain, key=lambda p: (p.count("/"), p))
|
||||
with open(ARGS.out, "w") as fh:
|
||||
fh.write("action\tsrc\tdst\tkind\n")
|
||||
for d in dirs:
|
||||
fh.write(f"mkdir\t\t{d}\tdir\n")
|
||||
for src, dst, kind in sorted(MOVES, key=lambda m: (m[1], m[0])):
|
||||
fh.write(f"move\t{src}\t{dst}\t{kind}\n")
|
||||
for src, dst, kind in sorted(KEEPS, key=lambda m: (m[1], m[0])):
|
||||
fh.write(f"keep\t{src}\t{dst}\t{kind}\n")
|
||||
parked = [m for m in MOVES if m[2] == "park"]
|
||||
sized = sum(HELD[s][0] for s, _d, _k in MOVES if s in HELD)
|
||||
print(f"plan: {len(MOVES) - len(parked)} moves, {len(parked)} parked"
|
||||
f" ({sized / 2**40:.2f} TiB touched), {len(KEEPS)} already placed,"
|
||||
f" {len(dirs)} folders -> {ARGS.out}")
|
||||
|
||||
|
||||
def write_exceptions():
|
||||
with open(ARGS.exceptions, "w") as fh:
|
||||
fh.write(f"Audiobook relayout exceptions for {ARGS.root}\n\n")
|
||||
fh.write(f"Everything below is parked under {ARGS.park_dir}, keeping the\n"
|
||||
"relative path it has today. Nothing is deleted and nothing is\n"
|
||||
"overwritten: a parked file is one rename from wherever the plan is\n"
|
||||
"pointed next.\n\n")
|
||||
for reason in sorted(PARKED, key=lambda r: -len(PARKED[r])):
|
||||
rs = PARKED[reason]
|
||||
fh.write(f"{len(rs):>6} files {reason}\n")
|
||||
for p in sorted(rs, key=rel)[:400]:
|
||||
fh.write(f" {rel(p)}\n")
|
||||
if len(rs) > 400:
|
||||
fh.write(f" … and {len(rs) - 400} more\n")
|
||||
fh.write("\n")
|
||||
for section in sorted(NOTES, key=lambda s: -len(NOTES[s])):
|
||||
lines = sorted(set(NOTES[section]))
|
||||
fh.write(f"{len(lines):>6} {section}\n")
|
||||
for line in lines[:200]:
|
||||
fh.write(f" {line}\n")
|
||||
if len(lines) > 200:
|
||||
fh.write(f" … and {len(lines) - 200} more\n")
|
||||
fh.write("\n")
|
||||
unindexed = sorted(f"{safe(dominant(g['narrators']))}/{safe(dominant(g['books']))}"
|
||||
for g in BOOKS.values()
|
||||
if 0 < g["indexed"] < len(g["members"]))
|
||||
if unindexed:
|
||||
fh.write(f"{len(unindexed):>6} books have chapters Plex does not number. Their\n")
|
||||
fh.write(" files keep the names they carry: playback order comes from the\n"
|
||||
" track tag inside each file, and a number written here would be\n"
|
||||
" invented for the chapters with no index. Fix it in the tags, not\n"
|
||||
" in the names — Plex re-reads tags on a scan and never names files.\n\n")
|
||||
fh.write(" Playlists move with their book and their contents are never\n"
|
||||
" rewritten: the paths inside an .m3u go stale.\n")
|
||||
print(f"exceptions -> {ARGS.exceptions}")
|
||||
|
||||
|
||||
def main():
|
||||
global ARGS
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--root", default=os.environ.get("MEDIA_AUDIOBOOKS_ROOT",
|
||||
"/mnt/entropy/Audiobooks"))
|
||||
ap.add_argument("--enum", default="/tmp/audiobooks-enum.tsv",
|
||||
help="the enumerate TSV (bytes, ext, path)")
|
||||
ap.add_argument("--rows", default="/tmp/plex-audiobooks.tsv",
|
||||
help="the chapter rows scripts/media-census.py audiobooks wrote")
|
||||
ap.add_argument("--out", default="/tmp/audiobooks-plan.tsv")
|
||||
ap.add_argument("--exceptions", default="/tmp/audiobooks-exceptions.txt")
|
||||
ap.add_argument("--park", default="_unsorted",
|
||||
help="the folder inside the root that holds what cannot be filed")
|
||||
ARGS = ap.parse_args()
|
||||
ARGS.root = os.path.normpath(ARGS.root)
|
||||
ARGS.park_dir = os.path.join(ARGS.root, ARGS.park)
|
||||
read_inputs()
|
||||
# Every path a file occupies today is claimed before a single destination is
|
||||
# handed out: the executor refuses an occupied target, so a plan that aimed at
|
||||
# one would only fail after the renames that were supposed to clear it.
|
||||
CLAIMED.update(p.lower() for p in HELD)
|
||||
group_books()
|
||||
place_books()
|
||||
place_other()
|
||||
write_plan()
|
||||
write_exceptions()
|
||||
|
||||
|
||||
def read_inputs():
|
||||
with open(ARGS.enum) as fh:
|
||||
for line in fh:
|
||||
parts = line.rstrip("\n").split("\t")
|
||||
if len(parts) >= 3:
|
||||
HELD[parts[2]] = (int(parts[0]), parts[1].lower())
|
||||
with open(ARGS.rows, newline="") as fh:
|
||||
for row in csv.DictReader(fh, delimiter="\t"):
|
||||
if row.get("file"):
|
||||
ROWS[row["file"]] = row
|
||||
gone = [p for p in ROWS if p not in HELD]
|
||||
if gone:
|
||||
NOTES["a chapter row whose file the census no longer sees"].extend(gone)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -5,6 +5,7 @@ r"""Read-only census of a media library — the inputs a relayout plan is built
|
||||
media-census.py probe [--enum TSV] [--out TSV] [--jobs N]
|
||||
media-census.py plex [--root DIR] [--db PATH] [--out TSV]
|
||||
media-census.py jellyfin [--root DIR] [--db PATH] [--container PREFIX] [--out TSV]
|
||||
media-census.py audiobooks [--root DIR] [--db PATH] [--out TSV]
|
||||
|
||||
Run it on the machine that mounts the library. Nothing here writes inside the
|
||||
library and nothing is deleted; the only outputs are the TSVs below, and the
|
||||
@@ -26,6 +27,22 @@ scripts/media-plan.py and scripts/media-shows.py read them instead of touching m
|
||||
SERIES' values, `extra` is Jellyfin's own extra type when the row is
|
||||
bonus material rather than an episode
|
||||
|
||||
music file <TAB> artist <TAB> album <TAB> year <TAB> disc <TAB> track <TAB> title
|
||||
<TAB> mb <TAB> performer (see MUSIC_COLS)
|
||||
one row per track Jellyfin has tagged: the tag read is what the row IS,
|
||||
so `artist` is the album artist (the track artist when that is empty) and
|
||||
`album`/`year` are the album item's provider values when the folder is an
|
||||
album item and the tags' otherwise; `mb` is the album's MusicBrainz
|
||||
release group, `performer` the track artist the tags carry
|
||||
|
||||
audiobooks file <TAB> narrator <TAB> book <TAB> year <TAB> track <TAB> chapter
|
||||
(see AB_COLS)
|
||||
one row per chapter Plex matched under the audiobook root: `book` is the
|
||||
album row Plex built from the files' own tags, `narrator` that book's
|
||||
artist row — Plex keeps exactly one artist per book, which is why it, and
|
||||
not an author parsed out of a title, names the shelf. `track` is empty
|
||||
wherever Plex has no chapter number, which is most of this library.
|
||||
|
||||
`probe` is the slow step (every container is opened for its header, ~10 min for
|
||||
4,800 files on a spindle array at 6 workers) so it is `nice`d: a census beside a
|
||||
live service must not starve it. `enumerate` is a stat pass, so re-censusing
|
||||
@@ -74,6 +91,43 @@ JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode",
|
||||
JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, IsVirtualItem
|
||||
from BaseItems where Type = ? and Path like ?""")
|
||||
|
||||
# The music tree is these two entity classes. The track row carries the tags
|
||||
# Jellyfin read out of the file; the album row carries what the provider matched,
|
||||
# which is the better folder name of the two.
|
||||
JF_AUDIO = "MediaBrowser.Controller.Entities.Audio.Audio"
|
||||
JF_ALBUM = "MediaBrowser.Controller.Entities.Audio.MusicAlbum"
|
||||
MUSIC_COLS = ["file", "artist", "album", "year", "disc", "track", "title", "mb",
|
||||
"performer"]
|
||||
MUSIC_SQL = ("""select b.Path, b.AlbumArtists, b.Artists, b.Album, b.ProductionYear,
|
||||
b.ParentIndexNumber, b.IndexNumber, b.Name, b.Id,
|
||||
a.Name, a.ProductionYear, a.Id
|
||||
from BaseItems b
|
||||
left join BaseItems a on a.Id = b.ParentId and a.Type = ?
|
||||
where b.Type = ? and b.Path like ?""")
|
||||
# The album's identity, in preference order. A release group is the album; a bare
|
||||
# release id is one reissue of it.
|
||||
MUSIC_PROVIDERS = ("MusicBrainzReleaseGroup", "MusicBrainzAlbum")
|
||||
|
||||
# Plex's audiobook tree is the same Artist/Album/Track shape as its music tree:
|
||||
# the leaf (10) is a CHAPTER, its parent (9) is the BOOK, and that book's artist
|
||||
# (8) is the NARRATOR. The track's own `index` is exported because the exceptions
|
||||
# report counts how many books have chapters Plex does not number — the reason the
|
||||
# relayout refuses to invent numbers into file names.
|
||||
PLEX_ARTIST, PLEX_ALBUM, PLEX_TRACK = 8, 9, 10
|
||||
AB_COLS = ["file", "narrator", "book", "year", "track", "chapter"]
|
||||
AB_SQL = f"""
|
||||
select p.file, coalesce(ar.title,''), coalesce(al.title,''), coalesce(al.year,0),
|
||||
coalesce(mi."index",-1), coalesce(mi.title,'')
|
||||
from media_parts p
|
||||
join media_items m on m.id = p.media_item_id
|
||||
join metadata_items mi on mi.id = m.metadata_item_id
|
||||
join metadata_items al on al.id = mi.parent_id and al.metadata_type = {PLEX_ALBUM}
|
||||
left join metadata_items ar on ar.id = al.parent_id
|
||||
and ar.metadata_type = {PLEX_ARTIST}
|
||||
where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
||||
and al.deleted_at is null
|
||||
"""
|
||||
|
||||
|
||||
def enumerate_lib(root, out):
|
||||
"""One row per real file under root: size, extension, path."""
|
||||
@@ -263,6 +317,124 @@ def jellyfin(root, db, container, out):
|
||||
print(f"episodes={written} virtual-or-blank={skipped} -> {out}")
|
||||
|
||||
|
||||
def music(root, db, container, out):
|
||||
"""The tracks Jellyfin has already tagged, for the library at root.
|
||||
|
||||
Jellyfin reads music metadata out of the file itself, so an Audio row IS the
|
||||
tag read, done once, by the vendor's own parser: its album-artist, album,
|
||||
year, disc and track columns are what the embedded tags say and its Name is
|
||||
the track title. That makes this export the whole authority for a music
|
||||
relayout — the path a ripper chose is not evidence of anything, and the
|
||||
folder Jellyfin put a track under is often a release pack, not an album.
|
||||
|
||||
Two gaps this export answers from the row rather than the tree. Many tracks
|
||||
sit under a folder that is not an album item at all (a disc folder, or a
|
||||
mixed one), so the album name and year fall back to the tags when the album
|
||||
row is absent. And an album's MusicBrainz ids are not in the row's own Data
|
||||
— the current schema leaves that empty for music — they are rows of
|
||||
BaseItemProviders keyed by the item's Id.
|
||||
"""
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||
except sqlite3.Error as e:
|
||||
sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n"
|
||||
"It is a WAL database: copy it with its -wal and -shm beside it and"
|
||||
" point --db at the copy. Never sudo it — the data dir belongs to"
|
||||
" the container's owner, and rootless storage is per-user.")
|
||||
have = {r[1] for r in con.execute("pragma table_info(BaseItems)")}
|
||||
need = {"Id", "ParentId", "Type", "Path", "Name", "IndexNumber",
|
||||
"ParentIndexNumber", "ProductionYear", "Album", "AlbumArtists",
|
||||
"Artists"}
|
||||
if not need <= have:
|
||||
sys.exit(f"{db} is not a Jellyfin library database with music columns"
|
||||
f" (BaseItems has no {','.join(sorted(need - have))})")
|
||||
# Provider ids, best identity first: the release group is the album, a bare
|
||||
# release id is one reissue of it and never overrides a group.
|
||||
mb = {}
|
||||
if "BaseItemProviders" in {r[0] for r in con.execute(
|
||||
"select name from sqlite_master where type='table'")}:
|
||||
rank = {p: i for i, p in enumerate(MUSIC_PROVIDERS)}
|
||||
best = {}
|
||||
for iid, pid, val in con.execute(
|
||||
"select ItemId, ProviderId, ProviderValue from BaseItemProviders"
|
||||
" where ProviderId in (?, ?)", MUSIC_PROVIDERS):
|
||||
if iid not in best or rank[pid] < best[iid][0]:
|
||||
best[iid] = (rank[pid], val)
|
||||
mb = {k: v[1] for k, v in best.items()}
|
||||
|
||||
# The fleet's quadlets bind the share at /data, so the container's view of
|
||||
# this library is /data/<the library's own folder name> unless told otherwise.
|
||||
if not container:
|
||||
container = "/data/" + os.path.basename(os.path.normpath(root))
|
||||
prefix = container.rstrip("/") + "/"
|
||||
written = skipped = 0
|
||||
with open(out, "w", newline="") as fh:
|
||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||
w.writerow(MUSIC_COLS)
|
||||
for (path, aal, arts, tag_album, tag_year, disc, track, name, iid,
|
||||
alb_name, alb_year, alb_id) in con.execute(
|
||||
MUSIC_SQL, (JF_ALBUM, JF_AUDIO, prefix + "%")):
|
||||
if not path or not path.startswith(prefix):
|
||||
skipped += 1 # a row for a file outside this library
|
||||
continue
|
||||
# The album artist owns the folder; the track artist is the fallback
|
||||
# for the compilations whose album-artist tag was left empty.
|
||||
w.writerow([root + "/" + path[len(prefix):],
|
||||
((aal or "").strip() or (arts or "").strip()),
|
||||
(alb_name or tag_album or "").replace("\t", " "),
|
||||
alb_year or tag_year or "",
|
||||
"" if disc is None else disc,
|
||||
"" if track is None else track,
|
||||
(name or "").replace("\t", " "),
|
||||
mb.get(alb_id or "") or mb.get(iid or "") or "",
|
||||
(arts or "").replace("\t", " ")])
|
||||
written += 1
|
||||
print(f"tracks={written} outside-root-or-blank={skipped} -> {out}")
|
||||
|
||||
|
||||
def audiobooks(root, db, out):
|
||||
"""The chapters Plex matched inside the audiobook library, read-only.
|
||||
|
||||
Plex stores HOST paths here (unlike Jellyfin, which stores the container's
|
||||
view), so no prefix mapping is needed: the part's `file` is the path the
|
||||
operator has today. Rows for files outside `root` are skipped, because one
|
||||
Plex library DB carries every library on the box.
|
||||
|
||||
The chapter's `index` is exported as an empty field when Plex has none. That
|
||||
is not a gap to fill in here: three quarters of the books on this box contain
|
||||
at least one un-indexed chapter, so the relayout that reads this file must
|
||||
not renumber anything from it.
|
||||
"""
|
||||
uri = f"file:{db}?mode=ro&immutable=1"
|
||||
try:
|
||||
con = sqlite3.connect(uri, uri=True)
|
||||
except sqlite3.Error as e:
|
||||
sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n"
|
||||
"Plex runs rootless here: run this as the container's owner, never "
|
||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||
tables = {r[0] for r in con.execute(
|
||||
"select name from sqlite_master where type='table'")}
|
||||
missing = {"media_parts", "media_items", "metadata_items"} - tables
|
||||
if missing:
|
||||
sys.exit(f"{db} is not a Plex library database (no "
|
||||
f"{', '.join(sorted(missing))})")
|
||||
written = skipped = 0
|
||||
with open(out, "w", newline="") as fh:
|
||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||
w.writerow(AB_COLS)
|
||||
for file, narrator, book, year, track, chapter in con.execute(
|
||||
AB_SQL, (root.rstrip("/") + "/%",)):
|
||||
if not file or not file.startswith(root.rstrip("/") + "/"):
|
||||
skipped += 1 # a row for a file outside this library
|
||||
continue
|
||||
w.writerow([file.replace("\t", " "), narrator.replace("\t", " "),
|
||||
book.replace("\t", " "), "" if not year else year,
|
||||
"" if track is None or int(track) < 0 else track,
|
||||
chapter.replace("\t", " ")])
|
||||
written += 1
|
||||
print(f"chapters={written} outside-root={skipped} -> {out}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
@@ -289,6 +461,23 @@ def main():
|
||||
help="the in-container prefix that stands for --root"
|
||||
" (default /data/<root's folder name>)")
|
||||
j.add_argument("--out", default="/tmp/jellyfin-tv.tsv")
|
||||
m = sub.add_parser("music",
|
||||
help="export the tracks Jellyfin has tagged for the root")
|
||||
m.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT",
|
||||
"/mnt/entropy/Music"),
|
||||
help="music library root on the host")
|
||||
m.add_argument("--db", default=JF_DB)
|
||||
m.add_argument("--container", default=None,
|
||||
help="the in-container prefix that stands for --root"
|
||||
" (default /data/<root's folder name>)")
|
||||
m.add_argument("--out", default="/tmp/jellyfin-music.tsv")
|
||||
b = sub.add_parser("audiobooks",
|
||||
help="export the chapters Plex matched for the audiobook root")
|
||||
b.add_argument("--root", default=os.environ.get("MEDIA_AUDIOBOOKS_ROOT",
|
||||
"/mnt/entropy/Audiobooks"),
|
||||
help="audiobook library root on the host")
|
||||
b.add_argument("--db", default=PLEX_DB)
|
||||
b.add_argument("--out", default="/tmp/plex-audiobooks.tsv")
|
||||
a = ap.parse_args()
|
||||
if a.cmd == "enumerate":
|
||||
enumerate_lib(a.root, a.out)
|
||||
@@ -296,6 +485,10 @@ def main():
|
||||
probe(a.enum, a.out, a.jobs)
|
||||
elif a.cmd == "plex":
|
||||
plex(a.root, a.db, a.out)
|
||||
elif a.cmd == "music":
|
||||
music(a.root, a.db, a.container, a.out)
|
||||
elif a.cmd == "audiobooks":
|
||||
audiobooks(a.root, a.db, a.out)
|
||||
else:
|
||||
jellyfin(a.root, a.db, a.container, a.out)
|
||||
|
||||
|
||||
Executable
+445
@@ -0,0 +1,445 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""media-music.py — a plan that re-files the music library into Jellyfin's own shape.
|
||||
|
||||
media-music.py --root DIR --enum TSV --rows TSV [--out TSV] [--exceptions FILE]
|
||||
|
||||
Jellyfin's music contract (https://jellyfin.org/docs/general/server/media/music/)
|
||||
is not the movie one. Jellyfin reads the metadata out of the file, so a folder is
|
||||
an ALBUM and nothing else, every disc of an album belongs in that one folder, the
|
||||
tags decide the track order, and a file name only matters when it carries a
|
||||
character known to break playback (`< > : " / \ | ? *`) or when a lyric file has
|
||||
to match the basename of the track it belongs to. The plan is therefore:
|
||||
|
||||
Music/<album artist>/<Album> (<year>)/NN - <Title>.<ext>
|
||||
... the same folder, D.NN - <Title> when the album has discs
|
||||
Music/Various Artists/... when its tracks name
|
||||
more than one artist
|
||||
|
||||
The authority is the tag row Jellyfin already read — `scripts/media-census.py
|
||||
music` — never the release-pack folder a ripper made. A file with no row is not
|
||||
guessed at: it parks under `<root>/_unsorted` with the relative path it has today.
|
||||
|
||||
The rules below exist because of what a downloaded discography pack actually
|
||||
contains:
|
||||
|
||||
* A cuesheet beside exactly one audio file means the album is one image with a
|
||||
track list inside it. That file keeps its own name: numbering it `01` would
|
||||
claim it is one track, and Jellyfin shows one track either way.
|
||||
* Sidecars — images, cuesheets, EAC logs, checksums, booklets, playlists —
|
||||
follow the album the audio in their own folder belongs to. A folder holding
|
||||
two albums gives a sidecar no single owner, so it parks instead of being
|
||||
guessed into one.
|
||||
* A video container (.mkv, .vob, .ifo, …) inside a music library is not music.
|
||||
Renaming it to .mka/.m4a would tell Jellyfin to index video as audio, so it
|
||||
parks for a human decision.
|
||||
* Playlists move with their album and their CONTENTS are never rewritten: the
|
||||
paths inside an .m3u go stale. Editing a file the operator owns to keep a
|
||||
convenience file tidy is not this tool's job.
|
||||
|
||||
Nothing is deleted and nothing is overwritten. The output is a list of rename(2)
|
||||
intents for `scripts/media-apply.py`, whose preflight refuses any target that is
|
||||
already claimed and whose journal is the rollback.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
# What Jellyfin will index as music. .mp4/.mkv/.webm are deliberately NOT here:
|
||||
# the vendor doc says an audio-only stream in those containers is never read as
|
||||
# music, and this tool renames for a reason it can prove, not to guess at one.
|
||||
AUDIO_EXT = {".flac", ".mp3", ".m4a", ".m4b", ".ogg", ".oga", ".opus", ".wav",
|
||||
".wma", ".aiff", ".aif", ".ape", ".dsf", ".dff", ".wv", ".mka",
|
||||
".mpc", ".aac"}
|
||||
VIDEO_EXT = {".mkv", ".vob", ".ifo", ".bup", ".mpg", ".mpeg", ".avi", ".mov",
|
||||
".ts", ".m2ts", ".wmv", ".webm", ".mp4", ".m4v", ".iso", ".img",
|
||||
".bdmv", ".divx", ".ogm"}
|
||||
# A lyric has to carry the basename of its track, so it is named after the audio
|
||||
# file and not after itself. .txt counts only when its stem matches an audio stem:
|
||||
# a loose .txt in an album folder is a note.
|
||||
LYRIC_EXT = {".lrc", ".elrc", ".txt"}
|
||||
IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tiff"}
|
||||
# The album-artist strings that mean "compilation", so such an album is not filed
|
||||
# under one artist who does not own it.
|
||||
VARIOUS = {"various", "various artists", "unknown", "unknown artist", "unknow"}
|
||||
# The characters Jellyfin documents as breaking playback. They are dropped, not
|
||||
# substituted: a replacement character is a guess at what the title meant.
|
||||
BAD_CHARS = set('<>:"/\\|?*')
|
||||
TRASH_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir"
|
||||
r"|lost\+found)(/|$)")
|
||||
|
||||
ARGS = None
|
||||
HELD = {} # path -> (bytes, ext)
|
||||
ROWS = {} # path -> the tag row
|
||||
PLACED = {} # audio path -> the destination the plan gave it
|
||||
CLAIMED = set() # every destination path the plan has handed out
|
||||
MOVES, KEEPS = [], []
|
||||
PARKED = defaultdict(list) # reason -> [path]
|
||||
NOTES = defaultdict(list) # section -> [line]
|
||||
CUE_DIRS = set() # source dirs holding a .cue
|
||||
AUDIO_BY_DIR = defaultdict(set) # source dir -> audio files a row names
|
||||
CHILD_OF = defaultdict(set) # dir -> the dirs directly below it holding audio
|
||||
FOLDERS_SEEN = {} # (dir, depth) -> album folders placed below it
|
||||
SUBTREE_DEPTH = 2 # how far below a folder its audio may sit
|
||||
ALBUMS = {} # (artist, album, year) -> the album group
|
||||
|
||||
AMBIGUOUS = object()
|
||||
|
||||
|
||||
def dominant(values):
|
||||
"""The spelling the tags mostly agree on: the most frequent value, ties broken
|
||||
by sort so the answer never depends on the order rows arrived in."""
|
||||
counts = Counter(values)
|
||||
return min(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0]
|
||||
|
||||
|
||||
def rel(path):
|
||||
"""The path as seen from the library root."""
|
||||
return os.path.relpath(path, ARGS.root)
|
||||
|
||||
|
||||
def safe(name):
|
||||
"""A name every client can carry: no reserved character, nothing
|
||||
unprintable, no trailing dot or space, and short enough to rename."""
|
||||
out = "".join(c for c in (name or "")
|
||||
if c not in BAD_CHARS and (c.isprintable() or c == " "))
|
||||
out = re.sub(r"\s+", " ", out).strip().rstrip(". ")
|
||||
while out and len(out.encode("utf-8", "ignore")) > 200:
|
||||
out = out[:-4].rstrip()
|
||||
return out
|
||||
|
||||
|
||||
def assign(src, folder, name, ext, kind, note):
|
||||
"""Book one file into `folder` under `name`, renamed if that exact destination
|
||||
is already taken — by another file in the plan, or by a file standing there
|
||||
today. Two files claiming one path is the fault that costs a re-run of a 90k
|
||||
plan, and so is a plan that only works if the renames happen in the right
|
||||
order: the executor refuses any target that is occupied, so an occupied path
|
||||
is never a destination, even when its occupant is scheduled to leave.
|
||||
A file already standing where the contract puts it is a keep, not a move."""
|
||||
stem = safe(name) or "file"
|
||||
dst = os.path.join(folder, stem + ext)
|
||||
if dst == src:
|
||||
CLAIMED.add(dst.lower()) # it is standing on its own target
|
||||
KEEPS.append((src, dst, kind))
|
||||
return dst
|
||||
# A file already in this folder under this name — or under the numbered
|
||||
# variant the collision rule gives it — keeps the name it has. Renumbering it
|
||||
# every run would turn the plan into a shuffle instead of a fixed point.
|
||||
base = os.path.splitext(os.path.basename(src))[0]
|
||||
if os.path.dirname(src) == folder and (base == stem
|
||||
or base.startswith(stem + " (")):
|
||||
CLAIMED.add(src.lower())
|
||||
KEEPS.append((src, src, kind))
|
||||
return src
|
||||
n = 1
|
||||
while dst.lower() in CLAIMED:
|
||||
n += 1
|
||||
dst = os.path.join(folder, f"{stem} ({n}){ext}")
|
||||
if n > 1:
|
||||
NOTES[note].append(rel(dst))
|
||||
CLAIMED.add(dst.lower())
|
||||
MOVES.append((src, dst, kind))
|
||||
return dst
|
||||
|
||||
|
||||
def park(path, reason):
|
||||
"""Hold a file where it is, under the park folder, keeping its relative path."""
|
||||
PARKED[reason].append(path)
|
||||
dst = os.path.join(ARGS.park_dir, rel(path))
|
||||
if dst != path:
|
||||
CLAIMED.add(dst.lower())
|
||||
MOVES.append((path, dst, "park"))
|
||||
|
||||
|
||||
def under_park(path):
|
||||
"""True for a file already parked: that pile is for a human to review, and
|
||||
re-parking it every run would only push it deeper."""
|
||||
return path.startswith(ARGS.park_dir + os.sep)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- albums
|
||||
def group_albums():
|
||||
"""Every tagged track into one group per (album-artist tag, album, year).
|
||||
|
||||
The key is the identity the tags assert, not the folder the file sits in: one
|
||||
album arrives as three discs in three folders, and two copies of an album in
|
||||
one pack are one album to Jellyfin, whose album item is that name, artist and
|
||||
year. The artist in the key is the ALBUM-artist tag and never the performer,
|
||||
because a compilation is one album whose tracks name many performers — keying
|
||||
on the performer would file one album once per artist.
|
||||
"""
|
||||
for path, r in sorted(ROWS.items()):
|
||||
if path not in HELD or under_park(path):
|
||||
continue
|
||||
d = os.path.dirname(path)
|
||||
AUDIO_BY_DIR[d].add(path)
|
||||
CHILD_OF[os.path.dirname(d)].add(d) # so a cover above CD1/CD2 sees both
|
||||
tag = (r["artist"] or "").strip()
|
||||
album = (r["album"] or "").strip()
|
||||
performer = (r["performer"] or "").strip()
|
||||
if not album:
|
||||
park(path, "a track no album names")
|
||||
continue
|
||||
if not tag and not performer:
|
||||
park(path, "a track no artist names")
|
||||
continue
|
||||
key = (tag.casefold(), album.casefold(), (r["year"] or "").strip())
|
||||
g = ALBUMS.setdefault(key, {"tags": [], "albums": [], "year": key[2],
|
||||
"members": [], "performers": set(),
|
||||
"mb": set()})
|
||||
g["members"].append(path)
|
||||
g["tags"].append(tag)
|
||||
g["albums"].append(album)
|
||||
g["performers"].add(performer or tag)
|
||||
if r.get("mb"):
|
||||
g["mb"].add(r["mb"])
|
||||
|
||||
|
||||
def folder_artist(g):
|
||||
"""Whose shelf the album sits on. The album-artist tag owns it when it names
|
||||
one; when the tags name only performers, a single performer names the album
|
||||
and several make it a compilation, which belongs to no one of them. The key is
|
||||
case-folded, so the spelling that reaches the folder is the one the tracks
|
||||
mostly carry — see dominant()."""
|
||||
tags = [t for t in g["tags"] if t]
|
||||
if tags:
|
||||
name = dominant(tags)
|
||||
return "Various Artists" if name.casefold() in VARIOUS else name
|
||||
if len({p.casefold() for p in g["performers"]}) == 1:
|
||||
return dominant(g["performers"])
|
||||
return "Various Artists"
|
||||
|
||||
|
||||
def album_folder(g):
|
||||
name = safe(dominant(g["albums"])) or "Untitled"
|
||||
return os.path.join(ARGS.root, safe(folder_artist(g)) or "Unknown Artist",
|
||||
f"{name} ({g['year']})" if g["year"] else name)
|
||||
|
||||
|
||||
def is_cue_image(path):
|
||||
"""A folder holding this one audio file and a cuesheet is an album image."""
|
||||
d = os.path.dirname(path)
|
||||
return len(AUDIO_BY_DIR.get(d, ())) == 1 and d in CUE_DIRS
|
||||
|
||||
|
||||
def number_of(value, width):
|
||||
return f"{int(value):0{width}d}" if str(value or "").isdigit() else ""
|
||||
|
||||
|
||||
def place_albums():
|
||||
for key in sorted(ALBUMS):
|
||||
g = ALBUMS[key]
|
||||
folder = album_folder(g)
|
||||
discs = {g0["disc"] for g0 in (ROWS[p] for p in g["members"]) if g0["disc"]}
|
||||
tracks = [int(ROWS[p]["track"]) for p in g["members"]
|
||||
if str(ROWS[p]["track"] or "").isdigit()]
|
||||
# More than one disc, or a disc numbered past the first: the track number
|
||||
# alone cannot name a file uniquely, so the disc goes in front of it.
|
||||
multi = len(discs) > 1 or any(str(d).isdigit() and int(d) > 1 for d in discs)
|
||||
width = 3 if multi or (max(tracks, default=0) > 99) else 2
|
||||
for path in sorted(g["members"]):
|
||||
r = ROWS[path]
|
||||
ext = HELD[path][1]
|
||||
base = os.path.splitext(os.path.basename(path))[0]
|
||||
if is_cue_image(path):
|
||||
name = safe(base) or "image"
|
||||
else:
|
||||
title = safe(r["title"] or base) or "untitled"
|
||||
num = number_of(r["track"], 2 if multi else width)
|
||||
if multi and num:
|
||||
num = f"{r['disc'] or '1'}.{num}"
|
||||
name = f"{num} - {title}" if num else title
|
||||
PLACED[path] = assign(path, folder, name, ext, "track",
|
||||
"two tracks claimed one name inside one album")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- sidecars
|
||||
def folders_below(d, depth):
|
||||
"""The album folders the audio in `d`, or within `depth` directories below it,
|
||||
was placed in. Memoised, because every loose file in a big discography folder
|
||||
asks the same question about it."""
|
||||
if (d, depth) not in FOLDERS_SEEN:
|
||||
seen = {os.path.dirname(PLACED[a]) for a in AUDIO_BY_DIR.get(d, ())
|
||||
if a in PLACED}
|
||||
if depth:
|
||||
for child in CHILD_OF.get(d, ()):
|
||||
seen |= folders_below(child, depth - 1)
|
||||
FOLDERS_SEEN[(d, depth)] = seen
|
||||
return FOLDERS_SEEN[(d, depth)]
|
||||
|
||||
|
||||
def owner_folder(d):
|
||||
"""The album folder that owns this directory — the first place up the tree
|
||||
where audio sits, in the directory itself or within SUBTREE_DEPTH levels below
|
||||
it, which is how a box with its discs in CD1/CD2 owns the cover lying above
|
||||
them. None when no audio is near enough to say, AMBIGUOUS when the audio there
|
||||
belongs to more than one album."""
|
||||
while d.startswith(ARGS.root + os.sep) and d != ARGS.root:
|
||||
folders = folders_below(d, SUBTREE_DEPTH)
|
||||
if folders:
|
||||
return next(iter(folders)) if len(folders) == 1 else AMBIGUOUS
|
||||
d = os.path.dirname(d)
|
||||
return None
|
||||
|
||||
|
||||
def place_sidecars():
|
||||
"""Everything that is not a tagged track: lyrics, images, cuesheets, logs,
|
||||
booklets, playlists — and the clutter that is none of those."""
|
||||
# The lyric rule needs the audio's stem as it is named TODAY, before the plan
|
||||
# renames it, because that is what the sidecar's own name was copied from.
|
||||
stem = {(d, os.path.splitext(os.path.basename(a))[0]): a
|
||||
for d, audios in AUDIO_BY_DIR.items() for a in audios}
|
||||
for path, (_size, ext) in sorted(HELD.items()):
|
||||
if path in ROWS or under_park(path) or TRASH_DIR.search(path):
|
||||
continue
|
||||
d, base = os.path.dirname(path), os.path.splitext(os.path.basename(path))[0]
|
||||
if ext in VIDEO_EXT:
|
||||
park(path, "a video container inside the music library: Jellyfin will "
|
||||
"not read it as music, and renaming it would claim it is")
|
||||
elif ext in AUDIO_EXT:
|
||||
park(path, "an audio file no provider row names — its tags are not in "
|
||||
"the library Jellyfin has matched")
|
||||
elif ext in (LYRIC_EXT | IMAGE_EXT) and (d, base) in stem:
|
||||
# A lyric, or a picture named after its track: it follows that track
|
||||
# and takes the name the track now has, or Jellyfin cannot find it.
|
||||
audio = stem[(d, base)]
|
||||
dst = PLACED.get(audio)
|
||||
if dst:
|
||||
folder, name = os.path.dirname(dst), os.path.splitext(
|
||||
os.path.basename(dst))[0]
|
||||
assign(path, folder, name, ext, "lyric",
|
||||
"a sidecar whose name collided with another in its album")
|
||||
else:
|
||||
park(path, "a sidecar of a track the plan could not place")
|
||||
else:
|
||||
owner = owner_folder(d)
|
||||
if owner is AMBIGUOUS:
|
||||
park(path, "a sidecar whose folder holds more than one album, so "
|
||||
"nothing says which one it belongs to")
|
||||
elif owner:
|
||||
assign(path, owner, base, ext,
|
||||
"playlist" if ext in (".m3u", ".m3u8") else "sidecar",
|
||||
"one album folder would have carried two files of the same "
|
||||
"sidecar name")
|
||||
else:
|
||||
park(path, "not music, or music with nothing beside it to belong to")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- outputs
|
||||
def write_plan():
|
||||
"""mkdir rows for every destination folder and its whole chain, then the moves.
|
||||
Parents first: the executor creates directories in the order they are listed."""
|
||||
chain = set()
|
||||
for _s, dst, _k in MOVES:
|
||||
d = os.path.dirname(dst)
|
||||
while d and d != ARGS.root and d != os.path.dirname(d):
|
||||
chain.add(d)
|
||||
d = os.path.dirname(d)
|
||||
dirs = sorted(chain, key=lambda p: (p.count("/"), p))
|
||||
with open(ARGS.out, "w") as fh:
|
||||
fh.write("action\tsrc\tdst\tkind\n")
|
||||
for d in dirs:
|
||||
fh.write(f"mkdir\t\t{d}\tdir\n")
|
||||
for src, dst, kind in sorted(MOVES, key=lambda m: (m[1], m[0])):
|
||||
fh.write(f"move\t{src}\t{dst}\t{kind}\n")
|
||||
for src, dst, kind in sorted(KEEPS, key=lambda m: (m[1], m[0])):
|
||||
fh.write(f"keep\t{src}\t{dst}\t{kind}\n")
|
||||
parked = [m for m in MOVES if m[2] == "park"]
|
||||
sized = sum(HELD[s][0] for s, _d, _k in MOVES if s in HELD)
|
||||
print(f"plan: {len(MOVES) - len(parked)} moves, {len(parked)} parked"
|
||||
f" ({sized / 2**40:.2f} TiB touched), {len(KEEPS)} already placed,"
|
||||
f" {len(dirs)} folders -> {ARGS.out}")
|
||||
|
||||
|
||||
def write_exceptions():
|
||||
with open(ARGS.exceptions, "w") as fh:
|
||||
fh.write(f"Music relayout exceptions for {ARGS.root}\n\n")
|
||||
fh.write(f"Everything below is parked under {ARGS.park_dir}, keeping the\n"
|
||||
"relative path it has today. Nothing is deleted and nothing is\n"
|
||||
"overwritten: a parked file is one rename from wherever the plan is\n"
|
||||
"pointed next.\n\n")
|
||||
for reason in sorted(PARKED, key=lambda r: -len(PARKED[r])):
|
||||
rs = PARKED[reason]
|
||||
fh.write(f"{len(rs):>6} files {reason}\n")
|
||||
for p in sorted(rs, key=rel)[:400]:
|
||||
fh.write(f" {rel(p)}\n")
|
||||
if len(rs) > 400:
|
||||
fh.write(f" … and {len(rs) - 400} more\n")
|
||||
fh.write("\n")
|
||||
for section in sorted(NOTES, key=lambda s: -len(NOTES[s])):
|
||||
lines = sorted(set(NOTES[section]))
|
||||
fh.write(f"{len(lines):>6} {section}\n")
|
||||
for line in lines[:200]:
|
||||
fh.write(f" {line}\n")
|
||||
if len(lines) > 200:
|
||||
fh.write(f" … and {len(lines) - 200} more\n")
|
||||
fh.write("\n")
|
||||
no_year = sorted(f"{folder_artist(g)}/{dominant(g['albums'])}"
|
||||
for g in ALBUMS.values() if not g["year"])
|
||||
if no_year:
|
||||
fh.write(f"{len(no_year):>6} albums carry no year in any tag, so their folders\n"
|
||||
" are named without one; Jellyfin still matches them.\n")
|
||||
fh.write(" " + ", ".join(no_year[:300]) + "\n\n")
|
||||
clash = sorted(f"{folder_artist(g)}/{dominant(g['albums'])}"
|
||||
for g in ALBUMS.values() if len(g["mb"]) > 1)
|
||||
if clash:
|
||||
fh.write(f"{len(clash):>6} albums put one folder name on more than one\n"
|
||||
" MusicBrainz release group — different releases, one name.\n"
|
||||
" They share the folder Jellyfin's own album item names.\n")
|
||||
fh.write(" " + ", ".join(clash[:200]) + "\n\n")
|
||||
fh.write(" Playlists move with their album and their contents are never\n"
|
||||
" rewritten: the paths inside an .m3u go stale.\n")
|
||||
print(f"exceptions -> {ARGS.exceptions}")
|
||||
|
||||
|
||||
def main():
|
||||
global ARGS
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT",
|
||||
"/mnt/entropy/Music"))
|
||||
ap.add_argument("--enum", default="/tmp/music-enum.tsv",
|
||||
help="the enumerate TSV (bytes, ext, path)")
|
||||
ap.add_argument("--rows", default="/tmp/jellyfin-music.tsv",
|
||||
help="the tag rows scripts/media-census.py music wrote")
|
||||
ap.add_argument("--out", default="/tmp/music-plan.tsv")
|
||||
ap.add_argument("--exceptions", default="/tmp/music-exceptions.txt")
|
||||
ap.add_argument("--park", default="_unsorted",
|
||||
help="the folder inside the root that holds what cannot be filed")
|
||||
ARGS = ap.parse_args()
|
||||
ARGS.root = os.path.normpath(ARGS.root)
|
||||
ARGS.park_dir = os.path.join(ARGS.root, ARGS.park)
|
||||
read_inputs()
|
||||
# Every path a file occupies today is claimed before a single destination is
|
||||
# handed out: the executor refuses an occupied target, so a plan that aimed at
|
||||
# one would only fail after the renames that were supposed to clear it.
|
||||
CLAIMED.update(p.lower() for p in HELD)
|
||||
for path, (_size, ext) in HELD.items():
|
||||
if ext == ".cue":
|
||||
CUE_DIRS.add(os.path.dirname(path))
|
||||
group_albums()
|
||||
place_albums()
|
||||
place_sidecars()
|
||||
write_plan()
|
||||
write_exceptions()
|
||||
|
||||
|
||||
def read_inputs():
|
||||
with open(ARGS.enum) as fh:
|
||||
for line in fh:
|
||||
parts = line.rstrip("\n").split("\t")
|
||||
if len(parts) >= 3:
|
||||
HELD[parts[2]] = (int(parts[0]), parts[1].lower())
|
||||
with open(ARGS.rows, newline="") as fh:
|
||||
for row in csv.DictReader(fh, delimiter="\t"):
|
||||
if row.get("file"):
|
||||
ROWS[row["file"]] = row
|
||||
gone = [p for p in ROWS if p not in HELD]
|
||||
if gone:
|
||||
NOTES["a tag row whose file the census no longer sees"].extend(gone)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+186
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove a relayout landed, using the media server's own tables.
|
||||
|
||||
A scan answers `204` whether or not it found anything, and a rename that the
|
||||
scanner has not followed looks exactly like a rename that never happened from
|
||||
the outside. The only honest proof is the run's journal replayed against the
|
||||
database the server actually serves from: every `move` row must have a row at
|
||||
its destination, and NO row may still name its source — a source row is a scan
|
||||
that has not caught up, and it is the one signature of a library pointing at
|
||||
files that no longer exist.
|
||||
|
||||
Two doors, both read-only, both without a lock while the server runs
|
||||
(`mode=ro&immutable=1`; there is no `sqlite3` binary on these hosts):
|
||||
|
||||
jellyfin `BaseItems.Path`. The container binds the share at `/data`, so the
|
||||
database holds `/data/<library>/…` where the journal holds
|
||||
`/mnt/entropy/<library>/…`. Translate, or every row reads as missing.
|
||||
plex `media_parts.file`, joined up to `metadata_items`. Plex is bound at
|
||||
the host path, so its rows match the journal verbatim. Filter
|
||||
`deleted_at is null` on BOTH tables — a deleted scan leaves its parts
|
||||
behind, and one of those rows is not proof anything landed.
|
||||
|
||||
Neither server holds a row for everything the journal renames: a `.log`, `.cue`
|
||||
or cover image has no row to land in, so its absence is not a fault. The report
|
||||
therefore breaks the missing rows down by extension, and `--ignore-ext` names
|
||||
the extensions this server does not index, which are then counted separately and
|
||||
kept out of the verdict.
|
||||
|
||||
Exit codes: 0 every move landed; 1 something is stale or missing (the details are
|
||||
printed); 2 the journal or the database is not usable at all.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db"
|
||||
PLEX_DB = ("/var/lib/plexmediaserver/Library/Application Support/Plex Media Server"
|
||||
"/Plug-in Support/Databases/com.plexapp.plugins.library.db")
|
||||
|
||||
# Per kind: the table whose columns are required, those columns, and the query
|
||||
# that returns every path the server believes it has.
|
||||
KINDS = {
|
||||
"jellyfin": {
|
||||
"table": "BaseItems",
|
||||
"columns": {"Path"},
|
||||
"sql": "select Path from BaseItems",
|
||||
"default_db": JF_DB,
|
||||
},
|
||||
"plex": {
|
||||
"table": "media_parts",
|
||||
"columns": {"file", "media_item_id", "deleted_at"},
|
||||
"sql": ("select p.file from media_parts p"
|
||||
" join media_items m on m.id = p.media_item_id"
|
||||
" join metadata_items mi on mi.id = m.metadata_item_id"
|
||||
" where p.deleted_at is null and mi.deleted_at is null"),
|
||||
"default_db": PLEX_DB,
|
||||
},
|
||||
}
|
||||
|
||||
def die(msg):
|
||||
print(f"media-verify: {msg}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def read_moves(path):
|
||||
"""The (src, dst) pairs of a journal, in the order the run performed them."""
|
||||
moves = []
|
||||
try:
|
||||
fh = open(path, encoding="utf-8")
|
||||
except OSError as e:
|
||||
die(f"cannot read the journal {path}: {e.strerror}")
|
||||
with fh:
|
||||
for n, line in enumerate(fh, 1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
die(f"{path} line {n} is not a journal record")
|
||||
if rec.get("op") == "move":
|
||||
if not rec.get("src") or not rec.get("dst"):
|
||||
die(f"{path} line {n} is a move without both src and dst")
|
||||
moves.append((rec["src"], rec["dst"]))
|
||||
return moves
|
||||
|
||||
|
||||
def server_paths(db, kind):
|
||||
"""Every path the server holds, or a refusal that names what is missing."""
|
||||
spec = KINDS[kind]
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db}?mode=ro&immutable=1", uri=True)
|
||||
except sqlite3.Error as e:
|
||||
die(f"cannot open {db}: {e}")
|
||||
with con:
|
||||
have = {r[1] for r in con.execute(f"pragma table_info({spec['table']})")}
|
||||
if not spec["columns"] <= have:
|
||||
die(f"{db} is not a {kind} library database ({spec['table']} has no "
|
||||
f"{','.join(sorted(spec['columns'] - have))})")
|
||||
try:
|
||||
return {r[0] for r in con.execute(spec["sql"]) if r[0]}
|
||||
except sqlite3.Error as e:
|
||||
die(f"{db} refused the {kind} query: {e}")
|
||||
|
||||
|
||||
def to_host(path, container, share):
|
||||
"""The database's path in the journal's terms; identity when it is not a
|
||||
container-side path."""
|
||||
if not container:
|
||||
return path
|
||||
prefix = container.rstrip("/") + "/"
|
||||
if path.startswith(prefix):
|
||||
return share.rstrip("/") + "/" + path[len(prefix):]
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--journal", required=True,
|
||||
help="the run's journal (scripts/media-apply.py writes it)")
|
||||
ap.add_argument("--db-kind", choices=sorted(KINDS), required=True)
|
||||
ap.add_argument("--db", default=None, help="the server's database on this host")
|
||||
ap.add_argument("--share", default="/mnt/entropy",
|
||||
help="the share every journal path lives under")
|
||||
ap.add_argument("--container", default=None,
|
||||
help="what the share is called inside the container; empty"
|
||||
" compares verbatim (default /data for jellyfin, nothing"
|
||||
" for plex)")
|
||||
ap.add_argument("--show", type=int, default=20,
|
||||
help="how many offenders to name before saying 'and N more'")
|
||||
ap.add_argument("--ignore-ext", default="",
|
||||
help="comma-separated extensions this server does not index,"
|
||||
" kept out of the verdict (e.g. log,cue,nfo,srt)")
|
||||
args = ap.parse_args()
|
||||
|
||||
db = args.db or KINDS[args.db_kind]["default_db"]
|
||||
container = ("/data" if args.db_kind == "jellyfin" else "") \
|
||||
if args.container is None else args.container
|
||||
ignore = set()
|
||||
for ext in args.ignore_ext.split(","):
|
||||
ext = ext.strip().lower()
|
||||
if ext:
|
||||
ignore.add(ext if ext.startswith(".") else "." + ext)
|
||||
|
||||
moves = read_moves(args.journal)
|
||||
stored = {to_host(p, container, args.share) for p in server_paths(db, args.db_kind)}
|
||||
|
||||
landed, stale, missing, ignored = 0, [], [], 0
|
||||
for src, dst in moves:
|
||||
if os.path.splitext(dst)[1].lower() in ignore:
|
||||
ignored += 1
|
||||
continue
|
||||
at_dst, at_src = dst in stored, src in stored
|
||||
if at_dst and not at_src:
|
||||
landed += 1
|
||||
continue
|
||||
if at_src:
|
||||
stale.append(src)
|
||||
if not at_dst:
|
||||
missing.append(dst)
|
||||
|
||||
print(f"{args.db_kind}: {len(moves)} moves checked")
|
||||
print(f"landed: {landed}")
|
||||
print(f"still at src: {len(stale)}")
|
||||
print(f"missing from the DB: {len(missing)}")
|
||||
if ignored:
|
||||
print(f"not indexed (extension ignored): {ignored}")
|
||||
if missing:
|
||||
exts = Counter(os.path.splitext(d)[1].lower() or "(no extension)"
|
||||
for d in missing)
|
||||
print("missing by extension: "
|
||||
+ ", ".join(f"{e} {n}" for e, n in sorted(exts.items())))
|
||||
for label, items in (("stale", stale), ("missing", missing)):
|
||||
for path in items[:args.show]:
|
||||
print(f" {label}: {path}")
|
||||
if len(items) > args.show:
|
||||
print(f" {label}: … and {len(items) - args.show} more")
|
||||
return 1 if (stale or missing) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -18,7 +18,7 @@ clients.
|
||||
public vhost `jelly.franksdank.cloud` behind the [[Anubis]] guard chain →
|
||||
[[Authentik]] outpost (post-auth route proxies `localhost:8096`).
|
||||
|
||||
## Facts (verified from the live unit, 2026-09-03)
|
||||
## Facts (the repo quadlet; the live unit is still the pre-view generation — see Data layout)
|
||||
|
||||
| Item | Value |
|
||||
|------|-------|
|
||||
@@ -26,7 +26,7 @@ public vhost `jelly.franksdank.cloud` behind the [[Anubis]] guard chain →
|
||||
| Image | `ghcr.io/jellyfin/jellyfin:latest` |
|
||||
| Network | `dankserver_appnet`; publishes 8096/tcp, 8920/tcp, 7359/udp, 1900/udp |
|
||||
| Data roots | `/home/dank/Containers/jellyfin/config:/config`, `/home/dank/Containers/jellyfin/cache:/cache` |
|
||||
| Media | `Volume=/var/lib/media-meta/view:/data` — the [[Media Metadata View]] overlay, **writable**; sidecars land on the NVMe, the [[Entropy Volume]] array underneath stays read-only and untouched |
|
||||
| Media | `Volume=/var/lib/media-meta/view:/data` in git — the [[Media Metadata View]] overlay, **writable**, sidecars land on the NVMe and the [[Entropy Volume]] array underneath stays untouched. **The live unit is still pre-view: `/mnt/entropy:/data:ro`** — nothing is written into the library until the view is deployed |
|
||||
| GPU | none — no `AddDevice`; CPU decode only (jellyfin-ffmpeg path) |
|
||||
| Env quirks | `DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true`, `LD_PRELOAD=libjemalloc.so.2`, `MALLOC_TRIM_THRESHOLD_=131072` kept verbatim from the compose-era container (zero-drift cutover) |
|
||||
| Health | `curl -f http://localhost:8096/health` every 30 s |
|
||||
@@ -36,23 +36,27 @@ public vhost `jelly.franksdank.cloud` behind the [[Anubis]] guard chain →
|
||||
```
|
||||
/home/dank/Containers/jellyfin/config -> /config (JELLYFIN_DATA_DIR)
|
||||
/home/dank/Containers/jellyfin/cache -> /cache (JELLYFIN_CACHE_DIR, XDG_CACHE_HOME)
|
||||
/var/lib/media-meta/view -> /data (the SSD-backed overlay view)
|
||||
/var/lib/media-meta/view -> /data (git: the overlay view)
|
||||
/mnt/entropy -> /data (live today: the array, read-only)
|
||||
```
|
||||
|
||||
Library folders point at `/data` paths, which are the view's paths: stable
|
||||
across container recreation, and identical to the raw array layout below it.
|
||||
Library folders point at `/data` paths: stable across container recreation and
|
||||
identical to the raw array layout below. Which side of that path the container
|
||||
is on is the open question on this host — the installed unit binds the read-only
|
||||
array, and `scripts/media-view.sh status` reports `NOT MOUNTED` (2026-10-31).
|
||||
|
||||
`Local metadata` — save metadata to media folders — is enabled, and it works
|
||||
because `/data` is the [[Media Metadata View]] rather than the read-only array:
|
||||
`movie.nfo`, `tvshow.nfo`, `episode.nfo` and artwork are written once, on the
|
||||
NVMe upper layer, and [[Plex]] reads the same files. Jellyfin is the fleet's
|
||||
only metadata writer.
|
||||
`Local metadata` — save metadata to media folders — is enabled, and it is the
|
||||
reason the view exists: `movie.nfo`, `tvshow.nfo`, `episode.nfo` and artwork
|
||||
written once on the NVMe upper layer are files [[Plex]] can read too. While
|
||||
`/data` is bound read-only every item update logs the error below and no sidecar
|
||||
is written; Jellyfin becomes the fleet's only metadata writer with the deploy
|
||||
that raises the view.
|
||||
|
||||
**Failure signature:** `System.IO.IOException: Read-only file system :
|
||||
'/data/Movies/<film>/movie.nfo'` means the container is binding the raw array
|
||||
instead of the view — the view is unmounted or the quadlet drifted. That used to
|
||||
be the expected state (pre-view, every item update logged it); today it is a
|
||||
fault. Run `scripts/media-view.sh status`.
|
||||
instead of the view. On Dankserver that is today's expected state — the view has
|
||||
never been installed (`scripts/media-view.sh status`). Once it is deployed, the
|
||||
same line is a fault: the quadlet drifted or the mount died.
|
||||
|
||||
## Ports & access
|
||||
|
||||
|
||||
@@ -20,6 +20,19 @@ The mount point is not a volume and holds nothing while unmounted. It sits insid
|
||||
merged door are one directory — and so the mount table grows no new `/mnt` entry
|
||||
next to the array.
|
||||
|
||||
**Not deployed on Dankserver — verified 2026-10-31.** `scripts/media-view.sh
|
||||
status` answers `view: /var/lib/media-meta/view NOT MOUNTED`,
|
||||
`/var/lib/media-meta` does not exist, and the installed quadlets under
|
||||
`~/.config/containers/systemd/` still carry the pre-view binds — `/mnt/entropy:/data:ro`
|
||||
for [[Jellyfin]], `/mnt/entropy:/mnt/entropy:ro` for [[Plex]]. Everything below
|
||||
is the shape git describes, not the shape the servers run. The cutover is an
|
||||
ordinary deploy: `deploy.sh dankserver --service jellyfin|plex` runs
|
||||
`ensure_media_view` (`install` then `verify`) before it restarts the container,
|
||||
so it must not be fired through the middle of a library scan. Until that deploy
|
||||
lands there is no upper layer, so no relayout can leave ghost metadata and
|
||||
`reap` has nothing to do — `media-apply.py` prints that follow-up only when the
|
||||
view really is a mountpoint.
|
||||
|
||||
## Shape
|
||||
|
||||
| Item | Value |
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
type: reference
|
||||
title: Media Relayout
|
||||
description: "Census, plan and apply the movie and TV libraries into Jellyfin format: gates, evidence chain, refusals, rollback."
|
||||
description: "Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback."
|
||||
tags: [dankserver, media-server, jellyfin, plex, procedure]
|
||||
timestamp: 2026-10-30T00:00:00Z
|
||||
---
|
||||
@@ -12,18 +12,24 @@ The movie library on [[Entropy Volume]] arrived as release folders —
|
||||
`Some.Movie.2019.1080p.WiKiTaLx.x265-FOO/` with the video inside named after
|
||||
the release. [[Jellyfin]] matches a movie by folder and file name, so the
|
||||
release noise cost posters, extras and versions. These scripts turn that
|
||||
library into Jellyfin's own shape — the movies, the TV packs, and the TV that
|
||||
turns up in the movie share — and nothing else in the repo touches a media file.
|
||||
library into the shape its media server reads — the movies, the TV packs, the TV
|
||||
that turns up in the movie share, the music library, and the audiobooks, whose
|
||||
only reader is Plex — and nothing else in the repo touches a media file.
|
||||
|
||||
| Script | Touches media? | Output |
|
||||
|---|---|---|
|
||||
| `scripts/media-census.py` | reads headers only | `enum`, `probed`, `plex` TSVs |
|
||||
| `scripts/media-census.py` | reads headers and the servers' own DBs, never writes | `enum`, `probed`, and one authority TSV per library |
|
||||
| `scripts/media-plan.py` | no — TSVs in, TSVs out | the plan + the exception report |
|
||||
| `scripts/media-apply.py` | **renames, never copies**; sweeps only what a run emptied | the moves, plus a journal |
|
||||
| `scripts/media-tv.py` | no — TSV out | a plan that moves TV into the TV library |
|
||||
| `scripts/media-shows.py` | no — TSV out | a plan that re-files the TV library itself |
|
||||
| `scripts/media-music.py` | no — TSV out | a plan that re-files the music library by its tags |
|
||||
| `scripts/media-audiobooks.py` | no — TSV out | a plan that re-files the audiobooks by the book Plex matched |
|
||||
| `scripts/media-verify.py` | no — reads the servers' DBs read-only | the verdict that every journaled move landed |
|
||||
|
||||
The seams are `tests/media_relayout.bats` and `tests/media_shows.bats`; the target contract is quoted from
|
||||
The seams are `tests/media_relayout.bats`, `tests/media_shows.bats`,
|
||||
`tests/media_music.bats`, `tests/media_audiobooks.bats` and `tests/media_verify.bats`;
|
||||
the movie and TV target contracts are quoted from
|
||||
[jellyfin.org/docs/general/server/media/movies](https://jellyfin.org/docs/general/server/media/movies)
|
||||
in `media-plan.py`'s docstring.
|
||||
|
||||
@@ -94,8 +100,8 @@ podman exec -i plex sh -c 'read -r T
|
||||
|
||||
[[Jellyfin]]'s scan key lives in `~/.config/secrets/jellyfin` (mode 600; the
|
||||
same value is `JELLYFIN_API_KEY` in the ignored `dankserver/.env`). It rides a
|
||||
mode-600 curl config, never a header argument, and it must use the
|
||||
`MediaBrowser` form — Jellyfin 12.4 refuses a bare `X-Emby-Token` for any key:
|
||||
mode-600 curl config, never a header argument, in the `MediaBrowser` form —
|
||||
Jellyfin 12.4 refuses a bare `X-Emby-Token`, and the backslashes below matter:
|
||||
|
||||
```bash
|
||||
umask 077
|
||||
@@ -107,7 +113,30 @@ curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n"
|
||||
shred -u /tmp/jf.hdr
|
||||
```
|
||||
|
||||
`204` is acceptance; the scan then runs on the server's own schedule.
|
||||
`204` is acceptance; anything else queued no scan. A `401` from a key that answers `200` on a hand-typed request is this block's quoting, not the server — bash `printf` collapses `\"` to `"`, curl ends the quoted value at that quote, and Jellyfin receives `Authorization: MediaBrowser Token=` with nothing after it (`\\"` answers 204, `\"` answers 401, both run on this host). One accepted refresh is the whole procedure: a second one while a scan runs moves `Scan Media Library` to `Cancelling`, and once the runner holds `Cancelling` every later `204` is accepted and then dropped — the fix is `systemctl --user restart jellyfin.service` (never `deploy.sh`: that runs the metadata-view cutover), one POST after health answers, and then believe `ScheduledTasks` State `Running`, not the status code.
|
||||
|
||||
### Proving the move landed
|
||||
|
||||
A scan answers `204` whether or not it finds anything, so proof is the journal
|
||||
replayed against both servers' own tables: every `move` row has a row at its
|
||||
`dst`, and **zero** rows still name its `src`. `scripts/media-verify.py` is that
|
||||
replay: it opens each database read-only (there is no `sqlite3` binary on these
|
||||
hosts or inside either container) and exits non-zero while a row is stale.
|
||||
|
||||
```bash
|
||||
scripts/media-verify.py --journal /mnt/entropy/media-relayout-music.jsonl \
|
||||
--db-kind jellyfin --ignore-ext jpg,jpeg,png,log,cue,toc,nfo,txt,m3u,accurip
|
||||
```
|
||||
|
||||
Ignore only what a server cannot index. A missing `.flac` or `.mkv` is the
|
||||
fault; a row still naming its `src` means the scan has not followed the rename
|
||||
yet, so re-run the check after the scan, never before.
|
||||
|
||||
Three doors that look right and are not: `GET /search?query=` matches metadata
|
||||
titles, never file paths; `?path=` on `/library/sections/N/all` makes Plex's
|
||||
HTTPD close without a single byte; and the same HTTPD drops requests that carry
|
||||
no `Accept` and client-identifier header (see the scan block above for the form
|
||||
that works).
|
||||
|
||||
## TV in the movie library
|
||||
|
||||
@@ -175,12 +204,12 @@ python3 scripts/media-census.py probe --root /mnt/entropy/TVShows \
|
||||
# 3. The plan. --tmdb needs TMDB_API_KEY; without it a series no provider dated
|
||||
# is simply named without a year, which Jellyfin accepts.
|
||||
python3 scripts/media-shows.py --root /mnt/entropy/TVShows --enum /tmp/tv_enum.tsv \
|
||||
--titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv \
|
||||
--out /tmp/tv_plan.tsv --exceptions /tmp/tv_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Then the same
|
||||
# executor, journal and rollback as the movie relayout above.
|
||||
scripts/media-apply.py apply --plan /tmp/tv_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-tv.jsonl --apply
|
||||
--titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv --out /mnt/entropy/media-relayout-shows-plan.tsv \
|
||||
--exceptions /mnt/entropy/media-relayout-shows-exceptions.txt
|
||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Same executor
|
||||
# as above, but on this arm's OWN journal: a shared journal cannot be unwound.
|
||||
scripts/media-apply.py apply --plan /mnt/entropy/media-relayout-shows-plan.tsv \
|
||||
--root /mnt/entropy --journal /mnt/entropy/media-relayout-shows.jsonl --apply
|
||||
```
|
||||
|
||||
Evidence, strongest first — the order the planner believes things:
|
||||
@@ -214,6 +243,120 @@ filename. Renaming a show is Jellyfin's work — a folder rename never touches i
|
||||
provider ids — so the mover keeps the name it is handed and strips only the pack
|
||||
index a release leads with (`4a. Seed DESTINY` → `Seed DESTINY`).
|
||||
|
||||
## The music library's own relayout
|
||||
|
||||
`/mnt/entropy/Music` is rips and discography packs, and Jellyfin has matched and
|
||||
tagged almost all of it. `scripts/media-music.py` re-files it into the shape
|
||||
Jellyfin's music scanner reads —
|
||||
|
||||
Music/<album artist>/<Album> (<year>)/NN - <Title>.ext
|
||||
|
||||
— and the authority is the tag row of the track, never the name the ripper gave
|
||||
the file.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Jellyfin's audio rows, one per track, with the album and
|
||||
# artist ids its own database carries (the DB opens read-only).
|
||||
python3 scripts/media-census.py music --root /mnt/entropy/Music \
|
||||
--out /tmp/music_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Music \
|
||||
--out /tmp/music_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-music.py --root /mnt/entropy/Music --enum /tmp/music_enum.tsv \
|
||||
--rows /tmp/music_rows.tsv --out /tmp/music_plan.tsv \
|
||||
--exceptions /tmp/music_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Then the same
|
||||
# executor, journal and rollback as above, on its own journal.
|
||||
scripts/media-apply.py apply --plan /tmp/music_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-music.jsonl --apply
|
||||
```
|
||||
|
||||
The group is what the tags assert, not where the files sit: **(album-artist tag,
|
||||
album, year)**. One album arrives as three disc folders and as two copies of one
|
||||
rip, and one MusicBrainz release group is one album. A compilation is one album
|
||||
whose tracks name many performers, so it files once under `Various Artists`:
|
||||
keying on the performer would file one album once per artist. Two different
|
||||
album-artist tags on one album name are two albums, which is what Jellyfin's two
|
||||
album items say.
|
||||
|
||||
What it refuses:
|
||||
|
||||
+- **A sidecar is placed by the audio around it, never by its own name.** A
|
||||
`.lrc`, `.txt` or picture that shares a track's stem follows that track and
|
||||
takes the name the track now has, or Jellyfin cannot find it; anything else
|
||||
goes to the album its own folder was placed in, or to the folder whose discs
|
||||
sit one level below it. A folder whose audio belongs to two albums parks its
|
||||
sidecar rather than guessing.
|
||||
+- **An image album keeps its name.** A folder holding one audio file and a
|
||||
`.cue` is a whole album in one file; numbering it `01 - …` tells Jellyfin to
|
||||
index it as one track.
|
||||
+- **A video container parks**, because Jellyfin will not index it as music and a
|
||||
rename to `.m4a` would claim it is; **audio no provider row names parks** too,
|
||||
because unmatched audio is a tagging job, not a renaming job.
|
||||
|
||||
The plan applies on 2026-10-31 through `/mnt/entropy/media-relayout-music.jsonl`;
|
||||
its rollback is that journal run through `media-apply.py rollback --apply`. ONE
|
||||
writer at a time on this array: two rename passes race, and neither journal tells.
|
||||
|
||||
## The audiobook library's own relayout
|
||||
|
||||
`/mnt/entropy/Audiobooks` is the one library here that [[Jellyfin]] does not read
|
||||
at all — its music scanner has no audiobook concept — while [[Plex]] indexes all
|
||||
of it as Artist → Album → Track: the album is the BOOK, that book's single artist
|
||||
is the NARRATOR, both matched out of the files' own tags. Today one book is split
|
||||
across `… [Part 1]`, `… [Part 2]` folders three or four deep inside a collection
|
||||
pack, so a scan serves three quarters of a novel; `scripts/media-audiobooks.py`
|
||||
re-files the tree so the property both servers' book scanners assume is true —
|
||||
|
||||
Audiobooks/<Narrator>/<Book> (<year>)/<the file keeps its own name>
|
||||
|
||||
```bash
|
||||
# 1. The authority: Plex's chapter rows for this root (the DB opens read-only).
|
||||
python3 scripts/media-census.py audiobooks --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-audiobooks.py --root /mnt/entropy/Audiobooks \
|
||||
--enum /tmp/ab_enum.tsv --rows /tmp/ab_rows.tsv --out /tmp/ab_plan.tsv \
|
||||
--exceptions /tmp/ab_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST. Then the same executor, journal and rollback
|
||||
# as above, on its own journal, and refresh the Plex section as in step 5 of
|
||||
# the movie Procedure — from inside the container, never over the host port.
|
||||
scripts/media-apply.py apply --plan /tmp/ab_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-audiobooks.jsonl --apply
|
||||
```
|
||||
|
||||
The group is the identity Plex asserts in its own database — **(narrator, book,
|
||||
year)** — so one book arriving in four part folders becomes one folder, and a
|
||||
1965 and a 2020 recording that share a title stay two books.
|
||||
|
||||
Two laws are forced by what the rows actually hold:
|
||||
|
||||
+- **A chapter is never renumbered.** Most books contain chapters whose index Plex
|
||||
does not know, so a number written into a name would be invented — and it would
|
||||
buy nothing, because both servers order playback by the embedded track tag, not
|
||||
by the file name. A wrong number in a name is the number a human later trusts.
|
||||
Fix order in the tags: a Plex scan re-reads tags and never renames a file.
|
||||
+- **The shelf is the narrator, not the author.** Every book has exactly one artist
|
||||
row and it is the narrator; the author survives only inside some album titles
|
||||
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving it would
|
||||
mean parsing titles where a ripper wrote one and inventing it everywhere else.
|
||||
The author stays visible in the folder name Plex itself shows.
|
||||
|
||||
What it refuses, the music arm's list with one addition: a video container, audio
|
||||
no chapter row names, and a loose file in a folder holding two books all park under
|
||||
`_unsorted` with the path they have today — and so does a chapter whose row names
|
||||
**no book or no narrator**: a placeholder (`[Unknown Album]`, `[Unknown Artist]`)
|
||||
that unrelated books share, or a chapter number mistagged as the artist (`0.4`).
|
||||
`Various Artists` is not a placeholder — that is a multi-cast read.
|
||||
|
||||
The plan stands reviewed at `/mnt/entropy/media-relayout-audiobooks-plan.tsv`
|
||||
(55,114 moves, 7,764 parked, executor preflight refused nothing); it applies on its
|
||||
own journal, behind the music writer, and then refreshes the Plex section.
|
||||
|
||||
## Laws
|
||||
|
||||
- **A moved file is a 404 in [[Plex]] until its section is rescanned.** The
|
||||
@@ -314,14 +457,29 @@ index a release leads with (`4a. Seed DESTINY` → `Seed DESTINY`).
|
||||
- **Sidecar metadata does not follow a rename, because it lives on another
|
||||
filesystem.** The media servers write `.nfo` and artwork through the
|
||||
[[Media Metadata View]] (NVMe upper layer); this tool moves bytes on the raw
|
||||
array path. After any sweep or TV move, run `scripts/media-view.sh reap`, then
|
||||
the same command with `--apply`: the sidecars left at the old coordinates become
|
||||
ghost folders — artwork and an `.nfo` with no video — that both servers browse
|
||||
and report as content.
|
||||
array path, so where that view is mounted a sweep or a move must be followed by
|
||||
`scripts/media-view.sh reap --apply` — stranded sidecars become ghost folders
|
||||
that both servers browse and report as content. Whether this host mounts the
|
||||
view at all is stated on that page; `media-apply.py` stays silent when it does not.
|
||||
- **Nothing deletes through the metadata view.** A delete there writes an overlayfs
|
||||
whiteout in the upper layer, which hides the array's copy without removing it;
|
||||
the file returns when the view is unmounted. Deletions in this pipeline happen
|
||||
on the raw path only, which is what the journal can undo.
|
||||
- **The music album key is case-folded; the folder name is not.** `Master Of
|
||||
Puppets` and `Master of Puppets` are one album to the matcher, so the spelling
|
||||
that reaches the folder is the one most tracks carry. Taking it from whichever
|
||||
row sorted first made the folder name depend on row order, and re-planning an
|
||||
already-filed library renamed albums that were already in place.
|
||||
- **An occupied path is never a destination**, not even when its occupant is
|
||||
scheduled to leave: the executor refuses an occupied target, so a plan that
|
||||
aims at one fails after the renames that were supposed to clear it. A file
|
||||
standing where the contract puts it is a keep, and a file already carrying a
|
||||
collision suffix keeps that suffix — otherwise the plan is a shuffle, not a
|
||||
fixed point, and a re-run moves albums that are already right.
|
||||
- **`media-census.py music` reads the `BaseItemProviders` table, not the item's
|
||||
JSON.** This Jellyfin build carries an empty `ProviderIds` inside
|
||||
`BaseItems.Data` for albums, so a census that parses the JSON finds no
|
||||
MusicBrainz or Spotify id on any album and reads as "the library is untagged".
|
||||
|
||||
## Rollback
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ public traffic can be fronted by Caddy like every other vhost.
|
||||
| Runtime | podman quadlet, `plex.service` (user manager) — migrated from the native `plexmediaserver.service` OS-package unit (2026-09) |
|
||||
| Image | `lscr.io/linuxserver/plex:1.43.5.11029-6bf8af3f0-ls329` (pinned; verified == the registry's `latest` manifest digest at pin time: `sha256:8f655bad…e9e2334`) |
|
||||
| Data root | `/var/lib/plexmediaserver` bind-mounted **in place** as `/config` (RW) — the migration moved zero bytes |
|
||||
| Media | `/var/lib/media-meta/view` (the [[Media Metadata View]] overlay) bound at the same absolute path `/mnt/entropy` **read-only** — every library path in the PMS database stayed valid, zero re-scan |
|
||||
| Media | git: `/var/lib/media-meta/view` (the [[Media Metadata View]] overlay) bound at the same absolute path `/mnt/entropy` **read-only**; live today: `/mnt/entropy` itself, read-only — the view has never been installed. Either way every library path in the PMS database stays valid, zero re-scan |
|
||||
| Identity | fleet rootless model — container root maps to the `dank` user (1000:1000); the deployer chowns the data root on first cutover |
|
||||
| Port | `32400/tcp` published on `0.0.0.0` — zero client reconfiguration |
|
||||
| Network | `dankserver_appnet`, alias `plex`; Tautulli reaches it at `http://plex:32400` |
|
||||
@@ -40,7 +40,8 @@ public traffic can be fronted by Caddy like every other vhost.
|
||||
The migration moved zero bytes: the container bind-mounts the native data
|
||||
root where it already lived, and the media archive re-mounts at
|
||||
`/mnt/entropy` inside the container — only the HOST side of that bind has moved
|
||||
(array, then the overlay view), so every absolute library path in the
|
||||
(the array today; the overlay view in git has never been deployed), so every
|
||||
absolute library path in the
|
||||
PMS database keeps resolving. The one metadata change: rootless podman
|
||||
cannot map the real host uid 997, so the fleet model applies — container
|
||||
root runs as the `dank` user, and the deployer adopts the data root with
|
||||
|
||||
@@ -52,7 +52,7 @@ is a locator: one row, one clause, no numbers.
|
||||
| [[Jellyfin]] | Jellyfin on Dankserver: quadlet unit, overlay metadata view, provider keys, ports, access, transcode status. |
|
||||
| [[mariadb]] | The MariaDB backend for RomM: quadlet unit, network alias, env-file secret handling, and the backup path. |
|
||||
| [[Media Metadata View]] | The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback. |
|
||||
| [[Media Relayout]] | Census, plan and apply the movie and TV libraries into Jellyfin format: gates, evidence chain, refusals, rollback. |
|
||||
| [[Media Relayout]] | Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback. |
|
||||
| [[Plex]] | Plex Media Server is a **containerized media server on Dankserver**: a podman |
|
||||
| [[redis]] | The Redis cache and task queue for RomM: quadlet unit, network alias topology, and the backup path. |
|
||||
| [[RomM]] | RomM (ROM Manager) is a self-hosted ROM library manager and player. |
|
||||
|
||||
Executable
+438
@@ -0,0 +1,438 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the audiobook-library relayout — scripts/media-census.py audiobooks (the
|
||||
# export of the rows Plex matched) and scripts/media-audiobooks.py (the plan).
|
||||
#
|
||||
# The audiobook tree is the one library on this box that Jellyfin does not read
|
||||
# at all and Plex reads completely: every chapter here is a Plex Track row under
|
||||
# an Album row (the BOOK) under one Artist row (the NARRATOR). That is the
|
||||
# authority this seam files by — never the collection folder a release pack made.
|
||||
#
|
||||
# Two laws here are the opposite of the music tool's, and both come from what the
|
||||
# rows actually contain. Files are NEVER renumbered: 7,556 of 10,485 books carry
|
||||
# at least one chapter whose index Plex does not know, so a `00 -` invented from
|
||||
# a missing index would order a book wrong, and playback order comes from the
|
||||
# embedded track tag either way — the name buys nothing and can only lie. And the
|
||||
# folder artist is the NARRATOR even though a reader thinks in authors: every
|
||||
# book has exactly one artist row, while the author survives only in 10% of the
|
||||
# album titles (`Kryptos by Graham McNeill`) and in no column Plex stores, so one
|
||||
# rule beats a shelf that is half authors and half narrators.
|
||||
#
|
||||
# Every test states which authority a file has and asserts where the PLAN files
|
||||
# it, never how the planner decided.
|
||||
|
||||
load helpers
|
||||
|
||||
PLAN="$REPO_ROOT/scripts/media-audiobooks.py"
|
||||
CENSUS="$REPO_ROOT/scripts/media-census.py"
|
||||
APPLY="$REPO_ROOT/scripts/media-apply.py"
|
||||
|
||||
# at PATH SIZE — a file with deterministic content.
|
||||
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
|
||||
|
||||
# row FILE NARRATOR BOOK YEAR TRACK CHAPTER
|
||||
# One row of the authority export scripts/media-census.py audiobooks writes.
|
||||
row() {
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$1" "$2" "$3" "${4:-}" "${5:-}" "$6" >> "$ROWS"
|
||||
}
|
||||
|
||||
# dst SRC — the destination the plan gives SRC, or nothing when it does not move.
|
||||
dst() {
|
||||
awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $3}' "$OUT"
|
||||
}
|
||||
|
||||
# kind SRC — the kind the plan gave SRC's move row.
|
||||
kind() {
|
||||
awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $4}' "$OUT"
|
||||
}
|
||||
|
||||
setup() {
|
||||
AB="$BATS_TEST_TMPDIR/Audiobooks"
|
||||
PARK="$AB/_unsorted"
|
||||
ENUM="$BATS_TEST_TMPDIR/enum.tsv"
|
||||
ROWS="$BATS_TEST_TMPDIR/rows.tsv"
|
||||
OUT="$BATS_TEST_TMPDIR/plan.tsv"
|
||||
EXC="$BATS_TEST_TMPDIR/exceptions.txt"
|
||||
# The header is part of the authority export, so the fixture carries it.
|
||||
printf 'file\tnarrator\tbook\tyear\ttrack\tchapter\n' > "$ROWS"
|
||||
|
||||
# One book of one author, split across the two part folders a pack shipped it
|
||||
# in, with a cover above part 1 and a playlist beside the chapters.
|
||||
A="$AB/Stephen King/It (1986)"
|
||||
at "$A [Part 1]/01 - Chapter 1.mp3" 9000
|
||||
at "$A [Part 1]/02 - Chapter 2.mp3" 9000
|
||||
at "$A [Part 1]/cover.jpg" 20
|
||||
at "$A [Part 1]/It.m3u" 30
|
||||
at "$A [Part 2]/03 - Chapter 3.mp3" 9000
|
||||
row "$A [Part 1]/01 - Chapter 1.mp3" "George Guidall" "It" 1986 1 "Chapter 1"
|
||||
row "$A [Part 1]/02 - Chapter 2.mp3" "George Guidall" "It" 1986 2 "Chapter 2"
|
||||
row "$A [Part 2]/03 - Chapter 3.mp3" "George Guidall" "It" 1986 3 "Chapter 3"
|
||||
|
||||
# A book whose album row carries no year at all.
|
||||
B="$AB/Collected/Project Gold"
|
||||
at "$B/01.ogg" 9000
|
||||
row "$B/01.ogg" "Operator X" "Project Gold" "" 1 "Opening"
|
||||
|
||||
# One book title, two narrators: two different books.
|
||||
C="$AB/Shared"
|
||||
at "$C/Dune side a.m4b" 9000
|
||||
at "$C/Dune side b.m4b" 9000
|
||||
at "$C/Dune 2020.m4b" 9000
|
||||
row "$C/Dune side a.m4b" "Scott Brick" "Dune" 1965 1 "Prologue"
|
||||
row "$C/Dune side b.m4b" "Scott Brick" "Dune" 1965 2 "Dune"
|
||||
row "$C/Dune 2020.m4b" "Malcolm Hillgartner" "Dune" 2020 1 "Book One"
|
||||
|
||||
# Two part folders of one book whose chapters share a file name.
|
||||
D="$AB/Pulp"
|
||||
at "$D/Book A/vol1.mp3" 9000
|
||||
at "$D/Book A vol2/vol1.mp3" 9000
|
||||
row "$D/Book A/vol1.mp3" "Nora Pace" "Hard Food" 2001 1 "Vol 1"
|
||||
row "$D/Book A vol2/vol1.mp3" "Nora Pace" "Hard Food" 2001 2 "Vol 1"
|
||||
|
||||
# The same narrator spelled two ways across one book's chapters.
|
||||
E="$AB/Spell"
|
||||
at "$E/kharn 1.mp3" 9000
|
||||
at "$E/kharn 2.mp3" 9000
|
||||
at "$E/kharn 3.mp3" 9000
|
||||
row "$E/kharn 1.mp3" "David Timson" "Kharn" 2013 1 "One"
|
||||
row "$E/kharn 2.mp3" "David timson" "Kharn" 2013 2 "Two"
|
||||
row "$E/kharn 3.mp3" "David Timson" "Kharn" 2013 3 "Three"
|
||||
|
||||
# A folder holding two different books: its loose note has no owner.
|
||||
F="$AB/Mixed"
|
||||
at "$F/book one chapter.mp3" 9000
|
||||
at "$F/book two chapter.mp3" 9000
|
||||
at "$F/notes.txt" 10
|
||||
row "$F/book one chapter.mp3" "Ann Reader" "Book One" 2001 1 "One"
|
||||
row "$F/book two chapter.mp3" "Bob Reader" "Book Two" 2002 1 "Two"
|
||||
|
||||
# A file already parked: the pile is for a human, so a row that would file it
|
||||
# must not pull it back into the tree.
|
||||
at "$PARK/Old/nobody.mp3" 9000
|
||||
row "$PARK/Old/nobody.mp3" "Ann Reader" "Book One" 2001 1 "One"
|
||||
|
||||
# The odd cases: no row, a video container, a name carrying a character Plex
|
||||
# says breaks playback, rows with a blank book and a blank narrator, and a
|
||||
# matched chapter whose file has no extension.
|
||||
G="$AB/Odd"
|
||||
at "$G/loose.mp3" 9000
|
||||
at "$G/concert.mp4" 9000
|
||||
at "$G/Bad: Name/01 chapter.mp3" 9000
|
||||
row "$G/Bad: Name/01 chapter.mp3" "Some Reader" "Trail: Marked" 1999 1 "Marked"
|
||||
at "$G/blank book.mp3" 9000
|
||||
row "$G/blank book.mp3" "Some Reader" "" 2000 1 "No Book"
|
||||
at "$G/blank narrator.mp3" 9000
|
||||
row "$G/blank narrator.mp3" "" "No Artist" 2000 1 "No Artist"
|
||||
at "$G/NoExt/part01" 9000
|
||||
row "$G/NoExt/part01" "Archivist" "Tapes" 1971 1 "Tape One"
|
||||
|
||||
# Tag-scan placeholders. Two unrelated books whose album rows both read
|
||||
# `[Unknown Album]`: the name is shared, so filing on it would merge books
|
||||
# that have nothing to do with each other. A rip timestamp can be glued to
|
||||
# the placeholder, and a multi-cast read says `Various Artists` — which IS a
|
||||
# narrator.
|
||||
H="$AB/Packs"
|
||||
at "$H/Stross/Ship/c1.mp3" 9000
|
||||
at "$H/Leckie/State/c1.mp3" 9000
|
||||
at "$H/Rip/tape.mp3" 9000
|
||||
at "$H/NoWho/Real Book.mp3" 9000
|
||||
at "$H/Multicast/Genesis.mp3" 9000
|
||||
row "$H/Stross/Ship/c1.mp3" "Various Artists" "[Unknown Album]" "" 1 "Part 1"
|
||||
row "$H/Leckie/State/c1.mp3" "Various Artists" "[Unknown Album]" "" 1 "Part 1"
|
||||
row "$H/Rip/tape.mp3" "Ann Reader" "Unknown album (03/03/2015 21:39:55)" "" 1 "One"
|
||||
row "$H/NoWho/Real Book.mp3" "[Unknown Artist]" "A Named Book" 2001 1 "One"
|
||||
row "$H/Multicast/Genesis.mp3" "Various Artists" "The Bible" "" 1 "Genesis"
|
||||
at "$H/Numbered/01 - Grimoire.mp3" 9000
|
||||
at "$H/Digits/1984.mp3" 9000
|
||||
row "$H/Numbered/01 - Grimoire.mp3" "0.4" "Grimoire of the Lamb" "" 1 "One"
|
||||
row "$H/Digits/1984.mp3" "Ann Reader" "1984" 1949 1 "One"
|
||||
|
||||
# The enum TSV is what scripts/media-census.py enumerate writes: bytes, ext,
|
||||
# path, no header.
|
||||
python3 - "$AB" "$ENUM" <<'PY'
|
||||
import os, sys
|
||||
root, out = sys.argv[1], sys.argv[2]
|
||||
with open(out, "w") as fh:
|
||||
for dirpath, _dirs, files in os.walk(root):
|
||||
for fn in files:
|
||||
p = os.path.join(dirpath, fn)
|
||||
fh.write(f"{os.path.getsize(p)}\t{os.path.splitext(fn)[1].lower()}\t{p}\n")
|
||||
PY
|
||||
}
|
||||
|
||||
plan() {
|
||||
run python3 "$PLAN" --root "$AB" --enum "$ENUM" --rows "$ROWS" \
|
||||
--out "$OUT" --exceptions "$EXC"
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "planner said: $output"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# db_with_part DB FILE NARRATOR BOOK YEAR TRACK CHAPTER — one Plex Track row
|
||||
# under an Album under an Artist, in the three tables the export joins.
|
||||
db_with_part() {
|
||||
local db="$1" file="$2" narrator="$3" book="$4" year="$5" track="$6" chapter="$7"
|
||||
local del="${8:-0}"
|
||||
python3 - "$db" "$file" "$narrator" "$book" "$year" "$track" "$chapter" "$del" <<'PY'
|
||||
import sqlite3, sys
|
||||
db, file, narrator, book, year, track, chapter, deleted = sys.argv[1:9]
|
||||
c = sqlite3.connect(db)
|
||||
c.execute("create table if not exists media_parts (id integer primary key,"
|
||||
" media_item_id, file, deleted_at)")
|
||||
c.execute("create table if not exists media_items (id integer primary key,"
|
||||
" metadata_item_id, deleted_at)")
|
||||
c.execute("""create table if not exists metadata_items (id integer primary key,
|
||||
parent_id, metadata_type, title, year, "index", deleted_at)""")
|
||||
now = None if deleted == "0" else "2026-01-01"
|
||||
ar = c.execute("insert into metadata_items (parent_id, metadata_type, title, year,"
|
||||
" \"index\", deleted_at) VALUES (NULL, 8, ?, NULL, NULL, NULL)",
|
||||
(narrator,)).lastrowid
|
||||
al = c.execute("insert into metadata_items (parent_id, metadata_type, title, year,"
|
||||
" \"index\", deleted_at) VALUES (?, 9, ?, ?, NULL, NULL)",
|
||||
(ar, book, int(year) if year else None)).lastrowid
|
||||
mi = c.execute("insert into metadata_items (parent_id, metadata_type, title, year,"
|
||||
" \"index\", deleted_at) VALUES (?, 10, ?, NULL, ?, NULL)",
|
||||
(al, chapter, int(track) if track else -1)).lastrowid
|
||||
it = c.execute("insert into media_items (metadata_item_id, deleted_at) VALUES (?, NULL)",
|
||||
(mi,)).lastrowid
|
||||
c.execute("insert into media_parts (media_item_id, file, deleted_at) VALUES (?,?,?)",
|
||||
(it, file, now))
|
||||
c.commit()
|
||||
PY
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ the export
|
||||
|
||||
@test "the census exports the narrator, book and chapter Plex matched for a part" {
|
||||
DB="$BATS_TEST_TMPDIR/plex.db"
|
||||
db_with_part "$DB" "$BATS_TEST_TMPDIR/Audiobooks/Pack/It part 1/01.mp3" \
|
||||
"George Guidall" "It" 1986 1 "Chapter 1"
|
||||
run python3 "$CENSUS" audiobooks --root "$BATS_TEST_TMPDIR/Audiobooks" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
run head -1 "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "${lines[0]}" = "file narrator book year track chapter" ]
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$BATS_TEST_TMPDIR/Audiobooks/Pack/It part 1/01\.mp3 George Guidall It 1986 1 Chapter 1$"
|
||||
}
|
||||
|
||||
@test "the census exports nothing outside the root, and nothing Plex deleted" {
|
||||
DB="$BATS_TEST_TMPDIR/plex.db"
|
||||
db_with_part "$DB" "/mnt/other/Library/01.mp3" "Someone" "Elsewhere" 1990 1 "One"
|
||||
db_with_part "$DB" "$BATS_TEST_TMPDIR/Audiobooks/Gone/01.mp3" "Someone" \
|
||||
"Deleted" 1990 1 "One" 1
|
||||
run python3 "$CENSUS" audiobooks --root "$BATS_TEST_TMPDIR/Audiobooks" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 1 ] # header only
|
||||
}
|
||||
|
||||
# -------------------------------------------------------------------- the plan
|
||||
|
||||
@test "a matched chapter files under its narrator and its book, name untouched" {
|
||||
plan
|
||||
[ "$(dst "$A [Part 1]/01 - Chapter 1.mp3")" = \
|
||||
"$AB/George Guidall/It (1986)/01 - Chapter 1.mp3" ]
|
||||
}
|
||||
|
||||
@test "one book arrives in one folder however many part folders it shipped in" {
|
||||
plan
|
||||
[ "$(dst "$A [Part 2]/03 - Chapter 3.mp3")" = \
|
||||
"$AB/George Guidall/It (1986)/03 - Chapter 3.mp3" ]
|
||||
[ "$(dst "$A [Part 1]/02 - Chapter 2.mp3")" = \
|
||||
"$AB/George Guidall/It (1986)/02 - Chapter 2.mp3" ]
|
||||
}
|
||||
|
||||
@test "a chapter is never renumbered, whether or not Plex knows its index" {
|
||||
# The law this guards: playback order comes from the embedded track tag, and
|
||||
# three quarters of the books have chapters with no index at all, so a
|
||||
# number written into a name would be invented and would lie.
|
||||
plan
|
||||
[ "$(dst "$B/01.ogg")" = "$AB/Operator X/Project Gold/01.ogg" ]
|
||||
[ "$(dst "$A [Part 1]/01 - Chapter 1.mp3")" = \
|
||||
"$AB/George Guidall/It (1986)/01 - Chapter 1.mp3" ]
|
||||
}
|
||||
|
||||
@test "a book whose album row has no year gets no parentheses" {
|
||||
plan
|
||||
[ "$(dst "$B/01.ogg")" = "$AB/Operator X/Project Gold/01.ogg" ]
|
||||
}
|
||||
|
||||
@test "two chapters claiming one name inside one book are named apart" {
|
||||
plan
|
||||
first="$(dst "$D/Book A/vol1.mp3")"
|
||||
second="$(dst "$D/Book A vol2/vol1.mp3")"
|
||||
[ "$first" != "$second" ]
|
||||
[ -n "$first" ] && [ -n "$second" ]
|
||||
# and the plan still passes the executor's preflight, which refuses a target
|
||||
# two sources claim.
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$AB"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" != *"double-claim"* ]]
|
||||
}
|
||||
|
||||
@test "a cover follows the book whose folder it lies in" {
|
||||
plan
|
||||
[ "$(dst "$A [Part 1]/cover.jpg")" = "$AB/George Guidall/It (1986)/cover.jpg" ]
|
||||
}
|
||||
|
||||
@test "a playlist moves with its book and its bytes are never rewritten" {
|
||||
plan
|
||||
[ "$(dst "$A [Part 1]/It.m3u")" = "$AB/George Guidall/It (1986)/It.m3u" ]
|
||||
before=$(md5sum "$A [Part 1]/It.m3u" | cut -d" " -f1)
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$AB" --journal "$BATS_TEST_TMPDIR/j.jsonl" --apply
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(md5sum "$AB/George Guidall/It (1986)/It.m3u" | cut -d" " -f1)" = "$before" ]
|
||||
# the stored paths inside it still name the old file names, which the tool
|
||||
# leaves alone rather than editing a file the operator owns.
|
||||
[ "$(kind "$A [Part 1]/It.m3u")" != "chapter" ]
|
||||
}
|
||||
|
||||
@test "a note in a folder holding two books has no owner and parks" {
|
||||
plan
|
||||
[ "$(kind "$F/notes.txt")" = "park" ]
|
||||
[ "$(dst "$F/notes.txt")" = "$PARK/Mixed/notes.txt" ]
|
||||
}
|
||||
|
||||
@test "an audio file no row names parks with the path it has today" {
|
||||
plan
|
||||
[ "$(kind "$G/loose.mp3")" = "park" ]
|
||||
[ "$(dst "$G/loose.mp3")" = "$PARK/Odd/loose.mp3" ]
|
||||
}
|
||||
|
||||
@test "a video container parks instead of being renamed into audio" {
|
||||
plan
|
||||
[ "$(kind "$G/concert.mp4")" = "park" ]
|
||||
run grep -c "concert\.mp4" "$EXC"
|
||||
[ "$output" != "0" ]
|
||||
}
|
||||
|
||||
@test "a character Plex says breaks playback never reaches a destination" {
|
||||
plan
|
||||
[ "$(dst "$G/Bad: Name/01 chapter.mp3")" = \
|
||||
"$AB/Some Reader/Trail Marked (1999)/01 chapter.mp3" ]
|
||||
run cut -f3 "$OUT"
|
||||
[[ "$output" != *":"* ]]
|
||||
}
|
||||
|
||||
@test "one book title with two narrators stays two books" {
|
||||
plan
|
||||
[ "$(dst "$C/Dune side a.m4b")" = "$AB/Scott Brick/Dune (1965)/Dune side a.m4b" ]
|
||||
[ "$(dst "$C/Dune 2020.m4b")" = \
|
||||
"$AB/Malcolm Hillgartner/Dune (2020)/Dune 2020.m4b" ]
|
||||
}
|
||||
|
||||
@test "the spelling most chapters carry names the folder, and they share one folder" {
|
||||
plan
|
||||
[ "$(dst "$E/kharn 1.mp3")" = "$AB/David Timson/Kharn (2013)/kharn 1.mp3" ]
|
||||
[ "$(dst "$E/kharn 2.mp3")" = "$AB/David Timson/Kharn (2013)/kharn 2.mp3" ]
|
||||
[ "$(dst "$E/kharn 3.mp3")" = "$AB/David Timson/Kharn (2013)/kharn 3.mp3" ]
|
||||
}
|
||||
|
||||
@test "a row with no book title parks rather than inventing a folder" {
|
||||
plan
|
||||
[ "$(kind "$G/blank book.mp3")" = "park" ]
|
||||
}
|
||||
|
||||
@test "a row with no narrator parks rather than inventing a shelf" {
|
||||
plan
|
||||
[ "$(kind "$G/blank narrator.mp3")" = "park" ]
|
||||
}
|
||||
|
||||
@test "books that share a placeholder album name never share a shelf" {
|
||||
plan
|
||||
[ "$(dst "$H/Stross/Ship/c1.mp3")" = "$PARK/Packs/Stross/Ship/c1.mp3" ]
|
||||
[ "$(dst "$H/Leckie/State/c1.mp3")" = "$PARK/Packs/Leckie/State/c1.mp3" ]
|
||||
}
|
||||
|
||||
@test "a placeholder with a rip timestamp glued to it is still a placeholder" {
|
||||
plan
|
||||
[ "$(kind "$H/Rip/tape.mp3")" = "park" ]
|
||||
}
|
||||
|
||||
@test "a narrator placeholder parks even when the book is named" {
|
||||
plan
|
||||
[ "$(kind "$H/NoWho/Real Book.mp3")" = "park" ]
|
||||
}
|
||||
|
||||
@test "Various Artists is a multi-cast narrator and files its book" {
|
||||
plan
|
||||
[ "$(dst "$H/Multicast/Genesis.mp3")" = \
|
||||
"$AB/Various Artists/The Bible/Genesis.mp3" ]
|
||||
}
|
||||
|
||||
@test "a chapter number mistagged as the narrator parks" {
|
||||
plan
|
||||
[ "$(kind "$H/Numbered/01 - Grimoire.mp3")" = "park" ]
|
||||
}
|
||||
|
||||
@test "a book whose title is digits still files, because only a name must have a letter" {
|
||||
plan
|
||||
[ "$(dst "$H/Digits/1984.mp3")" = "$AB/Ann Reader/1984 (1949)/1984.mp3" ]
|
||||
}
|
||||
|
||||
@test "the row, not the extension, decides what a chapter is" {
|
||||
plan
|
||||
[ "$(kind "$G/NoExt/part01")" = "chapter" ]
|
||||
[ "$(dst "$G/NoExt/part01")" = "$AB/Archivist/Tapes (1971)/part01" ]
|
||||
}
|
||||
|
||||
@test "the plan replays through the executor with nothing refused" {
|
||||
plan
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$AB" \
|
||||
--journal "$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" != *"refusing"* ]]
|
||||
# The refusal this guards is two sources claiming one target: the executor
|
||||
# stops the whole run on it, so the plan must never hand a path out twice.
|
||||
[ -z "$(awk -F '\t' '$1 == "move" {print $3}' "$OUT" | sort | uniq -d)" ]
|
||||
}
|
||||
|
||||
@test "an occupied destination is refused rather than overwritten" {
|
||||
plan
|
||||
mkdir -p "$AB/George Guidall/It (1986)"
|
||||
at "$AB/George Guidall/It (1986)/01 - Chapter 1.mp3" 1
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$AB"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"occupied"* ]]
|
||||
[ -e "$A [Part 1]/01 - Chapter 1.mp3" ]
|
||||
}
|
||||
|
||||
@test "applying the plan and re-planning moves nothing" {
|
||||
plan
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$AB" \
|
||||
--journal "$BATS_TEST_TMPDIR/j.jsonl" --apply
|
||||
[ "$status" -eq 0 ]
|
||||
python3 - "$AB" "$ENUM" <<'PY'
|
||||
import os, sys
|
||||
root, out = sys.argv[1], sys.argv[2]
|
||||
with open(out, "w") as fh:
|
||||
for dirpath, _dirs, files in os.walk(root):
|
||||
for fn in files:
|
||||
p = os.path.join(dirpath, fn)
|
||||
fh.write(f"{os.path.getsize(p)}\t{os.path.splitext(fn)[1].lower()}\t{p}\n")
|
||||
PY
|
||||
# The rows still name the paths the chapters were placed at: that is what a
|
||||
# server re-scan of the moved tree reports.
|
||||
python3 - "$OUT" "$ROWS" <<'PY'
|
||||
import csv, sys
|
||||
out, rows = sys.argv[1], sys.argv[2]
|
||||
moved = {r[1]: r[2] for r in csv.reader(open(out), delimiter="\t") if r[0] == "move"}
|
||||
with open(rows) as fh, open(rows + ".2", "w") as fh2:
|
||||
fh2.write(fh.readline().rstrip("\n") + "\n")
|
||||
for line in fh:
|
||||
f = line.split("\t")[0]
|
||||
fh2.write("\t".join([moved.get(f, f)] + line.rstrip("\n").split("\t")[1:]) + "\n")
|
||||
PY
|
||||
run python3 "$PLAN" --root "$AB" --enum "$ENUM" --rows "$ROWS.2" \
|
||||
--out "$OUT.2" --exceptions "$EXC.2"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(awk -F '\t' '$1 == "move"' "$OUT.2" | wc -l)" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "nothing under the park root is planned again" {
|
||||
plan
|
||||
run awk -F '\t' '$1 == "move" && $2 ~ /_unsorted/' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
Executable
+373
@@ -0,0 +1,373 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the music-library relayout — scripts/media-census.py music (the tag and
|
||||
# provider authority export) and scripts/media-music.py (the plan).
|
||||
#
|
||||
# Jellyfin's music rule is the opposite of the movie and TV ones: the embedded
|
||||
# tags are the metadata and the file name is only a fallback title, so the
|
||||
# authority here is the Audio row Jellyfin already matched (its album-artist,
|
||||
# album, year, disc and track columns) and NOT the path a ripper chose. Every
|
||||
# test states which authority a file has — row only, row with no album item,
|
||||
# sidecar of a file that has a row, nothing at all — and asserts where the PLAN
|
||||
# files it, never how the planner decided.
|
||||
#
|
||||
# The three outcomes that cost a re-run of a 90k-file plan: a track filed under
|
||||
# the wrong artist (it splits one album across two artists), a destination that
|
||||
# two tracks claim (the executor refuses, and the operator loses the run), and a
|
||||
# lyric file that stops matching the basename of the track it belongs to, which
|
||||
# silently loses the lyrics. Everything below is written to catch those, plus the
|
||||
# gate that matters most: the plan must replay through scripts/media-apply.py,
|
||||
# whose preflight is the fleet's only overwrite guard.
|
||||
|
||||
load helpers
|
||||
|
||||
PLAN="$REPO_ROOT/scripts/media-music.py"
|
||||
CENSUS="$REPO_ROOT/scripts/media-census.py"
|
||||
APPLY="$REPO_ROOT/scripts/media-apply.py"
|
||||
|
||||
# at PATH SIZE — a file with deterministic content.
|
||||
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
|
||||
|
||||
# row FILE ARTIST ALBUM YEAR DISC TRACK TITLE [MB] [PERFORMER]
|
||||
# One row of the authority export scripts/media-census.py music writes.
|
||||
row() {
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$1" "$2" "$3" "${4:-}" "${5:-}" "${6:-}" "$7" "${8:-}" "${9:-}" >> "$ROWS"
|
||||
}
|
||||
|
||||
# dst SRC — the destination the plan gives SRC, or nothing when it does not move.
|
||||
dst() {
|
||||
awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $3}' "$OUT"
|
||||
}
|
||||
|
||||
setup() {
|
||||
MUSIC="$BATS_TEST_TMPDIR/Music"
|
||||
PARK="$MUSIC/_unsorted"
|
||||
ENUM="$BATS_TEST_TMPDIR/enum.tsv"
|
||||
ROWS="$BATS_TEST_TMPDIR/rows.tsv"
|
||||
OUT="$BATS_TEST_TMPDIR/plan.tsv"
|
||||
EXC="$BATS_TEST_TMPDIR/exceptions.txt"
|
||||
|
||||
# A discography pack: the artist's own albums buried under "01. Albums",
|
||||
# with a lyric, a cover, an EAC log and a playlist beside the tracks.
|
||||
A="$MUSIC/Artist One - Discography [FLAC]/01. Albums/Album X"
|
||||
at "$A/01. First.flac" 9000
|
||||
at "$A/02. Second.flac" 9000
|
||||
at "$A/01. First.lrc" 10
|
||||
at "$A/cover.jpg" 20
|
||||
at "$A/rip.log" 10
|
||||
at "$A/Album X.m3u" 10
|
||||
|
||||
# A multi-disc compilation: one album (the album artist is the compilation
|
||||
# tag), two discs, and a different performer on each track.
|
||||
B="$MUSIC/Box (2019)"
|
||||
at "$B/Disc 1/01 Track A.flac" 9000
|
||||
at "$B/Disc 2/01 Track B.flac" 9000
|
||||
|
||||
# The box cover lies above the two disc folders: the discs below it own it.
|
||||
at "$B/box.jpg" 10
|
||||
|
||||
# One image plus its cuesheet: a whole album in one file.
|
||||
C="$MUSIC/Image Cue"
|
||||
at "$C/Image.flac" 9000
|
||||
at "$C/Image.cue" 10
|
||||
|
||||
# The hard cases: no row at all, a video container, two tracks claiming one
|
||||
# number and one title, a track with no number, and a track whose album
|
||||
# artist tag is empty so only the performer names it.
|
||||
D="$MUSIC/Odd"
|
||||
at "$D/loose.flac" 9000
|
||||
at "$D/concert.mkv" 9000
|
||||
at "$D/dup1.flac" 9000
|
||||
at "$D/dup2.flac" 9000
|
||||
at "$D/unnumbered.flac" 9000
|
||||
at "$D/lone.flac" 9000
|
||||
|
||||
# A sidecar in a folder that holds two different albums.
|
||||
E="$MUSIC/Mixed"
|
||||
at "$E/one.flac" 9000
|
||||
at "$E/two.flac" 9000
|
||||
at "$E/notes.pdf" 10
|
||||
|
||||
# An album title carrying a character Jellyfin says breaks playback.
|
||||
F="$MUSIC/Question Mark"
|
||||
at "$F/01. Bang.flac" 9000
|
||||
|
||||
# An album whose tags name no album artist at all, and two different
|
||||
# performers; and one album title carried by two different album artists.
|
||||
G="$MUSIC/Tagless"
|
||||
at "$G/one.flac" 9000
|
||||
at "$G/two.flac" 9000
|
||||
H="$MUSIC/Same Title"
|
||||
at "$H/a.flac" 9000
|
||||
at "$H/b.flac" 9000
|
||||
|
||||
# One album whose tracks disagree on the spelling of its name; the minority
|
||||
# spelling sorts first, so it must not win the folder.
|
||||
I="$MUSIC/Puppets"
|
||||
at "$I/one.flac" 9000
|
||||
at "$I/two.flac" 9000
|
||||
at "$I/three.flac" 9000
|
||||
|
||||
# The authority export carries a header; the planner reads it by column name.
|
||||
printf 'file\tartist\talbum\tyear\tdisc\ttrack\ttitle\tmb\tperformer\n' > "$ROWS"
|
||||
row "$A/01. First.flac" "Artist One" "Album X" 2015 1 1 "First Song"
|
||||
row "$A/02. Second.flac" "Artist One" "Album X" 2015 1 2 "Second Song: Reprise"
|
||||
row "$B/Disc 1/01 Track A.flac" "Various Artists" "Best Of" 2019 1 1 "Track A" \
|
||||
"" "Aaa Singer"
|
||||
row "$B/Disc 2/01 Track B.flac" "Various Artists" "Best Of" 2019 2 1 "Track B" \
|
||||
"" "Bbb Singer"
|
||||
row "$C/Image.flac" "Solo" "Single Album" 2001 1 1 "Single Album"
|
||||
row "$D/dup1.flac" "Dup Artist" "Dupes" "" "" 1 "Same"
|
||||
row "$D/dup2.flac" "Dup Artist" "Dupes" "" "" 1 "Same"
|
||||
row "$D/unnumbered.flac" "No Number" "No Number Album" "" "" "" "Untitled"
|
||||
row "$D/lone.flac" "" "Session Works" "" "" 1 "Lone" "" "Session Singer"
|
||||
row "$E/one.flac" "Mix Artist" "First Set" 2000 "" 1 "One"
|
||||
row "$E/two.flac" "Mix Artist" "Second Set" 2001 "" 1 "Two"
|
||||
row "$F/01. Bang.flac" "Billie" "WHEN WE ALL FALL ASLEEP, WHERE DO WE GO?" \
|
||||
2019 "" 1 "!!!!!!"
|
||||
row "$G/one.flac" "" "Two Seasons" 2005 "" 1 "One" "" "Alpha Band"
|
||||
row "$G/two.flac" "" "Two Seasons" 2005 "" 2 "Two" "" "Beta Band"
|
||||
row "$H/a.flac" "Aaa Singer" "Same Title" 2011 "" 1 "A Song"
|
||||
row "$H/b.flac" "Bbb Singer" "Same Title" 2011 "" 1 "B Song"
|
||||
row "$I/one.flac" "Metallica" "Master Of Puppets" 2015 "" 1 "One"
|
||||
row "$I/two.flac" "Metallica" "Master of Puppets" 2015 "" 2 "Two"
|
||||
row "$I/three.flac" "Metallica" "Master of Puppets" 2015 "" 3 "Three"
|
||||
|
||||
# The enum TSV is what scripts/media-census.py enumerate writes: bytes, ext,
|
||||
# path, no header.
|
||||
python3 - "$MUSIC" "$ENUM" <<'PY'
|
||||
import os, sys
|
||||
root, out = sys.argv[1], sys.argv[2]
|
||||
with open(out, "w") as fh:
|
||||
for dirpath, _dirs, files in os.walk(root):
|
||||
for fn in files:
|
||||
p = os.path.join(dirpath, fn)
|
||||
fh.write(f"{os.path.getsize(p)}\t{os.path.splitext(fn)[1].lower()}\t{p}\n")
|
||||
PY
|
||||
}
|
||||
|
||||
plan() {
|
||||
run python3 "$PLAN" --root "$MUSIC" --enum "$ENUM" --rows "$ROWS" \
|
||||
--out "$OUT" --exceptions "$EXC"
|
||||
[ "$status" -eq 0 ] || echo "planner said: $output"
|
||||
}
|
||||
|
||||
@test "the census exports the tag and provider row Jellyfin matched for a track" {
|
||||
DB="$BATS_TEST_TMPDIR/jellyfin.db"
|
||||
python3 - "$DB" <<'PY'
|
||||
import sqlite3, sys, uuid
|
||||
c = sqlite3.connect(sys.argv[1])
|
||||
c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber,
|
||||
ParentIndexNumber, ProductionYear, Album,
|
||||
AlbumArtists, Artists, Data)""")
|
||||
c.execute("""create table BaseItemProviders (ItemId, ProviderId, ProviderValue)""")
|
||||
AU = "MediaBrowser.Controller.Entities.Audio.Audio"
|
||||
AL = "MediaBrowser.Controller.Entities.Audio.MusicAlbum"
|
||||
aid, trk = str(uuid.uuid4()), "/data/Music/Pack/Album X/01. First.flac"
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(aid, None, AL, "Album X", "/data/Music/Pack/Album X", None, None, 2015,
|
||||
None, "Artist One", None, '{}'))
|
||||
c.execute("insert into BaseItemProviders values (?,?,?)",
|
||||
(aid, "MusicBrainzReleaseGroup", "rg-1234"))
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), aid, AU, "First Song", trk, 1, 1, 2015,
|
||||
"album x (the tag)", "Artist One", "Artist One", '{}'))
|
||||
c.commit()
|
||||
PY
|
||||
run python3 "$CENSUS" music --root "$BATS_TEST_TMPDIR/Music" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
run head -1 "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "${lines[0]}" = "file artist album year disc track title mb performer" ]
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$BATS_TEST_TMPDIR/Music/Pack/Album X/01\. First\.flac Artist One Album X 2015 1 1 First Song rg-1234 Artist One$"
|
||||
}
|
||||
|
||||
@test "a track whose folder is no album item still exports from its own tags" {
|
||||
DB="$BATS_TEST_TMPDIR/jellyfin.db"
|
||||
python3 - "$DB" <<'PY'
|
||||
import sqlite3, sys, uuid
|
||||
c = sqlite3.connect(sys.argv[1])
|
||||
c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber,
|
||||
ParentIndexNumber, ProductionYear, Album,
|
||||
AlbumArtists, Artists, Data)""")
|
||||
c.execute("""create table BaseItemProviders (ItemId, ProviderId, ProviderValue)""")
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), None, "MediaBrowser.Controller.Entities.Audio.Audio",
|
||||
"Lone", "/data/Music/Pack/lone.flac", 1, None, None,
|
||||
"Session Works", None, "Session Singer", '{}'))
|
||||
c.commit()
|
||||
PY
|
||||
run python3 "$CENSUS" music --root "$BATS_TEST_TMPDIR/Music" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$BATS_TEST_TMPDIR/Music/Pack/lone\.flac Session Singer Session Works 1 Lone Session Singer$"
|
||||
}
|
||||
|
||||
@test "a tagged track files under artist, album and year with a numbered title" {
|
||||
plan
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(dst "$A/01. First.flac")" = "$MUSIC/Artist One/Album X (2015)/01 - First Song.flac" ]
|
||||
}
|
||||
|
||||
@test "the tag title wins over the name the ripper gave the file" {
|
||||
plan
|
||||
[ "$(dst "$A/02. Second.flac")" = "$MUSIC/Artist One/Album X (2015)/02 - Second Song Reprise.flac" ]
|
||||
}
|
||||
|
||||
@test "a character Jellyfin says breaks playback never reaches a destination" {
|
||||
plan
|
||||
[ "$(dst "$F/01. Bang.flac")" = "$MUSIC/Billie/WHEN WE ALL FALL ASLEEP, WHERE DO WE GO (2019)/01 - !!!!!!.flac" ]
|
||||
run awk -F '\t' '$1 == "move" && $3 ~ /[<>:"|?*]/ {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "every disc of a multi-disc album lands in one folder, numbered apart" {
|
||||
plan
|
||||
[ "$(dst "$B/Disc 1/01 Track A.flac")" = "$MUSIC/Various Artists/Best Of (2019)/1.01 - Track A.flac" ]
|
||||
[ "$(dst "$B/Disc 2/01 Track B.flac")" = "$MUSIC/Various Artists/Best Of (2019)/2.01 - Track B.flac" ]
|
||||
}
|
||||
|
||||
@test "an album whose tracks name several artists files under Various Artists" {
|
||||
plan
|
||||
run awk -F '\t' '$1 == "move" && $3 ~ /\/Best Of \(2019\)\/.*\.flac$/ {print $3}' "$OUT"
|
||||
[ "${#lines[@]}" -eq 2 ]
|
||||
[[ "${lines[0]}" == "$MUSIC/Various Artists/"* ]]
|
||||
}
|
||||
|
||||
@test "an album with no album artist and two performers is a compilation" {
|
||||
plan
|
||||
[ "$(dst "$G/one.flac")" = "$MUSIC/Various Artists/Two Seasons (2005)/01 - One.flac" ]
|
||||
[ "$(dst "$G/two.flac")" = "$MUSIC/Various Artists/Two Seasons (2005)/02 - Two.flac" ]
|
||||
}
|
||||
|
||||
@test "the same album title under two album artists is two albums" {
|
||||
plan
|
||||
[ "$(dst "$H/a.flac")" = "$MUSIC/Aaa Singer/Same Title (2011)/01 - A Song.flac" ]
|
||||
[ "$(dst "$H/b.flac")" = "$MUSIC/Bbb Singer/Same Title (2011)/01 - B Song.flac" ]
|
||||
}
|
||||
|
||||
@test "the folder carries the album spelling most tracks carry" {
|
||||
plan
|
||||
[ "$(dst "$I/one.flac")" = "$MUSIC/Metallica/Master of Puppets (2015)/01 - One.flac" ]
|
||||
[ "$(dst "$I/two.flac")" = "$MUSIC/Metallica/Master of Puppets (2015)/02 - Two.flac" ]
|
||||
[ "$(dst "$I/three.flac")" = "$MUSIC/Metallica/Master of Puppets (2015)/03 - Three.flac" ]
|
||||
}
|
||||
|
||||
@test "a lyric file is renamed to match the track it belongs to" {
|
||||
plan
|
||||
[ "$(dst "$A/01. First.lrc")" = "$MUSIC/Artist One/Album X (2015)/01 - First Song.lrc" ]
|
||||
}
|
||||
|
||||
@test "the album image, the rip log and the playlist follow the album folder" {
|
||||
plan
|
||||
[ "$(dst "$A/cover.jpg")" = "$MUSIC/Artist One/Album X (2015)/cover.jpg" ]
|
||||
[ "$(dst "$A/rip.log")" = "$MUSIC/Artist One/Album X (2015)/rip.log" ]
|
||||
[ "$(dst "$A/Album X.m3u")" = "$MUSIC/Artist One/Album X (2015)/Album X.m3u" ]
|
||||
}
|
||||
|
||||
@test "a cover above the disc folders of one album follows that album" {
|
||||
plan
|
||||
[ "$(dst "$B/box.jpg")" = "$MUSIC/Various Artists/Best Of (2019)/box.jpg" ]
|
||||
}
|
||||
|
||||
@test "an image carrying a whole album keeps its own name beside its cuesheet" {
|
||||
plan
|
||||
[ "$(dst "$C/Image.flac")" = "$MUSIC/Solo/Single Album (2001)/Image.flac" ]
|
||||
[ "$(dst "$C/Image.cue")" = "$MUSIC/Solo/Single Album (2001)/Image.cue" ]
|
||||
}
|
||||
|
||||
@test "an audio file no provider row names is parked, not filed under an artist" {
|
||||
plan
|
||||
[[ "$(dst "$D/loose.flac")" == "$PARK/Odd/loose.flac" ]]
|
||||
}
|
||||
|
||||
@test "a video container in the music library is parked, never renamed to an audio one" {
|
||||
plan
|
||||
[[ "$(dst "$D/concert.mkv")" == "$PARK/Odd/concert.mkv" ]]
|
||||
run awk -F '\t' '$1 == "move" && $3 ~ /\.mka$|\.m4a$/ {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "two tracks claiming one number and one title get two destinations" {
|
||||
plan
|
||||
[ "$(dst "$D/dup1.flac")" = "$MUSIC/Dup Artist/Dupes/01 - Same.flac" ]
|
||||
[ "$(dst "$D/dup2.flac")" = "$MUSIC/Dup Artist/Dupes/01 - Same (2).flac" ]
|
||||
}
|
||||
|
||||
@test "an album with no year is named without one, a track with no number likewise" {
|
||||
plan
|
||||
[ "$(dst "$D/unnumbered.flac")" = "$MUSIC/No Number/No Number Album/Untitled.flac" ]
|
||||
}
|
||||
|
||||
@test "a track naming only a performer files under that performer" {
|
||||
plan
|
||||
[ "$(dst "$D/lone.flac")" = "$MUSIC/Session Singer/Session Works/01 - Lone.flac" ]
|
||||
}
|
||||
|
||||
@test "a sidecar in a folder holding two albums parks instead of guessing" {
|
||||
plan
|
||||
[[ "$(dst "$E/notes.pdf")" == "$PARK/Mixed/notes.pdf" ]]
|
||||
}
|
||||
|
||||
@test "the plan creates every folder before the move that needs it" {
|
||||
plan
|
||||
python3 - "$OUT" "$MUSIC" <<'PY'
|
||||
import os, sys
|
||||
made = set()
|
||||
for line in open(sys.argv[1]).read().splitlines()[1:]:
|
||||
action, _src, dest, _kind = line.split("\t")
|
||||
if action == "mkdir":
|
||||
made.add(dest)
|
||||
elif action == "move":
|
||||
parent = os.path.dirname(dest)
|
||||
while parent and parent != sys.argv[2]:
|
||||
assert parent in made, f"{dest} needs {parent}, which is never made"
|
||||
parent = os.path.dirname(parent)
|
||||
PY
|
||||
}
|
||||
|
||||
@test "no move lands outside the music library" {
|
||||
plan
|
||||
run awk -F '\t' -v l="$MUSIC" \
|
||||
'$1 == "move" && $3 !~ "^" l "/" {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "the executor preflights the plan and no target is claimed twice" {
|
||||
plan
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$BATS_TEST_TMPDIR" \
|
||||
--journal "$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
[ "$status" -eq 0 ]
|
||||
dup=$(awk -F '\t' '$1 == "move" {print $3}' "$OUT" | sort | uniq -d)
|
||||
[ -z "$dup" ]
|
||||
}
|
||||
|
||||
@test "re-planning a library the plan already satisfies moves nothing" {
|
||||
plan
|
||||
# Applying the plan and re-censusing is what the operator does next: both
|
||||
# authority files are keyed by path, so both are replayed through the plan
|
||||
# the way a fresh census of the moved tree would see them. (The enum has no
|
||||
# header; the rows export does, and its header row is not a path.)
|
||||
python3 - "$OUT" "$ENUM" "$ROWS" <<'PY'
|
||||
import os, sys
|
||||
plan = sys.argv[1]
|
||||
moves = {}
|
||||
for line in open(plan).read().splitlines()[1:]:
|
||||
action, src, dest, _kind = line.split("\t")
|
||||
if action == "move":
|
||||
moves[src] = dest
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
os.rename(src, dest)
|
||||
for path, col in ((sys.argv[2], 2), (sys.argv[3], 0)):
|
||||
rows = [line.rstrip("\n").split("\t") for line in open(path)]
|
||||
with open(path, "w") as fh:
|
||||
for f in rows:
|
||||
f[col] = moves.get(f[col], f[col])
|
||||
fh.write("\t".join(f) + "\n")
|
||||
PY
|
||||
plan
|
||||
run awk -F '\t' '$1 == "move" {print $2 " -> " $3}' "$OUT"
|
||||
[ -z "$output" ] || { echo "moved again: $output"; false; }
|
||||
}
|
||||
@@ -374,7 +374,10 @@ PY
|
||||
# apply_plan — plan, then apply for real; the journal goes in the tmpdir.
|
||||
apply_plan() {
|
||||
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
||||
# `/` is always a mountpoint, so the overlay note behaves as it does on a
|
||||
# host that runs the metadata view.
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" \
|
||||
--journal "$JOURNAL" --metadata-view / --apply
|
||||
}
|
||||
|
||||
@test "a run without --apply says what it would do and touches nothing" {
|
||||
@@ -409,6 +412,32 @@ print(sum(1 for l in open(sys.argv[1]) if json.loads(l).get("op") == "move"))' "
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
# The overlay note is only true where the metadata view is really mounted. On a
|
||||
# host whose media servers bind the array directly (dankserver today: both
|
||||
# quadlets mount /mnt/entropy, /var/lib/media-meta does not exist) the sidecars
|
||||
# moved with the media, and "next: run reap" is an order that cannot be carried
|
||||
# out. --metadata-view is how the caller names that path; the default is the
|
||||
# host's view path, and the note rides only when it is a mountpoint.
|
||||
@test "an apply with no mounted metadata view does not order a reap" {
|
||||
plan
|
||||
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"moved"* ]]
|
||||
[[ "$output" != *"media-view.sh reap"* ]]
|
||||
}
|
||||
|
||||
@test "a rollback with no mounted metadata view does not order a reap" {
|
||||
plan
|
||||
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
||||
[ "$status" -eq 0 ]
|
||||
run python3 "$APPLY" rollback --journal "$JOURNAL" --root "$LIB" --apply
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"rolled back"* ]]
|
||||
[[ "$output" != *"media-view.sh reap"* ]]
|
||||
}
|
||||
|
||||
@test "a target already holding a different file stops the whole run" {
|
||||
plan
|
||||
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
|
||||
Executable
+212
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: scripts/media-verify.py — the replay of a relayout journal against a
|
||||
# media server's OWN database, which is the documented proof that a rename
|
||||
# landed (a scan answers 204 whether or not it finds anything).
|
||||
#
|
||||
# Every test states what the database holds and asserts what the command prints
|
||||
# and returns; nothing here reaches into the script's internals. The fixtures are
|
||||
# minimal sqlite files with the columns the two servers really use, so the seam
|
||||
# fails if the tool ever reads the wrong table, forgets Plex's `deleted_at`
|
||||
# filter, or forgets that Jellyfin stores the container-side path (/data/…) while
|
||||
# Plex stores the host path verbatim.
|
||||
|
||||
load helpers
|
||||
|
||||
VERIFY="$REPO_ROOT/scripts/media-verify.py"
|
||||
|
||||
# jf_mk <db> — the Jellyfin shape: one BaseItems table, Path is the column.
|
||||
jf_mk() {
|
||||
python3 - "$1" <<'PY'
|
||||
import sqlite3, sys
|
||||
con = sqlite3.connect(sys.argv[1])
|
||||
con.execute("create table BaseItems (Id INTEGER PRIMARY KEY, Path TEXT, Type TEXT,"
|
||||
" ParentId INTEGER, Name TEXT)")
|
||||
con.commit()
|
||||
PY
|
||||
}
|
||||
|
||||
# jf_row <db> <path> — one item the server believes it has.
|
||||
jf_row() {
|
||||
python3 - "$1" "$2" <<'PY'
|
||||
import sqlite3, sys
|
||||
con = sqlite3.connect(sys.argv[1])
|
||||
con.execute("insert into BaseItems (Path, Type) values (?, 'x')", (sys.argv[2],))
|
||||
con.commit()
|
||||
PY
|
||||
}
|
||||
|
||||
# px_mk <db> — the Plex shape: media_parts joined up to metadata_items.
|
||||
px_mk() {
|
||||
python3 - "$1" <<'PY'
|
||||
import sqlite3, sys
|
||||
con = sqlite3.connect(sys.argv[1])
|
||||
con.execute("create table metadata_items (id INTEGER PRIMARY KEY, title TEXT,"
|
||||
" deleted_at TEXT)")
|
||||
con.execute("create table media_items (id INTEGER PRIMARY KEY, metadata_item_id"
|
||||
" INTEGER, container TEXT)")
|
||||
con.execute("create table media_parts (id INTEGER PRIMARY KEY, media_item_id"
|
||||
" INTEGER, file TEXT, deleted_at TEXT)")
|
||||
con.commit()
|
||||
PY
|
||||
}
|
||||
|
||||
# px_row <db> <path> [DELETED_AT] — one part; a non-empty DELETED_AT is a row a
|
||||
# deleted scan left behind, which must never count as proof.
|
||||
px_row() {
|
||||
python3 - "$1" "$2" "${3:-}" <<'PY'
|
||||
import sqlite3, sys
|
||||
db, path, gone = sys.argv[1], sys.argv[2], sys.argv[3] or None
|
||||
con = sqlite3.connect(db)
|
||||
cur = con.execute("insert into metadata_items (title, deleted_at) values ('t', ?)",
|
||||
(gone,))
|
||||
mi = cur.lastrowid
|
||||
cur = con.execute("insert into media_items (metadata_item_id, container)"
|
||||
" values (?, 'mkv')", (mi,))
|
||||
con.execute("insert into media_parts (media_item_id, file, deleted_at)"
|
||||
" values (?, ?, NULL)", (cur.lastrowid, path))
|
||||
con.commit()
|
||||
PY
|
||||
}
|
||||
|
||||
# move_row <src> <dst> — one move in the journal.
|
||||
move_row() {
|
||||
printf '{"op": "move", "src": "%s", "dst": "%s"}\n' "$1" "$2" >> "$JOURNAL"
|
||||
}
|
||||
|
||||
setup() {
|
||||
SHARE="$BATS_TEST_TMPDIR/mnt/entropy"
|
||||
JFDB="$BATS_TEST_TMPDIR/jellyfin.db"
|
||||
PXDB="$BATS_TEST_TMPDIR/plex.db"
|
||||
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
: > "$JOURNAL"
|
||||
jf_mk "$JFDB"
|
||||
px_mk "$PXDB"
|
||||
}
|
||||
|
||||
@test "every move that the database holds at its destination is reported landed" {
|
||||
move_row "$SHARE/Movies/Old Name (1999).mkv" "$SHARE/Movies/Old Name (1999)/Old Name (1999).mkv"
|
||||
move_row "$SHARE/Movies/Two.mkv" "$SHARE/Movies/Two/Two.mkv"
|
||||
# Jellyfin stores the container-side path: /data stands for the share.
|
||||
jf_row "$JFDB" "/data/Movies/Old Name (1999)/Old Name (1999).mkv"
|
||||
jf_row "$JFDB" "/data/Movies/Two/Two.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[0]}" = "jellyfin: 2 moves checked" ]
|
||||
[ "${lines[1]}" = "landed: 2" ]
|
||||
[ "${lines[2]}" = "still at src: 0" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||
}
|
||||
|
||||
@test "a database that has not rescanned yet is refused, naming the stale rows" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
jf_row "$JFDB" "/data/Movies/Old.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[1]}" = "landed: 0" ]
|
||||
[ "${lines[2]}" = "still at src: 1" ]
|
||||
[[ "$output" == *"$SHARE/Movies/Old.mkv"* ]]
|
||||
}
|
||||
|
||||
@test "a moved file no row of the database names is reported missing" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 1" ]
|
||||
[[ "$output" == *"$SHARE/Movies/Old/Old.mkv"* ]]
|
||||
}
|
||||
|
||||
@test "Plex stores the host path verbatim, so a row at the destination lands" {
|
||||
move_row "$SHARE/TVShows/Show/Season 01/Show S01E01.mkv" \
|
||||
"$SHARE/TVShows/Show (2002)/Season 01/Show (2002) S01E01.mkv"
|
||||
px_row "$PXDB" "$SHARE/TVShows/Show (2002)/Season 01/Show (2002) S01E01.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind plex --db "$PXDB" \
|
||||
--share "$SHARE"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[0]}" = "plex: 1 moves checked" ]
|
||||
[ "${lines[1]}" = "landed: 1" ]
|
||||
}
|
||||
|
||||
@test "a Plex row a deleted scan left behind is not proof a move landed" {
|
||||
move_row "$SHARE/TVShows/Old S01E01.mkv" "$SHARE/TVShows/Show/Old S01E01.mkv"
|
||||
px_row "$PXDB" "$SHARE/TVShows/Show/Old S01E01.mkv" "2026-10-30 00:00:00"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind plex --db "$PXDB" \
|
||||
--share "$SHARE"
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[1]}" = "landed: 0" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 1" ]
|
||||
}
|
||||
|
||||
@test "a database without the columns that kind needs is an error, not a zero" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$PXDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"$PXDB"* ]]
|
||||
[[ "$output" == *"BaseItems"* ]]
|
||||
}
|
||||
|
||||
@test "only move rows are counted, so folder operations cannot inflate the tally" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
jf_row "$JFDB" "/data/Movies/Old/Old.mkv"
|
||||
printf '{"op": "mkdir", "dst": "%s/Movies/Old"}\n' "$SHARE" >> "$JOURNAL"
|
||||
printf '{"op": "rmdir", "path": "%s/Movies/Empty"}\n' "$SHARE" >> "$JOURNAL"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[0]}" = "jellyfin: 1 moves checked" ]
|
||||
}
|
||||
|
||||
@test "a journal with no moves is clean, not a failure" {
|
||||
printf '{"op": "mkdir", "dst": "%s/Movies/Old"}\n' "$SHARE" >> "$JOURNAL"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[1]}" = "landed: 0" ]
|
||||
}
|
||||
|
||||
@test "the offender list stops at --show so a stale library stays readable" {
|
||||
for n in 1 2 3 4; do
|
||||
move_row "$SHARE/Movies/F$n.mkv" "$SHARE/Movies/F$n/F$n.mkv"
|
||||
done
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data --show 2
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 4" ]
|
||||
[[ "$output" == *"$SHARE/Movies/F1/F1.mkv"* ]]
|
||||
[[ "$output" != *"$SHARE/Movies/F4/F4.mkv"* ]]
|
||||
}
|
||||
|
||||
@test "the missing rows are counted by extension, so a sidecar is not mistaken for a lost film" {
|
||||
move_row "$SHARE/Music/Album/Mandatory Fun.log" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Mandatory Fun.log"
|
||||
move_row "$SHARE/Music/Album/Film.mkv" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Film.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 2" ]
|
||||
[ "${lines[4]}" = "missing by extension: .log 1, .mkv 1" ]
|
||||
}
|
||||
|
||||
@test "--ignore-ext keeps a file the server never indexes out of the verdict" {
|
||||
move_row "$SHARE/Music/Album/Song.flac" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Song.flac"
|
||||
jf_row "$JFDB" "/data/Music/Weird Al/Mandatory Fun (2014)/Song.flac"
|
||||
move_row "$SHARE/Music/Album/Mandatory Fun.cue" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Mandatory Fun.cue"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data --ignore-ext log,cue
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[1]}" = "landed: 1" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||
[ "${lines[4]}" = "not indexed (extension ignored): 1" ]
|
||||
}
|
||||
Reference in new issue
Block a user