Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dbcf8a2c35 | ||
|
|
a0dfa61623 | ||
|
|
7ee831adee |
No files matched your search
+14
-22
@@ -128,26 +128,6 @@ where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
||||
and al.deleted_at is null
|
||||
"""
|
||||
|
||||
def open_plex_ro(db):
|
||||
"""Plex's library, read the way Plex writes it.
|
||||
|
||||
`immutable=1` tells sqlite the file never changes, so it skips the
|
||||
write-ahead log: every row a scan has committed but not yet checkpointed
|
||||
is invisible, and reading a live database that way can fail outright as
|
||||
"database disk image is malformed". Plain ro reads the WAL through its
|
||||
-shm; immutable is only the fallback for a database whose -shm cannot be
|
||||
read. The same law scripts/media-verify.py states.
|
||||
"""
|
||||
last = None
|
||||
for opt in ("mode=ro", "mode=ro&immutable=1"):
|
||||
try:
|
||||
return sqlite3.connect(f"file:{db}?{opt}", uri=True)
|
||||
except sqlite3.Error as e:
|
||||
last = e
|
||||
sys.exit(f"cannot open the Plex database read-only ({db}): {last}\n"
|
||||
"Plex runs rootless here: run this as the container's owner, never "
|
||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||
|
||||
|
||||
def enumerate_lib(root, out):
|
||||
"""One row per real file under root: size, extension, path."""
|
||||
@@ -247,7 +227,13 @@ def probe(enum, out, jobs):
|
||||
|
||||
def plex(root, db, out):
|
||||
"""Plex's matched titles for files under root, from its DB, read-only."""
|
||||
con = open_plex_ro(db)
|
||||
uri = f"file:{db}?mode=ro&immutable=1"
|
||||
try:
|
||||
con = sqlite3.connect(uri, uri=True)
|
||||
except sqlite3.Error as e:
|
||||
sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n"
|
||||
"Plex runs rootless here: run this as the container's owner, never "
|
||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||
with open(out, "w", newline="") as fh:
|
||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||
w.writerow(PLEX_COLS)
|
||||
@@ -419,7 +405,13 @@ def audiobooks(root, db, out):
|
||||
at least one un-indexed chapter, so the relayout that reads this file must
|
||||
not renumber anything from it.
|
||||
"""
|
||||
con = open_plex_ro(db)
|
||||
uri = f"file:{db}?mode=ro&immutable=1"
|
||||
try:
|
||||
con = sqlite3.connect(uri, uri=True)
|
||||
except sqlite3.Error as e:
|
||||
sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n"
|
||||
"Plex runs rootless here: run this as the container's owner, never "
|
||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||
tables = {r[0] for r in con.execute(
|
||||
"select name from sqlite_master where type='table'")}
|
||||
missing = {"media_parts", "media_items", "metadata_items"} - tables
|
||||
|
||||
+12
-47
@@ -9,13 +9,8 @@ its destination, and NO row may still name its source — a source row is a scan
|
||||
that has not caught up, and it is the one signature of a library pointing at
|
||||
files that no longer exist.
|
||||
|
||||
Two doors, both read-only, both without a lock while the server runs (there is
|
||||
no `sqlite3` binary on these hosts). Open `mode=ro` PLAIN: both servers keep
|
||||
their libraries in WAL mode, and `immutable=1` — which claims the file never
|
||||
changes — makes sqlite skip the write-ahead log entirely, so every row the
|
||||
server has committed but not yet checkpointed reads as MISSING, and a read
|
||||
beside a running scanner can even fail as "database disk image is malformed".
|
||||
`immutable=1` is only the fallback, for a database whose `-shm` cannot be read.
|
||||
Two doors, both read-only, both without a lock while the server runs
|
||||
(`mode=ro&immutable=1`; there is no `sqlite3` binary on these hosts):
|
||||
|
||||
jellyfin `BaseItems.Path`. The container binds the share at `/data`, so the
|
||||
database holds `/data/<library>/…` where the journal holds
|
||||
@@ -27,14 +22,9 @@ beside a running scanner can even fail as "database disk image is malformed".
|
||||
|
||||
Neither server holds a row for everything the journal renames: a `.log`, `.cue`
|
||||
or cover image has no row to land in, so its absence is not a fault. The report
|
||||
therefore breaks the missing rows down by extension, and `--ignore-ext` names the
|
||||
extensions this server does not index, which are then counted separately and kept
|
||||
out of the verdict. What is left is split by what the filesystem says, because
|
||||
the database alone cannot: a file still at its `src` is `REAL` (this pass's own
|
||||
damage), a file at neither path is `gone` (a later pass or the server renamed it),
|
||||
and a file sitting at its `dst` with no row naming it is simply something the
|
||||
server does not index. A row that still names `src` is none of these: it is the
|
||||
server's own stale pointer — a phantom — and it costs a re-scan, never a restore.
|
||||
therefore breaks the missing rows down by extension, and `--ignore-ext` names
|
||||
the extensions this server does not index, which are then counted separately and
|
||||
kept out of the verdict.
|
||||
|
||||
Exit codes: 0 every move landed; 1 something is stale or missing (the details are
|
||||
printed); 2 the journal or the database is not usable at all.
|
||||
@@ -102,18 +92,10 @@ def read_moves(path):
|
||||
def server_paths(db, kind):
|
||||
"""Every path the server holds, or a refusal that names what is missing."""
|
||||
spec = KINDS[kind]
|
||||
con = None
|
||||
err = None
|
||||
# Plain ro reads the WAL the server commits through; immutable is the last
|
||||
# resort, because it cannot see a row the server has not checkpointed.
|
||||
for opt in ("mode=ro", "mode=ro&immutable=1"):
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db}?{opt}", uri=True)
|
||||
break
|
||||
except sqlite3.Error as e:
|
||||
err = e
|
||||
if con is None:
|
||||
die(f"cannot open {db}: {err}")
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db}?mode=ro&immutable=1", uri=True)
|
||||
except sqlite3.Error as e:
|
||||
die(f"cannot open {db}: {e}")
|
||||
with con:
|
||||
have = {r[1] for r in con.execute(f"pragma table_info({spec['table']})")}
|
||||
if not spec["columns"] <= have:
|
||||
@@ -179,37 +161,20 @@ def main():
|
||||
if at_src:
|
||||
stale.append(src)
|
||||
if not at_dst:
|
||||
missing.append((src, dst))
|
||||
|
||||
# The database cannot tell a lost file from one it never indexes: both read
|
||||
# as "no row at the destination". Stat both paths, so the report separates
|
||||
# the classes the runbook names — `REAL` is the only one that is this pass's
|
||||
# own damage, and `gone` is a later pass or the server's own rename.
|
||||
on_disk, real, gone = [], [], []
|
||||
for src, dst in missing:
|
||||
if os.path.exists(dst):
|
||||
on_disk.append(dst)
|
||||
elif os.path.exists(src):
|
||||
real.append(src)
|
||||
else:
|
||||
gone.append(dst)
|
||||
missing.append(dst)
|
||||
|
||||
print(f"{args.db_kind}: {len(moves)} moves checked")
|
||||
print(f"landed: {landed}")
|
||||
print(f"still at src: {len(stale)}")
|
||||
print(f"missing from the DB: {len(missing)}")
|
||||
print(f" on disk at dst, no row names it: {len(on_disk)}")
|
||||
print(f" REAL — file still at src: {len(real)}")
|
||||
print(f" gone — at neither path: {len(gone)}")
|
||||
if ignored:
|
||||
print(f"not indexed (extension ignored): {ignored}")
|
||||
if missing:
|
||||
exts = Counter(os.path.splitext(d)[1].lower() or "(no extension)"
|
||||
for _, d in missing)
|
||||
for d in missing)
|
||||
print("missing by extension: "
|
||||
+ ", ".join(f"{e} {n}" for e, n in sorted(exts.items())))
|
||||
for label, items in (("real", real), ("gone", gone), ("missing", on_disk),
|
||||
("stale", stale)):
|
||||
for label, items in (("stale", stale), ("missing", missing)):
|
||||
for path in items[:args.show]:
|
||||
print(f" {label}: {path}")
|
||||
if len(items) > args.show:
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
type: reference
|
||||
title: Media Audiobook Relayout
|
||||
description: "Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals."
|
||||
tags: [dankserver, media-server, plex, media-relayout]
|
||||
timestamp: 2026-10-31T00:00:00Z
|
||||
---
|
||||
|
||||
# Media Audiobook Relayout
|
||||
|
||||
`/mnt/entropy/Audiobooks` is the one library here that [[Jellyfin]] does not read
|
||||
at all — its music scanner has no audiobook concept — while [[Plex]] indexes all
|
||||
of it as Artist → Album → Track: the album is the BOOK, that book's single artist
|
||||
is the NARRATOR, both matched out of the files' own tags. Today one book is split
|
||||
across `… [Part 1]`, `… [Part 2]` folders three or four deep inside a collection
|
||||
pack, so a scan serves three quarters of a novel; `scripts/media-audiobooks.py`
|
||||
re-files the tree so the property both servers' book scanners assume is true —
|
||||
|
||||
Audiobooks/<Narrator>/<Book> (<year>)/<the file keeps its own name>
|
||||
|
||||
The shared gates, the executor, the journal law and the rollback live in
|
||||
[[Media Relayout]]; this page is the audiobook arm only.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Plex's chapter rows for this root (the DB opens read-only).
|
||||
python3 scripts/media-census.py audiobooks --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-audiobooks.py --root /mnt/entropy/Audiobooks \
|
||||
--enum /tmp/ab_enum.tsv --rows /tmp/ab_rows.tsv --out /tmp/ab_plan.tsv \
|
||||
--exceptions /tmp/ab_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST. Then the same executor, journal and rollback
|
||||
# as [[Media Relayout]] shows, on its own journal, and refresh the Plex section
|
||||
# as in step 5 of the movie Procedure — from inside the container, never over
|
||||
# the host port.
|
||||
scripts/media-apply.py apply --plan /tmp/ab_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-audiobooks.jsonl --apply
|
||||
```
|
||||
|
||||
The group is the identity Plex asserts in its own database — **(narrator, book,
|
||||
year)** — so one book arriving in four part folders becomes one folder, and a
|
||||
1965 and a 2020 recording that share a title stay two books.
|
||||
|
||||
Two laws are forced by what the rows actually hold:
|
||||
|
||||
+- **A chapter is never renumbered.** Most books contain chapters whose index Plex
|
||||
does not know, so a number written into a name would be invented — and it would
|
||||
buy nothing, because both servers order playback by the embedded track tag, not
|
||||
by the file name. A wrong number in a name is the number a human later trusts.
|
||||
Fix order in the tags: a Plex scan re-reads tags and never renames a file.
|
||||
+- **The shelf is the narrator, not the author.** Every book has exactly one artist
|
||||
row and it is the narrator; the author survives only inside some album titles
|
||||
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving it would
|
||||
mean parsing titles where a ripper wrote one and inventing it everywhere else.
|
||||
The author stays visible in the folder name Plex itself shows.
|
||||
|
||||
What it refuses, the music arm's list with one addition: a video container, audio
|
||||
no chapter row names, and a loose file in a folder holding two books all park under
|
||||
`_unsorted` with the path they have today — and so does a chapter whose row names
|
||||
**no book or no narrator**: a placeholder (`[Unknown Album]`, `[Unknown Artist]`)
|
||||
that unrelated books share, or a chapter number mistagged as the artist (`0.4`).
|
||||
`Various Artists` is not a placeholder — that is a multi-cast read.
|
||||
|
||||
The reviewed plan lives at `/mnt/entropy/media-relayout-audiobooks-plan.tsv`; it
|
||||
applies on its own journal, behind the music writer, and then refreshes the Plex
|
||||
section. Verify it with `--db-kind plex` — Jellyfin holds no rows here, so a
|
||||
Jellyfin pass proves nothing either way.
|
||||
|
||||
## See Also
|
||||
|
||||
[[Media Relayout]], [[Media Music Relayout]], [[Plex]]
|
||||
@@ -0,0 +1,68 @@
|
||||
---
|
||||
type: reference
|
||||
title: Media Music Relayout
|
||||
description: "Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals."
|
||||
tags: [dankserver, media-server, jellyfin, media-relayout]
|
||||
timestamp: 2026-10-31T00:00:00Z
|
||||
---
|
||||
|
||||
# Media Music Relayout
|
||||
|
||||
`/mnt/entropy/Music` is rips and discography packs, and Jellyfin has matched and
|
||||
tagged almost all of it. `scripts/media-music.py` re-files it into the shape
|
||||
Jellyfin's music scanner reads —
|
||||
|
||||
Music/<album artist>/<Album> (<year>)/NN - <Title>.ext
|
||||
|
||||
— and the authority is the tag row of the track, never the name the ripper gave
|
||||
the file. The shared gates, the executor, the journal law and the rollback live in
|
||||
[[Media Relayout]]; this page is the music arm only.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Jellyfin's audio rows, one per track, with the album and
|
||||
# artist ids its own database carries (the DB opens read-only).
|
||||
python3 scripts/media-census.py music --root /mnt/entropy/Music \
|
||||
--out /tmp/music_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Music \
|
||||
--out /tmp/music_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-music.py --root /mnt/entropy/Music --enum /tmp/music_enum.tsv \
|
||||
--rows /tmp/music_rows.tsv --out /tmp/music_plan.tsv \
|
||||
--exceptions /tmp/music_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Then the same
|
||||
# executor, journal and rollback as [[Media Relayout]] shows, on its own journal.
|
||||
scripts/media-apply.py apply --plan /tmp/music_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-music.jsonl --apply
|
||||
```
|
||||
|
||||
The group is what the tags assert, not where the files sit: **(album-artist tag,
|
||||
album, year)**. One album arrives as three disc folders and as two copies of one
|
||||
rip, and one MusicBrainz release group is one album. A compilation is one album
|
||||
whose tracks name many performers, so it files once under `Various Artists`:
|
||||
keying on the performer would file one album once per artist. Two different
|
||||
album-artist tags on one album name are two albums, which is what Jellyfin's two
|
||||
album items say.
|
||||
|
||||
What it refuses:
|
||||
|
||||
+- **A sidecar is placed by the audio around it, never by its own name.** A
|
||||
`.lrc`, `.txt` or picture that shares a track's stem follows that track and
|
||||
takes the name the track now has, or Jellyfin cannot find it; anything else
|
||||
goes to the album its own folder was placed in, or to the folder whose discs
|
||||
sit one level below it. A folder whose audio belongs to two albums parks its
|
||||
sidecar rather than guessing.
|
||||
+- **An image album keeps its name.** A folder holding one audio file and a
|
||||
`.cue` is a whole album in one file; numbering it `01 - …` tells Jellyfin to
|
||||
index it as one track.
|
||||
+- **A video container parks**, because Jellyfin will not index it as music and a
|
||||
rename to `.m4a` would claim it is; **audio no provider row names parks** too,
|
||||
because unmatched audio is a tagging job, not a renaming job.
|
||||
|
||||
ONE writer at a time on this array: two rename passes race, and neither journal
|
||||
tells. Verify the run against Jellyfin's own tables — see *Proving the move
|
||||
landed* in [[Media Relayout]].
|
||||
|
||||
## See Also
|
||||
|
||||
[[Media Relayout]], [[Jellyfin]], [[Media Audiobook Relayout]]
|
||||
@@ -47,8 +47,10 @@ $C plex --root /mnt/entropy/Movies --out /tmp/plex-movies.tsv
|
||||
```
|
||||
|
||||
`plex` reads the Plex metadata cache (`--db` defaults to the path the
|
||||
[[Plex]] container bind-mounts at `/var/lib/plexmediaserver`) with
|
||||
`mode=ro&immutable=1`: a live server is never locked or written.
|
||||
[[Plex]] container bind-mounts at `/var/lib/plexmediaserver`) with plain
|
||||
`mode=ro`: a live server is never locked or written, and plain ro is also what
|
||||
makes a scan's uncheckpointed rows visible. The same rule as
|
||||
`scripts/media-verify.py` — see *Proving the move landed*.
|
||||
|
||||
Then plan — this reads only the three TSVs, so it costs seconds and can run
|
||||
anywhere, including a laptop:
|
||||
@@ -123,14 +125,34 @@ replayed against both servers' own tables: every `move` row has a row at its
|
||||
replay: it opens each database read-only (there is no `sqlite3` binary on these
|
||||
hosts or inside either container) and exits non-zero while a row is stale.
|
||||
|
||||
Both libraries live in WAL mode, so the tool opens them plain `mode=ro`:
|
||||
`immutable=1` promises sqlite the file never changes, which skips the
|
||||
write-ahead log — a scanner's own uncheckpointed rows then read as missing, and
|
||||
a read beside a running scan can fail as "database disk image is malformed".
|
||||
`immutable` stays only the fallback for a database whose `-shm` cannot be read.
|
||||
|
||||
```bash
|
||||
scripts/media-verify.py --journal /mnt/entropy/media-relayout-music.jsonl \
|
||||
--db-kind jellyfin --ignore-ext jpg,jpeg,png,log,cue,toc,nfo,txt,m3u,accurip
|
||||
```
|
||||
|
||||
Ignore only what a server cannot index. A missing `.flac` or `.mkv` is the
|
||||
fault; a row still naming its `src` means the scan has not followed the rename
|
||||
yet, so re-run the check after the scan, never before.
|
||||
"No row at the destination" is not one verdict but four. **never indexed**: the
|
||||
server indexes no file of that extension (its own extension census says so) — a
|
||||
sidecar, not a lost film. **gone**: the file sits at neither src nor dst, because
|
||||
a later pass or the server's own extension sweep renamed it. **phantom**: a row
|
||||
still naming src while the file has left src. **REAL**: no row at dst *and* the
|
||||
file still at src — the only class that is this relayout's own damage. Phantoms
|
||||
are normal for both servers (Plex carries thousands that predate any relayout);
|
||||
re-scan to clear them and never read them as loss.
|
||||
|
||||
`media-verify.py` prints this split itself — `on disk at dst, no row names it`,
|
||||
`REAL — file still at src`, `gone — at neither path` — so nobody hand-stats a
|
||||
thousand paths to find out whether a film is lost or merely unindexed.
|
||||
|
||||
The owning server decides whether a pass can convict anything: Plex's roots are
|
||||
its `sections` rows joined to their location rows (`Movies`, `TVShows`, `Music`,
|
||||
`Audiobooks`, `LiveRecording`) and Jellyfin holds no audiobook library at all, so
|
||||
the audiobook journal verifies with `--db-kind plex` or it proves nothing.
|
||||
|
||||
Three doors that look right and are not: `GET /search?query=` matches metadata
|
||||
titles, never file paths; `?path=` on `/library/sections/N/all` makes Plex's
|
||||
@@ -167,16 +189,15 @@ from the box-set folder. What it refuses it prints and leaves alone:
|
||||
- a destination two files both claim: the executor refuses the whole plan
|
||||
before its first rename, and only Frank decides which copy is the keeper.
|
||||
- an episode the destination library already holds at the same byte count. The
|
||||
movie share carried a second copy of the whole Monk box set — 124 episodes
|
||||
both sides, the same episode list, every byte count identical, 31.5 GiB each
|
||||
way, and head+tail blake2b identical on a 6-episode sample. The mover leaves
|
||||
such a series in place and says so; a duplicate is deleted or kept by Frank,
|
||||
never copied into one library by a tool.
|
||||
movie share once carried a second copy of a whole box set — same episode list,
|
||||
every byte count identical. The mover leaves such a series in place and says
|
||||
so; a duplicate is deleted or kept by Frank, never copied into one library by
|
||||
a tool.
|
||||
|
||||
A sidecar that shares an episode's own name (`.srt`, `.nfo`) moves with it.
|
||||
|
||||
Below the threshold the mover prints and leaves, so a lone downloaded episode of
|
||||
a real show stays in the movie library on purpose — six did on 2026-10-30.
|
||||
a real show stays in the movie library on purpose.
|
||||
Filing one by hand is the same executor on a hand-written plan and the same
|
||||
journal; the trap is the series year. Read it from the file's own name or the
|
||||
folder it owns, never from the title whose bonus folder it happens to be parked
|
||||
@@ -243,119 +264,24 @@ filename. Renaming a show is Jellyfin's work — a folder rename never touches i
|
||||
provider ids — so the mover keeps the name it is handed and strips only the pack
|
||||
index a release leads with (`4a. Seed DESTINY` → `Seed DESTINY`).
|
||||
|
||||
## The music library's own relayout
|
||||
Release-group folder names are the one thing no provider can read, so the
|
||||
pending TVShows naming plan files 179 top-level directories (9,931 files) under
|
||||
the junk name they arrived with. A TVDB id map can name them, under the ladder's
|
||||
own acceptance rule: a hit counts only when TVDB's spelling equals the cleaned
|
||||
query, or differs from it by punctuation alone — the same letters are not a
|
||||
guess. Everything else is a choice for Frank, and the map is never applied
|
||||
unreviewed: `/mnt/entropy/media-relayout-shows-tvdb-proposal.tsv` carries one
|
||||
row per folder with `verdict` = `EXACT` / `PUNCT` / `NEAR` / `NONE` and a `note`
|
||||
naming the two traps — a folder marked `(US)` that matches the UK original, and
|
||||
several folders mapping to one id, which is a MERGE whose duplicate episodes the
|
||||
executor will refuse until a keeper is chosen.
|
||||
|
||||
`/mnt/entropy/Music` is rips and discography packs, and Jellyfin has matched and
|
||||
tagged almost all of it. `scripts/media-music.py` re-files it into the shape
|
||||
Jellyfin's music scanner reads —
|
||||
## The music and audiobook arms
|
||||
|
||||
Music/<album artist>/<Album> (<year>)/NN - <Title>.ext
|
||||
|
||||
— and the authority is the tag row of the track, never the name the ripper gave
|
||||
the file.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Jellyfin's audio rows, one per track, with the album and
|
||||
# artist ids its own database carries (the DB opens read-only).
|
||||
python3 scripts/media-census.py music --root /mnt/entropy/Music \
|
||||
--out /tmp/music_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Music \
|
||||
--out /tmp/music_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-music.py --root /mnt/entropy/Music --enum /tmp/music_enum.tsv \
|
||||
--rows /tmp/music_rows.tsv --out /tmp/music_plan.tsv \
|
||||
--exceptions /tmp/music_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Then the same
|
||||
# executor, journal and rollback as above, on its own journal.
|
||||
scripts/media-apply.py apply --plan /tmp/music_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-music.jsonl --apply
|
||||
```
|
||||
|
||||
The group is what the tags assert, not where the files sit: **(album-artist tag,
|
||||
album, year)**. One album arrives as three disc folders and as two copies of one
|
||||
rip, and one MusicBrainz release group is one album. A compilation is one album
|
||||
whose tracks name many performers, so it files once under `Various Artists`:
|
||||
keying on the performer would file one album once per artist. Two different
|
||||
album-artist tags on one album name are two albums, which is what Jellyfin's two
|
||||
album items say.
|
||||
|
||||
What it refuses:
|
||||
|
||||
+- **A sidecar is placed by the audio around it, never by its own name.** A
|
||||
`.lrc`, `.txt` or picture that shares a track's stem follows that track and
|
||||
takes the name the track now has, or Jellyfin cannot find it; anything else
|
||||
goes to the album its own folder was placed in, or to the folder whose discs
|
||||
sit one level below it. A folder whose audio belongs to two albums parks its
|
||||
sidecar rather than guessing.
|
||||
+- **An image album keeps its name.** A folder holding one audio file and a
|
||||
`.cue` is a whole album in one file; numbering it `01 - …` tells Jellyfin to
|
||||
index it as one track.
|
||||
+- **A video container parks**, because Jellyfin will not index it as music and a
|
||||
rename to `.m4a` would claim it is; **audio no provider row names parks** too,
|
||||
because unmatched audio is a tagging job, not a renaming job.
|
||||
|
||||
The plan applies on 2026-10-31 through `/mnt/entropy/media-relayout-music.jsonl`;
|
||||
its rollback is that journal run through `media-apply.py rollback --apply`. ONE
|
||||
writer at a time on this array: two rename passes race, and neither journal tells.
|
||||
|
||||
## The audiobook library's own relayout
|
||||
|
||||
`/mnt/entropy/Audiobooks` is the one library here that [[Jellyfin]] does not read
|
||||
at all — its music scanner has no audiobook concept — while [[Plex]] indexes all
|
||||
of it as Artist → Album → Track: the album is the BOOK, that book's single artist
|
||||
is the NARRATOR, both matched out of the files' own tags. Today one book is split
|
||||
across `… [Part 1]`, `… [Part 2]` folders three or four deep inside a collection
|
||||
pack, so a scan serves three quarters of a novel; `scripts/media-audiobooks.py`
|
||||
re-files the tree so the property both servers' book scanners assume is true —
|
||||
|
||||
Audiobooks/<Narrator>/<Book> (<year>)/<the file keeps its own name>
|
||||
|
||||
```bash
|
||||
# 1. The authority: Plex's chapter rows for this root (the DB opens read-only).
|
||||
python3 scripts/media-census.py audiobooks --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-audiobooks.py --root /mnt/entropy/Audiobooks \
|
||||
--enum /tmp/ab_enum.tsv --rows /tmp/ab_rows.tsv --out /tmp/ab_plan.tsv \
|
||||
--exceptions /tmp/ab_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST. Then the same executor, journal and rollback
|
||||
# as above, on its own journal, and refresh the Plex section as in step 5 of
|
||||
# the movie Procedure — from inside the container, never over the host port.
|
||||
scripts/media-apply.py apply --plan /tmp/ab_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-audiobooks.jsonl --apply
|
||||
```
|
||||
|
||||
The group is the identity Plex asserts in its own database — **(narrator, book,
|
||||
year)** — so one book arriving in four part folders becomes one folder, and a
|
||||
1965 and a 2020 recording that share a title stay two books.
|
||||
|
||||
Two laws are forced by what the rows actually hold:
|
||||
|
||||
+- **A chapter is never renumbered.** Most books contain chapters whose index Plex
|
||||
does not know, so a number written into a name would be invented — and it would
|
||||
buy nothing, because both servers order playback by the embedded track tag, not
|
||||
by the file name. A wrong number in a name is the number a human later trusts.
|
||||
Fix order in the tags: a Plex scan re-reads tags and never renames a file.
|
||||
+- **The shelf is the narrator, not the author.** Every book has exactly one artist
|
||||
row and it is the narrator; the author survives only inside some album titles
|
||||
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving it would
|
||||
mean parsing titles where a ripper wrote one and inventing it everywhere else.
|
||||
The author stays visible in the folder name Plex itself shows.
|
||||
|
||||
What it refuses, the music arm's list with one addition: a video container, audio
|
||||
no chapter row names, and a loose file in a folder holding two books all park under
|
||||
`_unsorted` with the path they have today — and so does a chapter whose row names
|
||||
**no book or no narrator**: a placeholder (`[Unknown Album]`, `[Unknown Artist]`)
|
||||
that unrelated books share, or a chapter number mistagged as the artist (`0.4`).
|
||||
`Various Artists` is not a placeholder — that is a multi-cast read.
|
||||
|
||||
The plan stands reviewed at `/mnt/entropy/media-relayout-audiobooks-plan.tsv`
|
||||
(55,114 moves, 7,764 parked, executor preflight refused nothing); it applies on its
|
||||
own journal, behind the music writer, and then refreshes the Plex section.
|
||||
Each is a page of its own, because each reads a different authority and refuses
|
||||
for a different reason: [[Media Music Relayout]] files by Jellyfin's track tags,
|
||||
[[Media Audiobook Relayout]] by Plex's chapter rows. Both share this page's
|
||||
executor, the one-journal-per-arm law, the sweep and the rollback.
|
||||
|
||||
## Laws
|
||||
|
||||
@@ -437,16 +363,15 @@ own journal, behind the music writer, and then refreshes the Plex section.
|
||||
recreates the folder, so the sweep is undone by the same command as the moves.
|
||||
Proof carries downward: inside a folder the journal proves emptied, every
|
||||
empty directory goes with it. A release's `Screens/` or `Sample/` is never a
|
||||
move source itself, so 146 of them held their emptied parents on disk after
|
||||
the first sweep.
|
||||
move source itself, so it can hold an emptied parent on disk after a sweep.
|
||||
A failed `rmdir` is not a removal: only a folder that is really gone leaves
|
||||
scope, so the next sweep finishes what a permission problem stopped.
|
||||
- **An episode-shaped name marks a SERIES, not a film — and the series is the
|
||||
name BEFORE the marker.** `Monk - S05E03 - Mr. Monk and the Big Game` is series
|
||||
`Monk`. Keyed on the whole title instead, every episode became a group of one
|
||||
and never reached the five-episode size that calls something a series, so a
|
||||
box set was filed as 124 movies in 124 folders. The threshold still earns its
|
||||
keep: a lone film named `Part 2` stays a film. Episodes of a series are parked
|
||||
`Monk`. Keyed on the whole title instead, every episode became a group of one,
|
||||
never reaching the five-episode size that calls something a series — a whole
|
||||
box set filed as one folder per episode. The threshold still earns its keep:
|
||||
a lone film named `Part 2` stays a film. Episodes of a series are parked
|
||||
in `_TV needs a TV library - <series>` for a human to move to a TV source —
|
||||
the movie tool never files TV.
|
||||
- **One journal is one undo.** The TV move runs on its own journal, and with
|
||||
@@ -489,6 +414,14 @@ any folder a sweep removed, restores every source path, removes only the folders
|
||||
that run created *and that are now empty*, and refuses everything if any moved
|
||||
file cannot be accounted for. A rolled-back journal refuses to roll back twice.
|
||||
|
||||
**Roll a chain back newest journal first, or the tool refuses.** Each pass
|
||||
re-files what an earlier one moved, so a journal is a clean inverse only until
|
||||
something downstream touches its files; `ls -l /mnt/entropy/media-relayout*.jsonl`
|
||||
is the order. Refusing is the tool working: on 2026-10-31 the movie journal could
|
||||
not restore files the later TV passes had re-filed, and the stray journal could
|
||||
not restore files Jellyfin's extension sweep had renamed. Roll back what still
|
||||
inverts and leave the rest as the current state — never force it.
|
||||
|
||||
## See Also
|
||||
|
||||
[[Jellyfin]], [[Plex]], [[Media Metadata View]], [[Entropy Volume]], [[Testing Policy]], [[Storage Policy]]
|
||||
|
||||
@@ -51,7 +51,9 @@ is a locator: one row, one clause, no numbers.
|
||||
| [[Entropy Volume]] | The Dankserver entropy volume is a RAID-backed LUKS filesystem mounted at |
|
||||
| [[Jellyfin]] | Jellyfin on Dankserver: quadlet unit, overlay metadata view, provider keys, ports, access, transcode status. |
|
||||
| [[mariadb]] | The MariaDB backend for RomM: quadlet unit, network alias, env-file secret handling, and the backup path. |
|
||||
| [[Media Audiobook Relayout]] | Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals. |
|
||||
| [[Media Metadata View]] | The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback. |
|
||||
| [[Media Music Relayout]] | Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals. |
|
||||
| [[Media Relayout]] | Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback. |
|
||||
| [[Plex]] | Plex Media Server is a **containerized media server on Dankserver**: a podman |
|
||||
| [[redis]] | The Redis cache and task queue for RomM: quadlet unit, network alias topology, and the backup path. |
|
||||
|
||||
@@ -228,51 +228,6 @@ PY
|
||||
[ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 1 ] # header only
|
||||
}
|
||||
|
||||
# wal_fresh DB ROOT OUT — a row Plex matched during a scan and committed, but
|
||||
# has not checkpointed: the normal state of a library being walked. The holder
|
||||
# connection stops the WAL folding into the main file when the writer closes,
|
||||
# and the census runs as a real subprocess against those same files.
|
||||
wal_fresh() {
|
||||
local db="$1" root="$2" out="$3"
|
||||
python3 - "$db" "$root" "$out" "$CENSUS" <<'PY'
|
||||
import sqlite3, subprocess, sys
|
||||
db, root, out, census = sys.argv[1:5]
|
||||
w = sqlite3.connect(db)
|
||||
w.execute("pragma journal_mode=wal")
|
||||
w.execute("pragma wal_autocheckpoint=0")
|
||||
w.commit()
|
||||
w.execute("pragma wal_checkpoint(truncate)") # the rows so far are on disk
|
||||
holder = sqlite3.connect(db) # last close would checkpoint
|
||||
ar = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||
' year, "index", deleted_at) VALUES (NULL, 8, ?, NULL, NULL, NULL)',
|
||||
("Fresh Narrator",)).lastrowid
|
||||
al = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||
' year, "index", deleted_at) VALUES (?, 9, ?, ?, NULL, NULL)',
|
||||
(ar, "Fresh Book", 2026)).lastrowid
|
||||
mi = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||
' year, "index", deleted_at) VALUES (?, 10, ?, NULL, ?, NULL)',
|
||||
(al, "Fresh Chapter", 1)).lastrowid
|
||||
it = w.execute("insert into media_items (metadata_item_id, deleted_at)"
|
||||
" VALUES (?, NULL)", (mi,)).lastrowid
|
||||
w.execute("insert into media_parts (media_item_id, file, deleted_at)"
|
||||
" VALUES (?, ?, NULL)", (it, root + "/Fresh/01.mp3"))
|
||||
w.commit()
|
||||
sys.exit(subprocess.run([sys.executable, census, "audiobooks", "--root", root,
|
||||
"--db", db, "--out", out]).returncode)
|
||||
PY
|
||||
}
|
||||
|
||||
@test "the census exports a row the scanner committed but has not checkpointed" {
|
||||
DB="$BATS_TEST_TMPDIR/plex.db"
|
||||
db_with_part "$DB" "$BATS_TEST_TMPDIR/Audiobooks/Old/01.mp3" \
|
||||
"Old Narrator" "Old Book" 1990 1 "Old Chapter"
|
||||
run wal_fresh "$DB" "$BATS_TEST_TMPDIR/Audiobooks" "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 3 ] # header + both rows
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$BATS_TEST_TMPDIR/Audiobooks/Fresh/01\.mp3 Fresh Narrator Fresh Book 2026 1 Fresh Chapter$"
|
||||
}
|
||||
|
||||
# -------------------------------------------------------------------- the plan
|
||||
|
||||
@test "a matched chapter files under its narrator and its book, name untouched" {
|
||||
|
||||
+2
-72
@@ -195,7 +195,7 @@ setup() {
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 2" ]
|
||||
[[ "$output" == *"missing by extension: .log 1, .mkv 1"* ]]
|
||||
[ "${lines[4]}" = "missing by extension: .log 1, .mkv 1" ]
|
||||
}
|
||||
|
||||
@test "--ignore-ext keeps a file the server never indexes out of the verdict" {
|
||||
@@ -208,75 +208,5 @@ setup() {
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[1]}" = "landed: 1" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||
[[ "$output" == *"not indexed (extension ignored): 1"* ]]
|
||||
}
|
||||
|
||||
# wal_case <db> — a Jellyfin database whose NEW row lives only in the
|
||||
# write-ahead log: the server has committed the rename but not checkpointed it,
|
||||
# which is the normal state of a library a scan is writing. The holder
|
||||
# connection keeps the WAL from folding into the main file, and the verifier
|
||||
# runs as a real subprocess so it faces the same files Plex and Jellyfin leave.
|
||||
wal_case() {
|
||||
python3 - "$JFDB" "$JOURNAL" "$SHARE" "$VERIFY" <<'PY'
|
||||
import sqlite3, subprocess, sys
|
||||
db, journal, share, verify = sys.argv[1:5]
|
||||
w = sqlite3.connect(db)
|
||||
w.execute("pragma journal_mode=wal")
|
||||
w.execute("pragma wal_autocheckpoint=0")
|
||||
w.execute("insert into BaseItems (Path, Type) values ('/data/Movies/Old.mkv', 'x')")
|
||||
w.commit()
|
||||
w.execute("pragma wal_checkpoint(truncate)")
|
||||
holder = sqlite3.connect(db) # last-connection close would checkpoint
|
||||
w.execute("update BaseItems set Path='/data/Movies/Old/Old.mkv'")
|
||||
w.commit()
|
||||
r = subprocess.run([sys.executable, verify, "--journal", journal,
|
||||
"--db-kind", "jellyfin", "--db", db,
|
||||
"--share", share, "--container", "/data"],
|
||||
capture_output=True, text=True)
|
||||
print(r.stdout, end="")
|
||||
sys.exit(r.returncode)
|
||||
PY
|
||||
}
|
||||
|
||||
@test "a row the server committed but has not checkpointed is proof, not a miss" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
run wal_case
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[1]}" = "landed: 1" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||
}
|
||||
|
||||
# The DB alone cannot tell a lost file from a sidecar: both are "no row at the
|
||||
# destination". The tool therefore stats the two paths and separates the classes
|
||||
# the runbook names, so an operator never reads a `.pdf` as a lost film or,
|
||||
# worse, reads a real loss as a sidecar.
|
||||
@test "a file the database does not name that still sits at src is the pass's own damage" {
|
||||
mkdir -p "$SHARE/Movies"
|
||||
echo film > "$SHARE/Movies/Old.mkv"
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 1" ]
|
||||
[ "${lines[4]}" = " on disk at dst, no row names it: 0" ]
|
||||
[ "${lines[5]}" = " REAL — file still at src: 1" ]
|
||||
[ "${lines[6]}" = " gone — at neither path: 0" ]
|
||||
[[ "$output" == *"real: $SHARE/Movies/Old.mkv"* ]]
|
||||
}
|
||||
|
||||
@test "a file at neither path is gone, and a file at dst the server ignores is not a loss" {
|
||||
move_row "$SHARE/Movies/Gone.mkv" "$SHARE/Movies/Gone/Gone.mkv"
|
||||
mkdir -p "$SHARE/Movies/Sidecar"
|
||||
echo cover > "$SHARE/Movies/Sidecar/cover.jpg"
|
||||
move_row "$SHARE/Movies/Album/cover.jpg" "$SHARE/Movies/Sidecar/cover.jpg"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 2" ]
|
||||
[ "${lines[4]}" = " on disk at dst, no row names it: 1" ]
|
||||
[ "${lines[5]}" = " REAL — file still at src: 0" ]
|
||||
[ "${lines[6]}" = " gone — at neither path: 1" ]
|
||||
[[ "$output" == *"gone: $SHARE/Movies/Gone/Gone.mkv"* ]]
|
||||
[ "${lines[4]}" = "not indexed (extension ignored): 1" ]
|
||||
}
|
||||
Reference in new issue
Block a user