diff --git a/scripts/media-verify.py b/scripts/media-verify.py new file mode 100755 index 0000000..23cca6d --- /dev/null +++ b/scripts/media-verify.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Prove a relayout landed, using the media server's own tables. + +A scan answers `204` whether or not it found anything, and a rename that the +scanner has not followed looks exactly like a rename that never happened from +the outside. The only honest proof is the run's journal replayed against the +database the server actually serves from: every `move` row must have a row at +its destination, and NO row may still name its source — a source row is a scan +that has not caught up, and it is the one signature of a library pointing at +files that no longer exist. + +Two doors, both read-only, both without a lock while the server runs +(`mode=ro&immutable=1`; there is no `sqlite3` binary on these hosts): + + jellyfin `BaseItems.Path`. The container binds the share at `/data`, so the + database holds `/data//…` where the journal holds + `/mnt/entropy//…`. Translate, or every row reads as missing. + plex `media_parts.file`, joined up to `metadata_items`. Plex is bound at + the host path, so its rows match the journal verbatim. Filter + `deleted_at is null` on BOTH tables — a deleted scan leaves its parts + behind, and one of those rows is not proof anything landed. + +Neither server holds a row for everything the journal renames: a `.log`, `.cue` +or cover image has no row to land in, so its absence is not a fault. The report +therefore breaks the missing rows down by extension, and `--ignore-ext` names +the extensions this server does not index, which are then counted separately and +kept out of the verdict. + +Exit codes: 0 every move landed; 1 something is stale or missing (the details are +printed); 2 the journal or the database is not usable at all. +""" + +import argparse +import json +import os +import sqlite3 +import sys +from collections import Counter + +JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db" +PLEX_DB = ("/var/lib/plexmediaserver/Library/Application Support/Plex Media Server" + "/Plug-in Support/Databases/com.plexapp.plugins.library.db") + +# Per kind: the table whose columns are required, those columns, and the query +# that returns every path the server believes it has. +KINDS = { + "jellyfin": { + "table": "BaseItems", + "columns": {"Path"}, + "sql": "select Path from BaseItems", + "default_db": JF_DB, + }, + "plex": { + "table": "media_parts", + "columns": {"file", "media_item_id", "deleted_at"}, + "sql": ("select p.file from media_parts p" + " join media_items m on m.id = p.media_item_id" + " join metadata_items mi on mi.id = m.metadata_item_id" + " where p.deleted_at is null and mi.deleted_at is null"), + "default_db": PLEX_DB, + }, +} + +def die(msg): + print(f"media-verify: {msg}", file=sys.stderr) + sys.exit(2) + + +def read_moves(path): + """The (src, dst) pairs of a journal, in the order the run performed them.""" + moves = [] + try: + fh = open(path, encoding="utf-8") + except OSError as e: + die(f"cannot read the journal {path}: {e.strerror}") + with fh: + for n, line in enumerate(fh, 1): + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + except json.JSONDecodeError: + die(f"{path} line {n} is not a journal record") + if rec.get("op") == "move": + if not rec.get("src") or not rec.get("dst"): + die(f"{path} line {n} is a move without both src and dst") + moves.append((rec["src"], rec["dst"])) + return moves + + +def server_paths(db, kind): + """Every path the server holds, or a refusal that names what is missing.""" + spec = KINDS[kind] + try: + con = sqlite3.connect(f"file:{db}?mode=ro&immutable=1", uri=True) + except sqlite3.Error as e: + die(f"cannot open {db}: {e}") + with con: + have = {r[1] for r in con.execute(f"pragma table_info({spec['table']})")} + if not spec["columns"] <= have: + die(f"{db} is not a {kind} library database ({spec['table']} has no " + f"{','.join(sorted(spec['columns'] - have))})") + try: + return {r[0] for r in con.execute(spec["sql"]) if r[0]} + except sqlite3.Error as e: + die(f"{db} refused the {kind} query: {e}") + + +def to_host(path, container, share): + """The database's path in the journal's terms; identity when it is not a + container-side path.""" + if not container: + return path + prefix = container.rstrip("/") + "/" + if path.startswith(prefix): + return share.rstrip("/") + "/" + path[len(prefix):] + return path + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--journal", required=True, + help="the run's journal (scripts/media-apply.py writes it)") + ap.add_argument("--db-kind", choices=sorted(KINDS), required=True) + ap.add_argument("--db", default=None, help="the server's database on this host") + ap.add_argument("--share", default="/mnt/entropy", + help="the share every journal path lives under") + ap.add_argument("--container", default=None, + help="what the share is called inside the container; empty" + " compares verbatim (default /data for jellyfin, nothing" + " for plex)") + ap.add_argument("--show", type=int, default=20, + help="how many offenders to name before saying 'and N more'") + ap.add_argument("--ignore-ext", default="", + help="comma-separated extensions this server does not index," + " kept out of the verdict (e.g. log,cue,nfo,srt)") + args = ap.parse_args() + + db = args.db or KINDS[args.db_kind]["default_db"] + container = ("/data" if args.db_kind == "jellyfin" else "") \ + if args.container is None else args.container + ignore = set() + for ext in args.ignore_ext.split(","): + ext = ext.strip().lower() + if ext: + ignore.add(ext if ext.startswith(".") else "." + ext) + + moves = read_moves(args.journal) + stored = {to_host(p, container, args.share) for p in server_paths(db, args.db_kind)} + + landed, stale, missing, ignored = 0, [], [], 0 + for src, dst in moves: + if os.path.splitext(dst)[1].lower() in ignore: + ignored += 1 + continue + at_dst, at_src = dst in stored, src in stored + if at_dst and not at_src: + landed += 1 + continue + if at_src: + stale.append(src) + if not at_dst: + missing.append(dst) + + print(f"{args.db_kind}: {len(moves)} moves checked") + print(f"landed: {landed}") + print(f"still at src: {len(stale)}") + print(f"missing from the DB: {len(missing)}") + if ignored: + print(f"not indexed (extension ignored): {ignored}") + if missing: + exts = Counter(os.path.splitext(d)[1].lower() or "(no extension)" + for d in missing) + print("missing by extension: " + + ", ".join(f"{e} {n}" for e, n in sorted(exts.items()))) + for label, items in (("stale", stale), ("missing", missing)): + for path in items[:args.show]: + print(f" {label}: {path}") + if len(items) > args.show: + print(f" {label}: … and {len(items) - args.show} more") + return 1 if (stale or missing) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/serverAppConfigs-docs/Dankserver/Media Relayout.md b/serverAppConfigs-docs/Dankserver/Media Relayout.md index 8a441aa..9933e84 100644 --- a/serverAppConfigs-docs/Dankserver/Media Relayout.md +++ b/serverAppConfigs-docs/Dankserver/Media Relayout.md @@ -25,10 +25,11 @@ only reader is Plex — and nothing else in the repo touches a media file. | `scripts/media-shows.py` | no — TSV out | a plan that re-files the TV library itself | | `scripts/media-music.py` | no — TSV out | a plan that re-files the music library by its tags | | `scripts/media-audiobooks.py` | no — TSV out | a plan that re-files the audiobooks by the book Plex matched | +| `scripts/media-verify.py` | no — reads the servers' DBs read-only | the verdict that every journaled move landed | The seams are `tests/media_relayout.bats`, `tests/media_shows.bats`, -`tests/media_music.bats` and `tests/media_audiobooks.bats`; the movie and TV -target contracts are quoted from +`tests/media_music.bats`, `tests/media_audiobooks.bats` and `tests/media_verify.bats`; +the movie and TV target contracts are quoted from [jellyfin.org/docs/general/server/media/movies](https://jellyfin.org/docs/general/server/media/movies) in `media-plan.py`'s docstring. @@ -118,19 +119,18 @@ shred -u /tmp/jf.hdr A scan answers `204` whether or not it finds anything, so proof is the journal replayed against both servers' own tables: every `move` row has a row at its -`dst`, and **zero** rows still name its `src`. There is no `sqlite3` binary on -these hosts and none inside either container — read the DBs with the host -`python3` and `mode=ro&immutable=1`, which takes no lock while the server runs. +`dst`, and **zero** rows still name its `src`. `scripts/media-verify.py` is that +replay: it opens each database read-only (there is no `sqlite3` binary on these +hosts or inside either container) and exits non-zero while a row is stale. -| | path | path column | shape | -|---|---|---|---| -| Jellyfin | `/home/dank/Containers/jellyfin/config/data/jellyfin.db` | `BaseItems.Path` | one `BaseItems` table; `Type` holds the .NET class (`…Entities.TV.Episode`). There is no `MediaItems`/`TypedItems`. | -| Plex | `/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db` | `media_parts.file` | join `media_parts → media_items → metadata_items`; filter `deleted_at is null` on **both** or deleted scans pollute the count. | +```bash +scripts/media-verify.py --journal /mnt/entropy/media-relayout-music.jsonl \ + --db-kind jellyfin --ignore-ext jpg,jpeg,png,log,cue,toc,nfo,txt,m3u,accurip +``` -**Jellyfin stores container-side paths.** The share binds `/mnt/entropy → /data` -in that container, so a journal `src` of `/mnt/entropy/TVShows/…` is stored as -`/data/TVShows/…`. Plex is bound at the same path it has on the host, so its -rows match the journal verbatim. +Ignore only what a server cannot index. A missing `.flac` or `.mkv` is the +fault; a row still naming its `src` means the scan has not followed the rename +yet, so re-run the check after the scan, never before. Three doors that look right and are not: `GET /search?query=` matches metadata titles, never file paths; `?path=` on `/library/sections/N/all` makes Plex's diff --git a/tests/media_verify.bats b/tests/media_verify.bats new file mode 100755 index 0000000..1d9f8e1 --- /dev/null +++ b/tests/media_verify.bats @@ -0,0 +1,212 @@ +#!/usr/bin/env bats +# Seam: scripts/media-verify.py — the replay of a relayout journal against a +# media server's OWN database, which is the documented proof that a rename +# landed (a scan answers 204 whether or not it finds anything). +# +# Every test states what the database holds and asserts what the command prints +# and returns; nothing here reaches into the script's internals. The fixtures are +# minimal sqlite files with the columns the two servers really use, so the seam +# fails if the tool ever reads the wrong table, forgets Plex's `deleted_at` +# filter, or forgets that Jellyfin stores the container-side path (/data/…) while +# Plex stores the host path verbatim. + +load helpers + +VERIFY="$REPO_ROOT/scripts/media-verify.py" + +# jf_mk — the Jellyfin shape: one BaseItems table, Path is the column. +jf_mk() { + python3 - "$1" <<'PY' +import sqlite3, sys +con = sqlite3.connect(sys.argv[1]) +con.execute("create table BaseItems (Id INTEGER PRIMARY KEY, Path TEXT, Type TEXT," + " ParentId INTEGER, Name TEXT)") +con.commit() +PY +} + +# jf_row — one item the server believes it has. +jf_row() { + python3 - "$1" "$2" <<'PY' +import sqlite3, sys +con = sqlite3.connect(sys.argv[1]) +con.execute("insert into BaseItems (Path, Type) values (?, 'x')", (sys.argv[2],)) +con.commit() +PY +} + +# px_mk — the Plex shape: media_parts joined up to metadata_items. +px_mk() { + python3 - "$1" <<'PY' +import sqlite3, sys +con = sqlite3.connect(sys.argv[1]) +con.execute("create table metadata_items (id INTEGER PRIMARY KEY, title TEXT," + " deleted_at TEXT)") +con.execute("create table media_items (id INTEGER PRIMARY KEY, metadata_item_id" + " INTEGER, container TEXT)") +con.execute("create table media_parts (id INTEGER PRIMARY KEY, media_item_id" + " INTEGER, file TEXT, deleted_at TEXT)") +con.commit() +PY +} + +# px_row [DELETED_AT] — one part; a non-empty DELETED_AT is a row a +# deleted scan left behind, which must never count as proof. +px_row() { + python3 - "$1" "$2" "${3:-}" <<'PY' +import sqlite3, sys +db, path, gone = sys.argv[1], sys.argv[2], sys.argv[3] or None +con = sqlite3.connect(db) +cur = con.execute("insert into metadata_items (title, deleted_at) values ('t', ?)", + (gone,)) +mi = cur.lastrowid +cur = con.execute("insert into media_items (metadata_item_id, container)" + " values (?, 'mkv')", (mi,)) +con.execute("insert into media_parts (media_item_id, file, deleted_at)" + " values (?, ?, NULL)", (cur.lastrowid, path)) +con.commit() +PY +} + +# move_row — one move in the journal. +move_row() { + printf '{"op": "move", "src": "%s", "dst": "%s"}\n' "$1" "$2" >> "$JOURNAL" +} + +setup() { + SHARE="$BATS_TEST_TMPDIR/mnt/entropy" + JFDB="$BATS_TEST_TMPDIR/jellyfin.db" + PXDB="$BATS_TEST_TMPDIR/plex.db" + JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl" + : > "$JOURNAL" + jf_mk "$JFDB" + px_mk "$PXDB" +} + +@test "every move that the database holds at its destination is reported landed" { + move_row "$SHARE/Movies/Old Name (1999).mkv" "$SHARE/Movies/Old Name (1999)/Old Name (1999).mkv" + move_row "$SHARE/Movies/Two.mkv" "$SHARE/Movies/Two/Two.mkv" + # Jellyfin stores the container-side path: /data stands for the share. + jf_row "$JFDB" "/data/Movies/Old Name (1999)/Old Name (1999).mkv" + jf_row "$JFDB" "/data/Movies/Two/Two.mkv" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \ + --share "$SHARE" --container /data + [ "$status" -eq 0 ] + [ "${lines[0]}" = "jellyfin: 2 moves checked" ] + [ "${lines[1]}" = "landed: 2" ] + [ "${lines[2]}" = "still at src: 0" ] + [ "${lines[3]}" = "missing from the DB: 0" ] +} + +@test "a database that has not rescanned yet is refused, naming the stale rows" { + move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv" + jf_row "$JFDB" "/data/Movies/Old.mkv" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \ + --share "$SHARE" --container /data + [ "$status" -eq 1 ] + [ "${lines[1]}" = "landed: 0" ] + [ "${lines[2]}" = "still at src: 1" ] + [[ "$output" == *"$SHARE/Movies/Old.mkv"* ]] +} + +@test "a moved file no row of the database names is reported missing" { + move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \ + --share "$SHARE" --container /data + [ "$status" -eq 1 ] + [ "${lines[3]}" = "missing from the DB: 1" ] + [[ "$output" == *"$SHARE/Movies/Old/Old.mkv"* ]] +} + +@test "Plex stores the host path verbatim, so a row at the destination lands" { + move_row "$SHARE/TVShows/Show/Season 01/Show S01E01.mkv" \ + "$SHARE/TVShows/Show (2002)/Season 01/Show (2002) S01E01.mkv" + px_row "$PXDB" "$SHARE/TVShows/Show (2002)/Season 01/Show (2002) S01E01.mkv" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind plex --db "$PXDB" \ + --share "$SHARE" + [ "$status" -eq 0 ] + [ "${lines[0]}" = "plex: 1 moves checked" ] + [ "${lines[1]}" = "landed: 1" ] +} + +@test "a Plex row a deleted scan left behind is not proof a move landed" { + move_row "$SHARE/TVShows/Old S01E01.mkv" "$SHARE/TVShows/Show/Old S01E01.mkv" + px_row "$PXDB" "$SHARE/TVShows/Show/Old S01E01.mkv" "2026-10-30 00:00:00" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind plex --db "$PXDB" \ + --share "$SHARE" + [ "$status" -eq 1 ] + [ "${lines[1]}" = "landed: 0" ] + [ "${lines[3]}" = "missing from the DB: 1" ] +} + +@test "a database without the columns that kind needs is an error, not a zero" { + move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$PXDB" \ + --share "$SHARE" --container /data + [ "$status" -eq 2 ] + [[ "$output" == *"$PXDB"* ]] + [[ "$output" == *"BaseItems"* ]] +} + +@test "only move rows are counted, so folder operations cannot inflate the tally" { + move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv" + jf_row "$JFDB" "/data/Movies/Old/Old.mkv" + printf '{"op": "mkdir", "dst": "%s/Movies/Old"}\n' "$SHARE" >> "$JOURNAL" + printf '{"op": "rmdir", "path": "%s/Movies/Empty"}\n' "$SHARE" >> "$JOURNAL" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \ + --share "$SHARE" --container /data + [ "$status" -eq 0 ] + [ "${lines[0]}" = "jellyfin: 1 moves checked" ] +} + +@test "a journal with no moves is clean, not a failure" { + printf '{"op": "mkdir", "dst": "%s/Movies/Old"}\n' "$SHARE" >> "$JOURNAL" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \ + --share "$SHARE" --container /data + [ "$status" -eq 0 ] + [ "${lines[1]}" = "landed: 0" ] +} + +@test "the offender list stops at --show so a stale library stays readable" { + for n in 1 2 3 4; do + move_row "$SHARE/Movies/F$n.mkv" "$SHARE/Movies/F$n/F$n.mkv" + done + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \ + --share "$SHARE" --container /data --show 2 + [ "$status" -eq 1 ] + [ "${lines[3]}" = "missing from the DB: 4" ] + [[ "$output" == *"$SHARE/Movies/F1/F1.mkv"* ]] + [[ "$output" != *"$SHARE/Movies/F4/F4.mkv"* ]] +} + +@test "the missing rows are counted by extension, so a sidecar is not mistaken for a lost film" { + move_row "$SHARE/Music/Album/Mandatory Fun.log" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Mandatory Fun.log" + move_row "$SHARE/Music/Album/Film.mkv" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Film.mkv" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \ + --share "$SHARE" --container /data + [ "$status" -eq 1 ] + [ "${lines[3]}" = "missing from the DB: 2" ] + [ "${lines[4]}" = "missing by extension: .log 1, .mkv 1" ] +} + +@test "--ignore-ext keeps a file the server never indexes out of the verdict" { + move_row "$SHARE/Music/Album/Song.flac" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Song.flac" + jf_row "$JFDB" "/data/Music/Weird Al/Mandatory Fun (2014)/Song.flac" + move_row "$SHARE/Music/Album/Mandatory Fun.cue" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Mandatory Fun.cue" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \ + --share "$SHARE" --container /data --ignore-ext log,cue + [ "$status" -eq 0 ] + [ "${lines[1]}" = "landed: 1" ] + [ "${lines[3]}" = "missing from the DB: 0" ] + [ "${lines[4]}" = "not indexed (extension ignored): 1" ] +}