diff --git a/scripts/media-census.py b/scripts/media-census.py index fd9c5b3..a03251f 100755 --- a/scripts/media-census.py +++ b/scripts/media-census.py @@ -128,6 +128,26 @@ where p.file like ? and p.deleted_at is null and mi.deleted_at is null and al.deleted_at is null """ +def open_plex_ro(db): + """Plex's library, read the way Plex writes it. + + `immutable=1` tells sqlite the file never changes, so it skips the + write-ahead log: every row a scan has committed but not yet checkpointed + is invisible, and reading a live database that way can fail outright as + "database disk image is malformed". Plain ro reads the WAL through its + -shm; immutable is only the fallback for a database whose -shm cannot be + read. The same law scripts/media-verify.py states. + """ + last = None + for opt in ("mode=ro", "mode=ro&immutable=1"): + try: + return sqlite3.connect(f"file:{db}?{opt}", uri=True) + except sqlite3.Error as e: + last = e + sys.exit(f"cannot open the Plex database read-only ({db}): {last}\n" + "Plex runs rootless here: run this as the container's owner, never " + "with sudo — rootless storage is per-user, so root sees nothing.") + def enumerate_lib(root, out): """One row per real file under root: size, extension, path.""" @@ -227,13 +247,7 @@ def probe(enum, out, jobs): def plex(root, db, out): """Plex's matched titles for files under root, from its DB, read-only.""" - uri = f"file:{db}?mode=ro&immutable=1" - try: - con = sqlite3.connect(uri, uri=True) - except sqlite3.Error as e: - sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n" - "Plex runs rootless here: run this as the container's owner, never " - "with sudo — rootless storage is per-user, so root sees nothing.") + con = open_plex_ro(db) with open(out, "w", newline="") as fh: w = csv.writer(fh, delimiter="\t", lineterminator="\n") w.writerow(PLEX_COLS) @@ -405,13 +419,7 @@ def audiobooks(root, db, out): at least one un-indexed chapter, so the relayout that reads this file must not renumber anything from it. """ - uri = f"file:{db}?mode=ro&immutable=1" - try: - con = sqlite3.connect(uri, uri=True) - except sqlite3.Error as e: - sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n" - "Plex runs rootless here: run this as the container's owner, never " - "with sudo — rootless storage is per-user, so root sees nothing.") + con = open_plex_ro(db) tables = {r[0] for r in con.execute( "select name from sqlite_master where type='table'")} missing = {"media_parts", "media_items", "metadata_items"} - tables diff --git a/scripts/media-verify.py b/scripts/media-verify.py index 23cca6d..346177c 100755 --- a/scripts/media-verify.py +++ b/scripts/media-verify.py @@ -9,8 +9,13 @@ its destination, and NO row may still name its source — a source row is a scan that has not caught up, and it is the one signature of a library pointing at files that no longer exist. -Two doors, both read-only, both without a lock while the server runs -(`mode=ro&immutable=1`; there is no `sqlite3` binary on these hosts): +Two doors, both read-only, both without a lock while the server runs (there is +no `sqlite3` binary on these hosts). Open `mode=ro` PLAIN: both servers keep +their libraries in WAL mode, and `immutable=1` — which claims the file never +changes — makes sqlite skip the write-ahead log entirely, so every row the +server has committed but not yet checkpointed reads as MISSING, and a read +beside a running scanner can even fail as "database disk image is malformed". +`immutable=1` is only the fallback, for a database whose `-shm` cannot be read. jellyfin `BaseItems.Path`. The container binds the share at `/data`, so the database holds `/data//…` where the journal holds @@ -22,9 +27,14 @@ Two doors, both read-only, both without a lock while the server runs Neither server holds a row for everything the journal renames: a `.log`, `.cue` or cover image has no row to land in, so its absence is not a fault. The report -therefore breaks the missing rows down by extension, and `--ignore-ext` names -the extensions this server does not index, which are then counted separately and -kept out of the verdict. +therefore breaks the missing rows down by extension, and `--ignore-ext` names the +extensions this server does not index, which are then counted separately and kept +out of the verdict. What is left is split by what the filesystem says, because +the database alone cannot: a file still at its `src` is `REAL` (this pass's own +damage), a file at neither path is `gone` (a later pass or the server renamed it), +and a file sitting at its `dst` with no row naming it is simply something the +server does not index. A row that still names `src` is none of these: it is the +server's own stale pointer — a phantom — and it costs a re-scan, never a restore. Exit codes: 0 every move landed; 1 something is stale or missing (the details are printed); 2 the journal or the database is not usable at all. @@ -92,10 +102,18 @@ def read_moves(path): def server_paths(db, kind): """Every path the server holds, or a refusal that names what is missing.""" spec = KINDS[kind] - try: - con = sqlite3.connect(f"file:{db}?mode=ro&immutable=1", uri=True) - except sqlite3.Error as e: - die(f"cannot open {db}: {e}") + con = None + err = None + # Plain ro reads the WAL the server commits through; immutable is the last + # resort, because it cannot see a row the server has not checkpointed. + for opt in ("mode=ro", "mode=ro&immutable=1"): + try: + con = sqlite3.connect(f"file:{db}?{opt}", uri=True) + break + except sqlite3.Error as e: + err = e + if con is None: + die(f"cannot open {db}: {err}") with con: have = {r[1] for r in con.execute(f"pragma table_info({spec['table']})")} if not spec["columns"] <= have: @@ -161,20 +179,37 @@ def main(): if at_src: stale.append(src) if not at_dst: - missing.append(dst) + missing.append((src, dst)) + + # The database cannot tell a lost file from one it never indexes: both read + # as "no row at the destination". Stat both paths, so the report separates + # the classes the runbook names — `REAL` is the only one that is this pass's + # own damage, and `gone` is a later pass or the server's own rename. + on_disk, real, gone = [], [], [] + for src, dst in missing: + if os.path.exists(dst): + on_disk.append(dst) + elif os.path.exists(src): + real.append(src) + else: + gone.append(dst) print(f"{args.db_kind}: {len(moves)} moves checked") print(f"landed: {landed}") print(f"still at src: {len(stale)}") print(f"missing from the DB: {len(missing)}") + print(f" on disk at dst, no row names it: {len(on_disk)}") + print(f" REAL — file still at src: {len(real)}") + print(f" gone — at neither path: {len(gone)}") if ignored: print(f"not indexed (extension ignored): {ignored}") if missing: exts = Counter(os.path.splitext(d)[1].lower() or "(no extension)" - for d in missing) + for _, d in missing) print("missing by extension: " + ", ".join(f"{e} {n}" for e, n in sorted(exts.items()))) - for label, items in (("stale", stale), ("missing", missing)): + for label, items in (("real", real), ("gone", gone), ("missing", on_disk), + ("stale", stale)): for path in items[:args.show]: print(f" {label}: {path}") if len(items) > args.show: diff --git a/tests/media_audiobooks.bats b/tests/media_audiobooks.bats index 6f50db0..3214065 100755 --- a/tests/media_audiobooks.bats +++ b/tests/media_audiobooks.bats @@ -228,6 +228,51 @@ PY [ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 1 ] # header only } +# wal_fresh DB ROOT OUT — a row Plex matched during a scan and committed, but +# has not checkpointed: the normal state of a library being walked. The holder +# connection stops the WAL folding into the main file when the writer closes, +# and the census runs as a real subprocess against those same files. +wal_fresh() { + local db="$1" root="$2" out="$3" + python3 - "$db" "$root" "$out" "$CENSUS" <<'PY' +import sqlite3, subprocess, sys +db, root, out, census = sys.argv[1:5] +w = sqlite3.connect(db) +w.execute("pragma journal_mode=wal") +w.execute("pragma wal_autocheckpoint=0") +w.commit() +w.execute("pragma wal_checkpoint(truncate)") # the rows so far are on disk +holder = sqlite3.connect(db) # last close would checkpoint +ar = w.execute('insert into metadata_items (parent_id, metadata_type, title,' + ' year, "index", deleted_at) VALUES (NULL, 8, ?, NULL, NULL, NULL)', + ("Fresh Narrator",)).lastrowid +al = w.execute('insert into metadata_items (parent_id, metadata_type, title,' + ' year, "index", deleted_at) VALUES (?, 9, ?, ?, NULL, NULL)', + (ar, "Fresh Book", 2026)).lastrowid +mi = w.execute('insert into metadata_items (parent_id, metadata_type, title,' + ' year, "index", deleted_at) VALUES (?, 10, ?, NULL, ?, NULL)', + (al, "Fresh Chapter", 1)).lastrowid +it = w.execute("insert into media_items (metadata_item_id, deleted_at)" + " VALUES (?, NULL)", (mi,)).lastrowid +w.execute("insert into media_parts (media_item_id, file, deleted_at)" + " VALUES (?, ?, NULL)", (it, root + "/Fresh/01.mp3")) +w.commit() +sys.exit(subprocess.run([sys.executable, census, "audiobooks", "--root", root, + "--db", db, "--out", out]).returncode) +PY +} + +@test "the census exports a row the scanner committed but has not checkpointed" { + DB="$BATS_TEST_TMPDIR/plex.db" + db_with_part "$DB" "$BATS_TEST_TMPDIR/Audiobooks/Old/01.mp3" \ + "Old Narrator" "Old Book" 1990 1 "Old Chapter" + run wal_fresh "$DB" "$BATS_TEST_TMPDIR/Audiobooks" "$BATS_TEST_TMPDIR/exp.tsv" + [ "$status" -eq 0 ] + [ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 3 ] # header + both rows + tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \ + | grep -q "^$BATS_TEST_TMPDIR/Audiobooks/Fresh/01\.mp3 Fresh Narrator Fresh Book 2026 1 Fresh Chapter$" +} + # -------------------------------------------------------------------- the plan @test "a matched chapter files under its narrator and its book, name untouched" { diff --git a/tests/media_verify.bats b/tests/media_verify.bats index 1d9f8e1..340e32a 100755 --- a/tests/media_verify.bats +++ b/tests/media_verify.bats @@ -195,7 +195,7 @@ setup() { --share "$SHARE" --container /data [ "$status" -eq 1 ] [ "${lines[3]}" = "missing from the DB: 2" ] - [ "${lines[4]}" = "missing by extension: .log 1, .mkv 1" ] + [[ "$output" == *"missing by extension: .log 1, .mkv 1"* ]] } @test "--ignore-ext keeps a file the server never indexes out of the verdict" { @@ -208,5 +208,75 @@ setup() { [ "$status" -eq 0 ] [ "${lines[1]}" = "landed: 1" ] [ "${lines[3]}" = "missing from the DB: 0" ] - [ "${lines[4]}" = "not indexed (extension ignored): 1" ] + [[ "$output" == *"not indexed (extension ignored): 1"* ]] +} + +# wal_case — a Jellyfin database whose NEW row lives only in the +# write-ahead log: the server has committed the rename but not checkpointed it, +# which is the normal state of a library a scan is writing. The holder +# connection keeps the WAL from folding into the main file, and the verifier +# runs as a real subprocess so it faces the same files Plex and Jellyfin leave. +wal_case() { + python3 - "$JFDB" "$JOURNAL" "$SHARE" "$VERIFY" <<'PY' +import sqlite3, subprocess, sys +db, journal, share, verify = sys.argv[1:5] +w = sqlite3.connect(db) +w.execute("pragma journal_mode=wal") +w.execute("pragma wal_autocheckpoint=0") +w.execute("insert into BaseItems (Path, Type) values ('/data/Movies/Old.mkv', 'x')") +w.commit() +w.execute("pragma wal_checkpoint(truncate)") +holder = sqlite3.connect(db) # last-connection close would checkpoint +w.execute("update BaseItems set Path='/data/Movies/Old/Old.mkv'") +w.commit() +r = subprocess.run([sys.executable, verify, "--journal", journal, + "--db-kind", "jellyfin", "--db", db, + "--share", share, "--container", "/data"], + capture_output=True, text=True) +print(r.stdout, end="") +sys.exit(r.returncode) +PY +} + +@test "a row the server committed but has not checkpointed is proof, not a miss" { + move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv" + run wal_case + [ "$status" -eq 0 ] + [ "${lines[1]}" = "landed: 1" ] + [ "${lines[3]}" = "missing from the DB: 0" ] +} + +# The DB alone cannot tell a lost file from a sidecar: both are "no row at the +# destination". The tool therefore stats the two paths and separates the classes +# the runbook names, so an operator never reads a `.pdf` as a lost film or, +# worse, reads a real loss as a sidecar. +@test "a file the database does not name that still sits at src is the pass's own damage" { + mkdir -p "$SHARE/Movies" + echo film > "$SHARE/Movies/Old.mkv" + move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \ + --share "$SHARE" --container /data + [ "$status" -eq 1 ] + [ "${lines[3]}" = "missing from the DB: 1" ] + [ "${lines[4]}" = " on disk at dst, no row names it: 0" ] + [ "${lines[5]}" = " REAL — file still at src: 1" ] + [ "${lines[6]}" = " gone — at neither path: 0" ] + [[ "$output" == *"real: $SHARE/Movies/Old.mkv"* ]] +} + +@test "a file at neither path is gone, and a file at dst the server ignores is not a loss" { + move_row "$SHARE/Movies/Gone.mkv" "$SHARE/Movies/Gone/Gone.mkv" + mkdir -p "$SHARE/Movies/Sidecar" + echo cover > "$SHARE/Movies/Sidecar/cover.jpg" + move_row "$SHARE/Movies/Album/cover.jpg" "$SHARE/Movies/Sidecar/cover.jpg" + + run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \ + --share "$SHARE" --container /data + [ "$status" -eq 1 ] + [ "${lines[3]}" = "missing from the DB: 2" ] + [ "${lines[4]}" = " on disk at dst, no row names it: 1" ] + [ "${lines[5]}" = " REAL — file still at src: 0" ] + [ "${lines[6]}" = " gone — at neither path: 1" ] + [[ "$output" == *"gone: $SHARE/Movies/Gone/Gone.mkv"* ]] }