Frank
a5a9c04da2
"No row at the destination" was one number, and it conflated three different things: a sidecar the server never indexes, a file a later pass renamed again, and a file that never left its source — the last one being actual damage. The runbook already named the four classes; the tool never computed them, so an operator reading `missing from the DB: 1851` had no way to tell a booklet.pdf from a lost album and might have run a rollback over a false alarm. `missing` rows are now stat'd on both paths and reported as `on disk at dst, no row names it` / `REAL — file still at src` / `gone — at neither path`, with offenders listed in that order. A row still naming `src` stays a separate line: that is the server's own stale pointer, a phantom, and it costs a re-scan rather than a restore. Two new tests (REAL, and gone-vs-sidecar) plus two existing layout assertions loosened from line positions to content. 14/14 in tests/media_verify.bats.
222 lines
9.3 KiB
Python
Executable File
222 lines
9.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Prove a relayout landed, using the media server's own tables.
|
|
|
|
A scan answers `204` whether or not it found anything, and a rename that the
|
|
scanner has not followed looks exactly like a rename that never happened from
|
|
the outside. The only honest proof is the run's journal replayed against the
|
|
database the server actually serves from: every `move` row must have a row at
|
|
its destination, and NO row may still name its source — a source row is a scan
|
|
that has not caught up, and it is the one signature of a library pointing at
|
|
files that no longer exist.
|
|
|
|
Two doors, both read-only, both without a lock while the server runs (there is
|
|
no `sqlite3` binary on these hosts). Open `mode=ro` PLAIN: both servers keep
|
|
their libraries in WAL mode, and `immutable=1` — which claims the file never
|
|
changes — makes sqlite skip the write-ahead log entirely, so every row the
|
|
server has committed but not yet checkpointed reads as MISSING, and a read
|
|
beside a running scanner can even fail as "database disk image is malformed".
|
|
`immutable=1` is only the fallback, for a database whose `-shm` cannot be read.
|
|
|
|
jellyfin `BaseItems.Path`. The container binds the share at `/data`, so the
|
|
database holds `/data/<library>/…` where the journal holds
|
|
`/mnt/entropy/<library>/…`. Translate, or every row reads as missing.
|
|
plex `media_parts.file`, joined up to `metadata_items`. Plex is bound at
|
|
the host path, so its rows match the journal verbatim. Filter
|
|
`deleted_at is null` on BOTH tables — a deleted scan leaves its parts
|
|
behind, and one of those rows is not proof anything landed.
|
|
|
|
Neither server holds a row for everything the journal renames: a `.log`, `.cue`
|
|
or cover image has no row to land in, so its absence is not a fault. The report
|
|
therefore breaks the missing rows down by extension, and `--ignore-ext` names the
|
|
extensions this server does not index, which are then counted separately and kept
|
|
out of the verdict. What is left is split by what the filesystem says, because
|
|
the database alone cannot: a file still at its `src` is `REAL` (this pass's own
|
|
damage), a file at neither path is `gone` (a later pass or the server renamed it),
|
|
and a file sitting at its `dst` with no row naming it is simply something the
|
|
server does not index. A row that still names `src` is none of these: it is the
|
|
server's own stale pointer — a phantom — and it costs a re-scan, never a restore.
|
|
|
|
Exit codes: 0 every move landed; 1 something is stale or missing (the details are
|
|
printed); 2 the journal or the database is not usable at all.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
from collections import Counter
|
|
|
|
JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db"
|
|
PLEX_DB = ("/var/lib/plexmediaserver/Library/Application Support/Plex Media Server"
|
|
"/Plug-in Support/Databases/com.plexapp.plugins.library.db")
|
|
|
|
# Per kind: the table whose columns are required, those columns, and the query
|
|
# that returns every path the server believes it has.
|
|
KINDS = {
|
|
"jellyfin": {
|
|
"table": "BaseItems",
|
|
"columns": {"Path"},
|
|
"sql": "select Path from BaseItems",
|
|
"default_db": JF_DB,
|
|
},
|
|
"plex": {
|
|
"table": "media_parts",
|
|
"columns": {"file", "media_item_id", "deleted_at"},
|
|
"sql": ("select p.file from media_parts p"
|
|
" join media_items m on m.id = p.media_item_id"
|
|
" join metadata_items mi on mi.id = m.metadata_item_id"
|
|
" where p.deleted_at is null and mi.deleted_at is null"),
|
|
"default_db": PLEX_DB,
|
|
},
|
|
}
|
|
|
|
def die(msg):
|
|
print(f"media-verify: {msg}", file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
|
|
def read_moves(path):
|
|
"""The (src, dst) pairs of a journal, in the order the run performed them."""
|
|
moves = []
|
|
try:
|
|
fh = open(path, encoding="utf-8")
|
|
except OSError as e:
|
|
die(f"cannot read the journal {path}: {e.strerror}")
|
|
with fh:
|
|
for n, line in enumerate(fh, 1):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
rec = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
die(f"{path} line {n} is not a journal record")
|
|
if rec.get("op") == "move":
|
|
if not rec.get("src") or not rec.get("dst"):
|
|
die(f"{path} line {n} is a move without both src and dst")
|
|
moves.append((rec["src"], rec["dst"]))
|
|
return moves
|
|
|
|
|
|
def server_paths(db, kind):
|
|
"""Every path the server holds, or a refusal that names what is missing."""
|
|
spec = KINDS[kind]
|
|
con = None
|
|
err = None
|
|
# Plain ro reads the WAL the server commits through; immutable is the last
|
|
# resort, because it cannot see a row the server has not checkpointed.
|
|
for opt in ("mode=ro", "mode=ro&immutable=1"):
|
|
try:
|
|
con = sqlite3.connect(f"file:{db}?{opt}", uri=True)
|
|
break
|
|
except sqlite3.Error as e:
|
|
err = e
|
|
if con is None:
|
|
die(f"cannot open {db}: {err}")
|
|
with con:
|
|
have = {r[1] for r in con.execute(f"pragma table_info({spec['table']})")}
|
|
if not spec["columns"] <= have:
|
|
die(f"{db} is not a {kind} library database ({spec['table']} has no "
|
|
f"{','.join(sorted(spec['columns'] - have))})")
|
|
try:
|
|
return {r[0] for r in con.execute(spec["sql"]) if r[0]}
|
|
except sqlite3.Error as e:
|
|
die(f"{db} refused the {kind} query: {e}")
|
|
|
|
|
|
def to_host(path, container, share):
|
|
"""The database's path in the journal's terms; identity when it is not a
|
|
container-side path."""
|
|
if not container:
|
|
return path
|
|
prefix = container.rstrip("/") + "/"
|
|
if path.startswith(prefix):
|
|
return share.rstrip("/") + "/" + path[len(prefix):]
|
|
return path
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--journal", required=True,
|
|
help="the run's journal (scripts/media-apply.py writes it)")
|
|
ap.add_argument("--db-kind", choices=sorted(KINDS), required=True)
|
|
ap.add_argument("--db", default=None, help="the server's database on this host")
|
|
ap.add_argument("--share", default="/mnt/entropy",
|
|
help="the share every journal path lives under")
|
|
ap.add_argument("--container", default=None,
|
|
help="what the share is called inside the container; empty"
|
|
" compares verbatim (default /data for jellyfin, nothing"
|
|
" for plex)")
|
|
ap.add_argument("--show", type=int, default=20,
|
|
help="how many offenders to name before saying 'and N more'")
|
|
ap.add_argument("--ignore-ext", default="",
|
|
help="comma-separated extensions this server does not index,"
|
|
" kept out of the verdict (e.g. log,cue,nfo,srt)")
|
|
args = ap.parse_args()
|
|
|
|
db = args.db or KINDS[args.db_kind]["default_db"]
|
|
container = ("/data" if args.db_kind == "jellyfin" else "") \
|
|
if args.container is None else args.container
|
|
ignore = set()
|
|
for ext in args.ignore_ext.split(","):
|
|
ext = ext.strip().lower()
|
|
if ext:
|
|
ignore.add(ext if ext.startswith(".") else "." + ext)
|
|
|
|
moves = read_moves(args.journal)
|
|
stored = {to_host(p, container, args.share) for p in server_paths(db, args.db_kind)}
|
|
|
|
landed, stale, missing, ignored = 0, [], [], 0
|
|
for src, dst in moves:
|
|
if os.path.splitext(dst)[1].lower() in ignore:
|
|
ignored += 1
|
|
continue
|
|
at_dst, at_src = dst in stored, src in stored
|
|
if at_dst and not at_src:
|
|
landed += 1
|
|
continue
|
|
if at_src:
|
|
stale.append(src)
|
|
if not at_dst:
|
|
missing.append((src, dst))
|
|
|
|
# The database cannot tell a lost file from one it never indexes: both read
|
|
# as "no row at the destination". Stat both paths, so the report separates
|
|
# the classes the runbook names — `REAL` is the only one that is this pass's
|
|
# own damage, and `gone` is a later pass or the server's own rename.
|
|
on_disk, real, gone = [], [], []
|
|
for src, dst in missing:
|
|
if os.path.exists(dst):
|
|
on_disk.append(dst)
|
|
elif os.path.exists(src):
|
|
real.append(src)
|
|
else:
|
|
gone.append(dst)
|
|
|
|
print(f"{args.db_kind}: {len(moves)} moves checked")
|
|
print(f"landed: {landed}")
|
|
print(f"still at src: {len(stale)}")
|
|
print(f"missing from the DB: {len(missing)}")
|
|
print(f" on disk at dst, no row names it: {len(on_disk)}")
|
|
print(f" REAL — file still at src: {len(real)}")
|
|
print(f" gone — at neither path: {len(gone)}")
|
|
if ignored:
|
|
print(f"not indexed (extension ignored): {ignored}")
|
|
if missing:
|
|
exts = Counter(os.path.splitext(d)[1].lower() or "(no extension)"
|
|
for _, d in missing)
|
|
print("missing by extension: "
|
|
+ ", ".join(f"{e} {n}" for e, n in sorted(exts.items())))
|
|
for label, items in (("real", real), ("gone", gone), ("missing", on_disk),
|
|
("stale", stale)):
|
|
for path in items[:args.show]:
|
|
print(f" {label}: {path}")
|
|
if len(items) > args.show:
|
|
print(f" {label}: … and {len(items) - args.show} more")
|
|
return 1 if (stale or missing) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|