dank
272e06daf6
The movie library on the entropy volume is release-folded: the video inside
`Some.Movie.2019.1080p…-FOO/` is named after the release, and Jellyfin matches
a movie by folder and file name, so the noise cost posters, extras and versions.
Three scripts own the fix, and nothing else in the repo moves a media file.
* media-census.py the read-only inputs: a stat pass, an ffprobe header pass
(nice'd, so it is safe beside a live endpoint), and the
Plex metadata cache read that outranks any filename guess
* media-plan.py the Jellyfin-format planner: one folder per movie, the
video named exactly like the folder, extras in the
canonical subfolders, copies of one film as versions. It
opens no media file, so a whole-library plan costs seconds,
and it ends in gates — bad-names, occupied, double-claim,
left-unmoved, unplanned — that must all read zero before a
human points the executor at 24 TB
* media-apply.py the executor: rename(2) only, the intent fsynced to a
write-ahead journal BEFORE the move it authorises, so an
interrupted run is finishable and every run is undoable.
Refusals are all-or-nothing and it never overwrites a file
Laws the fixture pins (tests/media_relayout.bats, 16 tests, each proven red
against a reverted copy of the rule it guards):
* a Plex year of 0 means "indexed, never matched" — trusting it produced
folders "Title (0)": items with no year, no poster, no identity
* a Plex title carrying "/" is sanitised, or "V/H/S/2" became a five-level
tree neither matcher reads as a movie
* a folder inside a movie's folder is bonus territory even when its name is
misspelled ("Exrtas"), unless the file's own name carries another year
* matching words split on everything but a letter or digit — the earlier
tokenizer left "(1960)/extras" glued, so a film could not be found by the
folder holding it and one box set's featurettes drifted into a neighbour
* a name with no words in it is bonus material, never a movie folder named 1
Docs: the runbook page owns the procedure, the gates and the rollback; AGENTS.md
links it from the runbook index and the summary table.
322 lines
13 KiB
Python
Executable File
322 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""media-apply.py — the only thing that moves movie-library files.
|
|
|
|
Reads the plan written by scripts/media-plan.py (rows: `action src dst kind`,
|
|
plus `mkdir` rows) and performs it with rename(2) — bytes are never copied, so
|
|
an interrupted run has destroyed nothing. Every run is resumable and every run
|
|
is undoable: the journal is a write-ahead log, one JSON line per intent, fsynced
|
|
BEFORE the rename it authorises. That ordering is the whole design: a crash
|
|
leaves either an intent whose move happened or an intent whose move never ran,
|
|
and both are recognised on the next run. A file that moved with no journal line
|
|
cannot have been moved by this tool, and saying so is better than guessing.
|
|
|
|
Three subcommands:
|
|
|
|
apply --plan PLAN [--root ROOT] [--journal J] [--apply]
|
|
no --apply => report only and write NOTHING, not even folders
|
|
rollback --journal J [--root ROOT] [--apply]
|
|
back to the paths the census saw; folders this run created are
|
|
removed only when empty, never anything else
|
|
verify --plan PLAN --journal J [--root ROOT]
|
|
every planned file is at its target with the journalled byte count
|
|
|
|
Refusals (exit 2, before the first rename, with nothing touched): a target
|
|
already occupied by a foreign file, a source that has gone, a duplicate target,
|
|
a path outside --root, a journal that was already rolled back, a move that would
|
|
cross a filesystem (rename cannot do it and a silent copy at 24 TB would not).
|
|
|
|
--force relaxes only the guards that are opinion (journal already rolled back,
|
|
paths outside --root); it never allows overwriting a file.
|
|
|
|
The journal defaults to `media-relayout.jsonl` BESIDE the library root, never
|
|
inside it: a log inside the library is a file Jellyfin will index.
|
|
"""
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
DEFAULT_ROOT = os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies")
|
|
|
|
|
|
class Refuse(Exception):
|
|
"""A gate the operator has to see; every refusal leaves the library alone."""
|
|
|
|
|
|
def journal_default(root):
|
|
return os.path.join(os.path.dirname(os.path.abspath(root)), "media-relayout.jsonl")
|
|
|
|
|
|
class Journal:
|
|
"""Append-only JSONL of intents. Writes are fsynced before the tool acts."""
|
|
|
|
def __init__(self, path, writing):
|
|
self.path, self.writing = path, writing
|
|
self.records = []
|
|
if os.path.exists(path):
|
|
for line in open(path):
|
|
line = line.strip()
|
|
if line:
|
|
self.records.append(json.loads(line))
|
|
self._fh = None
|
|
|
|
def already(self, dst, src):
|
|
"""An intent for this exact (src, dst) was written by an earlier run."""
|
|
return any(r.get("op") == "move" and r.get("dst") == dst
|
|
and r.get("src") == src for r in self.records)
|
|
|
|
@property
|
|
def rolled_back(self):
|
|
return any(r.get("op") == "rollback" for r in self.records)
|
|
|
|
def record(self, **fields):
|
|
self.records.append(fields)
|
|
if not self.writing:
|
|
return
|
|
if self._fh is None:
|
|
self._fh = open(self.path, "a")
|
|
self._fh.write(json.dumps(fields) + "\n")
|
|
self._fh.flush()
|
|
os.fsync(self._fh.fileno())
|
|
|
|
def close(self):
|
|
if self._fh:
|
|
self._fh.close()
|
|
|
|
|
|
def read_plan(path, root, force):
|
|
"""Plan rows: (mkdirs, keeps, moves), header dropped, all paths under root."""
|
|
mkdirs, keeps, moves = [], [], []
|
|
with open(path) as fh:
|
|
header = fh.readline()
|
|
if not header.startswith("action\t"):
|
|
raise Refuse(f"{path} is not a plan (first column is "
|
|
f"{header.split(chr(9))[0]!r}, expected 'action') —"
|
|
" regenerate it with scripts/media-plan.py")
|
|
for line in fh:
|
|
f = line.rstrip("\n").split("\t")
|
|
if f[0] == "mkdir":
|
|
mkdirs.append(f[2])
|
|
elif f[0] == "keep":
|
|
keeps.append(f[1])
|
|
elif f[0] == "move":
|
|
moves.append((f[1], f[2], f[3] if len(f) > 3 else ""))
|
|
if not force:
|
|
for p in mkdirs + keeps + [p for m in moves for p in m[:2]]:
|
|
if p and not (p == root or p.startswith(root + os.sep)):
|
|
raise Refuse(f"{p} is outside --root {root}"
|
|
" (a plan built for another library?)")
|
|
return mkdirs, keeps, moves
|
|
|
|
|
|
def do_apply(args, root, journal, mkdirs, keeps, moves):
|
|
"""Preflight everything, then folders, then renames. Never a partial start."""
|
|
problems = []
|
|
for p in keeps:
|
|
if not os.path.exists(p):
|
|
problems.append(f"missing: keep row source is gone: {p}")
|
|
seen = {}
|
|
for src, dst, _kind in moves:
|
|
if src == dst:
|
|
problems.append(f"no-op: move row whose target is its source: {src}")
|
|
if dst in seen:
|
|
problems.append(f"double-claim: {dst} claimed by {seen[dst]} and {src}")
|
|
seen[dst] = src
|
|
if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
|
|
continue # this intent already ran
|
|
if not os.path.exists(src):
|
|
if os.path.exists(dst):
|
|
problems.append(f"missing: {src} is gone and {dst} carries no journal"
|
|
" line — it moved outside this tool, refusing to guess")
|
|
else:
|
|
problems.append(f"missing: {src} is gone and {dst} is not there either")
|
|
continue
|
|
if os.path.exists(dst):
|
|
problems.append(f"occupied: target already holds another file: {dst}")
|
|
if problems:
|
|
print(f"refusing: {len(problems)} problem(s), nothing moved")
|
|
for p in sorted(problems)[:20]:
|
|
print(" " + p)
|
|
if len(problems) > 20:
|
|
print(f" ... and {len(problems) - 20} more")
|
|
return 2
|
|
|
|
if not args.apply:
|
|
print(f"dry run: {len(moves)} moves, {len(mkdirs)} folders, {len(keeps)} files stay"
|
|
" — nothing written (pass --apply to run)")
|
|
for src, dst, _k in moves[:5]:
|
|
print(f" would move {src} -> {dst}")
|
|
return 0
|
|
|
|
made = {r.get("dst") for r in journal.records if r.get("op") == "mkdir"}
|
|
dirs_new = 0
|
|
for d in sorted(mkdirs, key=lambda q: q.count(os.sep)):
|
|
if d in made or os.path.isdir(d):
|
|
continue
|
|
journal.record(op="mkdir", dst=d)
|
|
os.makedirs(d, exist_ok=True)
|
|
dirs_new += 1
|
|
|
|
# same-filesystem gate: rename cannot cross a device, and a silent copy of a
|
|
# library this size is not an option anyone would choose.
|
|
cross = []
|
|
for src, dst, _k in moves:
|
|
if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
|
|
continue
|
|
parent = os.path.dirname(dst)
|
|
if os.path.exists(parent) and os.stat(parent).st_dev != os.stat(src).st_dev:
|
|
cross.append(dst)
|
|
if cross:
|
|
print(f"refusing: {len(cross)} moves would cross a filesystem"
|
|
f" (first: {cross[0]}) — this tool only renames")
|
|
return 2
|
|
|
|
moved = done = 0
|
|
for src, dst, _kind in moves:
|
|
if journal.already(dst, src):
|
|
if os.path.exists(dst) and not os.path.exists(src):
|
|
done += 1
|
|
continue
|
|
if os.path.exists(src) and not os.path.exists(dst):
|
|
os.rename(src, dst) # the intent ran, the move did not
|
|
moved += 1
|
|
continue
|
|
st = os.stat(src)
|
|
journal.record(op="move", src=src, dst=dst, bytes=st.st_size, mode=st.st_mode,
|
|
mtime=st.st_mtime, ino=st.st_ino, dev=st.st_dev)
|
|
os.rename(src, dst)
|
|
moved += 1
|
|
print(f"applied: {moved} moved, {done} already done in an earlier run,"
|
|
f" {dirs_new} folders created, {len(keeps)} files stayed put")
|
|
print(f"journal: {journal.path} — undo with: media-apply.py rollback"
|
|
f" --journal {journal.path} --root {root} --apply")
|
|
return 0
|
|
|
|
|
|
def do_verify(root, journal, moves, keeps):
|
|
"""Every planned file sits at its target carrying the journalled bytes."""
|
|
journalled = {(r["src"], r["dst"]): r for r in journal.records
|
|
if r.get("op") == "move"}
|
|
todo = moves or [(r["src"], r["dst"], "") for r in journal.records
|
|
if r.get("op") == "move"]
|
|
bad, missing, bytes_total = [], [], 0
|
|
for src, dst, _kind in todo:
|
|
rec = journalled.get((src, dst))
|
|
if not os.path.exists(dst):
|
|
missing.append(dst)
|
|
continue
|
|
size = os.path.getsize(dst)
|
|
bytes_total += size
|
|
if rec and rec.get("bytes") not in (None, size):
|
|
bad.append(f"{dst}: {size} bytes on disk, {rec['bytes']} at move time")
|
|
if os.path.exists(src) and src != dst:
|
|
bad.append(f"{src}: still in place as well as at {dst}")
|
|
missing += [p for p in keeps if not os.path.exists(p)]
|
|
if bad or missing:
|
|
print(f"verify FAILED: {len(missing)} absent, {len(bad)} wrong")
|
|
for line in (missing + bad)[:20]:
|
|
print(" " + line)
|
|
return 2
|
|
empty = [r["dst"] for r in journal.records
|
|
if r.get("op") == "mkdir" and os.path.isdir(r["dst"])
|
|
and not os.listdir(r["dst"])]
|
|
print(f"verified: {len(todo)} files at their targets"
|
|
f" ({bytes_total / 2**40:.2f} TiB), {len(keeps)} kept")
|
|
if empty:
|
|
print(f"note: {len(empty)} folders are empty (left in place — see the trash law)")
|
|
return 0
|
|
|
|
|
|
def do_rollback(args, root, journal):
|
|
"""Undo every journalled move, newest first, after a full preflight."""
|
|
moves = [r for r in journal.records if r.get("op") == "move"]
|
|
made = [r["dst"] for r in journal.records if r.get("op") == "mkdir"]
|
|
problems = []
|
|
for r in reversed(moves):
|
|
src, dst = r["src"], r["dst"]
|
|
if not args.force and not (src.startswith(root + os.sep)
|
|
or dst.startswith(root + os.sep)):
|
|
problems.append(f"outside --root: {dst}")
|
|
continue
|
|
if os.path.exists(dst) and os.path.exists(src):
|
|
problems.append(f"occupied: both paths exist, refusing to choose:"
|
|
f" {src} | {dst}")
|
|
elif os.path.exists(dst):
|
|
pass
|
|
elif os.path.exists(src):
|
|
continue # already undone
|
|
else:
|
|
problems.append(f"missing: {dst} is gone and {src} never came back")
|
|
if problems:
|
|
print(f"refusing: {len(problems)} problem(s), nothing moved back")
|
|
for p in problems[:20]:
|
|
print(" " + p)
|
|
return 2
|
|
if not args.apply:
|
|
print(f"dry run: would move {len(moves)} files back to their census paths"
|
|
" (pass --apply)")
|
|
return 0
|
|
undone = 0
|
|
for r in reversed(moves):
|
|
if os.path.exists(r["dst"]) and not os.path.exists(r["src"]):
|
|
os.makedirs(os.path.dirname(r["src"]), exist_ok=True)
|
|
os.rename(r["dst"], r["src"])
|
|
undone += 1
|
|
dirs_gone = 0
|
|
for d in sorted(set(made), key=lambda q: -q.count(os.sep)):
|
|
if d == root or not os.path.isdir(d) or os.listdir(d):
|
|
continue
|
|
try:
|
|
os.rmdir(d)
|
|
dirs_gone += 1
|
|
except OSError as e:
|
|
print(f"left in place: {d} ({e})")
|
|
journal.record(op="rollback", undone=undone, dirs_removed=dirs_gone,
|
|
t=datetime.datetime.now(datetime.timezone.utc).isoformat())
|
|
print(f"rolled back: {undone} files back where the census found them,"
|
|
f" {dirs_gone} empty folders removed")
|
|
return 0
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description="apply or undo a media relayout plan")
|
|
ap.add_argument("cmd", choices=["apply", "rollback", "verify"])
|
|
ap.add_argument("--plan")
|
|
ap.add_argument("--root", default=DEFAULT_ROOT)
|
|
ap.add_argument("--journal")
|
|
ap.add_argument("--apply", action="store_true", help="actually change the library")
|
|
ap.add_argument("--force", action="store_true", help="relax the opinion guards")
|
|
args = ap.parse_args()
|
|
|
|
root = os.path.abspath(args.root)
|
|
if not os.path.isdir(root):
|
|
raise Refuse(f"no library at {root}")
|
|
journal_path = args.journal or journal_default(root)
|
|
journal = Journal(journal_path, args.apply)
|
|
try:
|
|
if args.cmd in ("rollback", "verify") and not os.path.exists(journal_path):
|
|
raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
|
|
if args.cmd == "rollback" and not args.force and journal.rolled_back:
|
|
raise Refuse("this journal has already been rolled back"
|
|
" (--force to replay it)")
|
|
if args.cmd in ("apply", "verify") and not args.plan:
|
|
raise Refuse(f"{args.cmd} needs --plan")
|
|
mkdirs, keeps, moves = (read_plan(args.plan, root, args.force)
|
|
if args.plan else ([], [], []))
|
|
if args.cmd == "apply":
|
|
return do_apply(args, root, journal, mkdirs, keeps, moves)
|
|
if args.cmd == "verify":
|
|
return do_verify(root, journal, moves, keeps)
|
|
return do_rollback(args, root, journal)
|
|
finally:
|
|
journal.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.exit(main())
|
|
except Refuse as r:
|
|
print(f"refusing: {r}")
|
|
sys.exit(2)
|