Frank
a803aa046a
The live sweep removed 2271 husks and left 146 standing, every one a release folder blocked by a junk subfolder (Screens/, Sample/, a BDMV leaf) that no file ever moved out of. Eligibility was per-folder, so the blocker was never a target and its parent could never be removed. Proof now carries downward: inside a folder the journal shows emptied, every empty directory qualifies. Nothing under such a folder can hold data, and every removal is still journalled as rmdir and undone by rollback. The new test is red against the previous revision, green now. The live sweep touched no file: 8193 before, 8193 after.
422 lines
18 KiB
Python
Executable File
422 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""media-apply.py — the only thing that moves movie-library files.
|
|
|
|
Reads the plan written by scripts/media-plan.py (rows: `action src dst kind`,
|
|
plus `mkdir` rows) and performs it with rename(2) — bytes are never copied, so
|
|
an interrupted run has destroyed nothing. Every run is resumable and every run
|
|
is undoable: the journal is a write-ahead log, one JSON line per intent, fsynced
|
|
BEFORE the rename it authorises. That ordering is the whole design: a crash
|
|
leaves either an intent whose move happened or an intent whose move never ran,
|
|
and both are recognised on the next run. A file that moved with no journal line
|
|
cannot have been moved by this tool, and saying so is better than guessing.
|
|
|
|
Four subcommands:
|
|
|
|
apply --plan PLAN [--root ROOT] [--journal J] [--apply]
|
|
no --apply => report only and write NOTHING, not even folders
|
|
rollback --journal J [--root ROOT] [--apply]
|
|
back to the paths the census saw; folders this run created are
|
|
removed only when empty, never anything else
|
|
verify --plan PLAN --journal J [--root ROOT]
|
|
every planned file is at its target with the journalled byte count
|
|
sweep --journal J [--root ROOT] [--apply]
|
|
removes the folders THIS RUN emptied — the journal has to show a
|
|
file that moved out of one, so a folder someone made by hand can
|
|
never be removed by this tool, and rollback puts them back
|
|
|
|
Refusals (exit 2, before the first rename, with nothing touched): a target
|
|
already occupied by a foreign file, a source that has gone, a duplicate target,
|
|
a path outside --root, a journal that was already rolled back, a move that would
|
|
cross a filesystem (rename cannot do it and a silent copy at 24 TB would not).
|
|
|
|
--force relaxes only the guards that are opinion (journal already rolled back,
|
|
paths outside --root); it never allows overwriting a file.
|
|
|
|
The journal defaults to `media-relayout.jsonl` BESIDE the library root, never
|
|
inside it: a log inside the library is a file Jellyfin will index.
|
|
"""
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
DEFAULT_ROOT = os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies")
|
|
|
|
|
|
class Refuse(Exception):
|
|
"""A gate the operator has to see; every refusal leaves the library alone."""
|
|
|
|
|
|
def journal_default(root):
|
|
return os.path.join(os.path.dirname(os.path.abspath(root)), "media-relayout.jsonl")
|
|
|
|
|
|
class Journal:
|
|
"""Append-only JSONL of intents. Writes are fsynced before the tool acts."""
|
|
|
|
def __init__(self, path, writing):
|
|
self.path, self.writing = path, writing
|
|
self.records = []
|
|
if os.path.exists(path):
|
|
for line in open(path):
|
|
line = line.strip()
|
|
if line:
|
|
self.records.append(json.loads(line))
|
|
self._fh = None
|
|
|
|
def already(self, dst, src):
|
|
"""An intent for this exact (src, dst) was written by an earlier run."""
|
|
return any(r.get("op") == "move" and r.get("dst") == dst
|
|
and r.get("src") == src for r in self.records)
|
|
|
|
@property
|
|
def rolled_back(self):
|
|
return any(r.get("op") == "rollback" for r in self.records)
|
|
|
|
def record(self, **fields):
|
|
self.records.append(fields)
|
|
if not self.writing:
|
|
return
|
|
if self._fh is None:
|
|
self._fh = open(self.path, "a")
|
|
self._fh.write(json.dumps(fields) + "\n")
|
|
self._fh.flush()
|
|
os.fsync(self._fh.fileno())
|
|
|
|
def close(self):
|
|
if self._fh:
|
|
self._fh.close()
|
|
|
|
|
|
def read_plan(path, root, force):
|
|
"""Plan rows: (mkdirs, keeps, moves), header dropped, all paths under root."""
|
|
mkdirs, keeps, moves = [], [], []
|
|
with open(path) as fh:
|
|
header = fh.readline()
|
|
if not header.startswith("action\t"):
|
|
raise Refuse(f"{path} is not a plan (first column is "
|
|
f"{header.split(chr(9))[0]!r}, expected 'action') —"
|
|
" regenerate it with scripts/media-plan.py")
|
|
for line in fh:
|
|
f = line.rstrip("\n").split("\t")
|
|
if f[0] == "mkdir":
|
|
mkdirs.append(f[2])
|
|
elif f[0] == "keep":
|
|
keeps.append(f[1])
|
|
elif f[0] == "move":
|
|
moves.append((f[1], f[2], f[3] if len(f) > 3 else ""))
|
|
if not force:
|
|
for p in mkdirs + keeps + [p for m in moves for p in m[:2]]:
|
|
if p and not (p == root or p.startswith(root + os.sep)):
|
|
raise Refuse(f"{p} is outside --root {root}"
|
|
" (a plan built for another library?)")
|
|
return mkdirs, keeps, moves
|
|
|
|
|
|
def do_apply(args, root, journal, mkdirs, keeps, moves):
|
|
"""Preflight everything, then folders, then renames. Never a partial start."""
|
|
problems = []
|
|
for p in keeps:
|
|
if not os.path.exists(p):
|
|
problems.append(f"missing: keep row source is gone: {p}")
|
|
seen = {}
|
|
for src, dst, _kind in moves:
|
|
if src == dst:
|
|
problems.append(f"no-op: move row whose target is its source: {src}")
|
|
if dst in seen:
|
|
problems.append(f"double-claim: {dst} claimed by {seen[dst]} and {src}")
|
|
seen[dst] = src
|
|
if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
|
|
continue # this intent already ran
|
|
if not os.path.exists(src):
|
|
if os.path.exists(dst):
|
|
problems.append(f"missing: {src} is gone and {dst} carries no journal"
|
|
" line — it moved outside this tool, refusing to guess")
|
|
else:
|
|
problems.append(f"missing: {src} is gone and {dst} is not there either")
|
|
continue
|
|
if os.path.exists(dst):
|
|
problems.append(f"occupied: target already holds another file: {dst}")
|
|
if problems:
|
|
print(f"refusing: {len(problems)} problem(s), nothing moved")
|
|
for p in sorted(problems)[:20]:
|
|
print(" " + p)
|
|
if len(problems) > 20:
|
|
print(f" ... and {len(problems) - 20} more")
|
|
return 2
|
|
|
|
if not args.apply:
|
|
print(f"dry run: {len(moves)} moves, {len(mkdirs)} folders, {len(keeps)} files stay"
|
|
" — nothing written (pass --apply to run)")
|
|
for src, dst, _k in moves[:5]:
|
|
print(f" would move {src} -> {dst}")
|
|
return 0
|
|
|
|
made = {r.get("dst") for r in journal.records if r.get("op") == "mkdir"}
|
|
dirs_new = 0
|
|
for d in sorted(mkdirs, key=lambda q: q.count(os.sep)):
|
|
if d in made or os.path.isdir(d):
|
|
continue
|
|
journal.record(op="mkdir", dst=d)
|
|
os.makedirs(d, exist_ok=True)
|
|
dirs_new += 1
|
|
|
|
# same-filesystem gate: rename cannot cross a device, and a silent copy of a
|
|
# library this size is not an option anyone would choose.
|
|
cross = []
|
|
for src, dst, _k in moves:
|
|
if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
|
|
continue
|
|
parent = os.path.dirname(dst)
|
|
if os.path.exists(parent) and os.stat(parent).st_dev != os.stat(src).st_dev:
|
|
cross.append(dst)
|
|
if cross:
|
|
print(f"refusing: {len(cross)} moves would cross a filesystem"
|
|
f" (first: {cross[0]}) — this tool only renames")
|
|
return 2
|
|
|
|
moved = done = 0
|
|
for src, dst, _kind in moves:
|
|
if journal.already(dst, src):
|
|
if os.path.exists(dst) and not os.path.exists(src):
|
|
done += 1
|
|
continue
|
|
if os.path.exists(src) and not os.path.exists(dst):
|
|
os.rename(src, dst) # the intent ran, the move did not
|
|
moved += 1
|
|
continue
|
|
st = os.stat(src)
|
|
journal.record(op="move", src=src, dst=dst, bytes=st.st_size, mode=st.st_mode,
|
|
mtime=st.st_mtime, ino=st.st_ino, dev=st.st_dev)
|
|
os.rename(src, dst)
|
|
moved += 1
|
|
print(f"applied: {moved} moved, {done} already done in an earlier run,"
|
|
f" {dirs_new} folders created, {len(keeps)} files stayed put")
|
|
print(f"journal: {journal.path} — undo with: media-apply.py rollback"
|
|
f" --journal {journal.path} --root {root} --apply")
|
|
return 0
|
|
|
|
|
|
def do_verify(root, journal, moves, keeps):
|
|
"""Every planned file sits at its target carrying the journalled bytes."""
|
|
journalled = {(r["src"], r["dst"]): r for r in journal.records
|
|
if r.get("op") == "move"}
|
|
todo = moves or [(r["src"], r["dst"], "") for r in journal.records
|
|
if r.get("op") == "move"]
|
|
bad, missing, bytes_total = [], [], 0
|
|
for src, dst, _kind in todo:
|
|
rec = journalled.get((src, dst))
|
|
if not os.path.exists(dst):
|
|
missing.append(dst)
|
|
continue
|
|
size = os.path.getsize(dst)
|
|
bytes_total += size
|
|
if rec and rec.get("bytes") not in (None, size):
|
|
bad.append(f"{dst}: {size} bytes on disk, {rec['bytes']} at move time")
|
|
if os.path.exists(src) and src != dst:
|
|
bad.append(f"{src}: still in place as well as at {dst}")
|
|
missing += [p for p in keeps if not os.path.exists(p)]
|
|
if bad or missing:
|
|
print(f"verify FAILED: {len(missing)} absent, {len(bad)} wrong")
|
|
for line in (missing + bad)[:20]:
|
|
print(" " + line)
|
|
return 2
|
|
empty = [r["dst"] for r in journal.records
|
|
if r.get("op") == "mkdir" and os.path.isdir(r["dst"])
|
|
and not os.listdir(r["dst"])]
|
|
print(f"verified: {len(todo)} files at their targets"
|
|
f" ({bytes_total / 2**40:.2f} TiB), {len(keeps)} kept")
|
|
if empty:
|
|
print(f"note: {len(empty)} folders are empty (left in place — see the trash law)")
|
|
return 0
|
|
|
|
|
|
def sweep_targets(root, journal):
|
|
"""The folders this run emptied, deepest first — never one it did not touch.
|
|
|
|
Eligibility is evidence, not appearance. A folder is a target only when the
|
|
journal shows a file moved out of it or out of something beneath it, it holds
|
|
nothing but empty directories now, it is not a folder this run created, and no
|
|
journalled destination lives under it. A symlink anywhere inside disqualifies
|
|
the whole subtree: this tool never follows a link, and deleting the folder
|
|
that holds one would delete someone's mount point.
|
|
|
|
Proof carries downward: once a folder qualifies, every empty directory inside
|
|
it qualifies too. A release's junk (``Screens/``, ``Sample/``) is never itself
|
|
a move source, so without that rule it holds its emptied parent on disk
|
|
forever — the parent can never be removed while the child stands.
|
|
"""
|
|
moves = [r for r in journal.records if r.get("op") == "move"]
|
|
made = {r["dst"] for r in journal.records if r.get("op") == "mkdir"}
|
|
gone = {r["path"] for r in journal.records if r.get("op") == "rmdir"}
|
|
dsts = [r["dst"] for r in moves]
|
|
marked = set()
|
|
for r in moves:
|
|
d = os.path.dirname(r["src"])
|
|
while d.startswith(root + os.sep):
|
|
marked.add(d)
|
|
d = os.path.dirname(d)
|
|
|
|
def removable(d):
|
|
if d in made or d in gone or os.path.islink(d):
|
|
return False
|
|
if any(p == d or p.startswith(d + os.sep) for p in dsts):
|
|
return False
|
|
try:
|
|
entries = os.listdir(d)
|
|
except OSError:
|
|
return False
|
|
for e in entries:
|
|
c = os.path.join(d, e)
|
|
if os.path.islink(c) or not os.path.isdir(c) or not removable(c):
|
|
return False
|
|
return True
|
|
|
|
roots = [d for d in marked if removable(d)]
|
|
targets = set(roots)
|
|
for d in roots:
|
|
for dirpath, dirnames, _ in os.walk(d):
|
|
for c in dirnames:
|
|
p = os.path.join(dirpath, c)
|
|
if removable(p):
|
|
targets.add(p)
|
|
return sorted(targets, key=lambda q: (-q.count(os.sep), q))
|
|
|
|
|
|
def do_sweep(args, root, journal):
|
|
"""Remove the empty husks an apply left standing at the library root."""
|
|
moves = [r for r in journal.records if r.get("op") == "move"]
|
|
if not moves:
|
|
raise Refuse("the journal holds no moves — a sweep follows an apply, it"
|
|
" does not guess at emptiness")
|
|
outside = sorted({p for r in moves for p in (r["src"], r["dst"])
|
|
if not (p == root or p.startswith(root + os.sep))})
|
|
if outside and not args.force:
|
|
raise Refuse(f"{len(outside)} journalled paths are outside --root {root}"
|
|
f" (first: {outside[0]}) — this journal belongs to another"
|
|
" library")
|
|
targets = sweep_targets(root, journal)
|
|
if not targets:
|
|
print("swept: 0 folders — nothing this run emptied is still standing empty")
|
|
return 0
|
|
if not args.apply:
|
|
print(f"dry run: {len(targets)} folders this run emptied would be removed"
|
|
" (pass --apply to run)")
|
|
for d in targets[:5]:
|
|
print(f" would remove {d}")
|
|
return 0
|
|
removed = 0
|
|
for d in targets:
|
|
journal.record(op="rmdir", path=d)
|
|
try:
|
|
os.rmdir(d)
|
|
except OSError as e:
|
|
print(f"left in place: {d} ({e})")
|
|
continue
|
|
removed += 1
|
|
print(f"swept: {removed} folders removed, {len(targets) - removed} left in place")
|
|
print(f"journal: {journal.path} — rollback recreates them with their files")
|
|
return 0
|
|
|
|
|
|
def do_rollback(args, root, journal):
|
|
"""Undo every journalled move, newest first, after a full preflight."""
|
|
moves = [r for r in journal.records if r.get("op") == "move"]
|
|
made = [r["dst"] for r in journal.records if r.get("op") == "mkdir"]
|
|
swept = [r["path"] for r in journal.records if r.get("op") == "rmdir"]
|
|
problems = []
|
|
for r in reversed(moves):
|
|
src, dst = r["src"], r["dst"]
|
|
if not args.force and not (src.startswith(root + os.sep)
|
|
or dst.startswith(root + os.sep)):
|
|
problems.append(f"outside --root: {dst}")
|
|
continue
|
|
if os.path.exists(dst) and os.path.exists(src):
|
|
problems.append(f"occupied: both paths exist, refusing to choose:"
|
|
f" {src} | {dst}")
|
|
elif os.path.exists(dst):
|
|
pass
|
|
elif os.path.exists(src):
|
|
continue # already undone
|
|
else:
|
|
problems.append(f"missing: {dst} is gone and {src} never came back")
|
|
if problems:
|
|
print(f"refusing: {len(problems)} problem(s), nothing moved back")
|
|
for p in problems[:20]:
|
|
print(" " + p)
|
|
return 2
|
|
if not args.apply:
|
|
print(f"dry run: would move {len(moves)} files back to their census paths"
|
|
f" and recreate {len(set(swept))} swept folders (pass --apply)")
|
|
return 0
|
|
undone = 0
|
|
recreated = 0
|
|
for d in sorted(set(swept), key=lambda q: q.count(os.sep)):
|
|
if not os.path.isdir(d):
|
|
os.makedirs(d, exist_ok=True)
|
|
recreated += 1
|
|
for r in reversed(moves):
|
|
if os.path.exists(r["dst"]) and not os.path.exists(r["src"]):
|
|
os.makedirs(os.path.dirname(r["src"]), exist_ok=True)
|
|
os.rename(r["dst"], r["src"])
|
|
undone += 1
|
|
dirs_gone = 0
|
|
for d in sorted(set(made), key=lambda q: -q.count(os.sep)):
|
|
if d == root or not os.path.isdir(d) or os.listdir(d):
|
|
continue
|
|
try:
|
|
os.rmdir(d)
|
|
dirs_gone += 1
|
|
except OSError as e:
|
|
print(f"left in place: {d} ({e})")
|
|
journal.record(op="rollback", undone=undone, dirs_removed=dirs_gone,
|
|
t=datetime.datetime.now(datetime.timezone.utc).isoformat())
|
|
print(f"rolled back: {undone} files back where the census found them,"
|
|
f" {dirs_gone} empty folders removed, {recreated} swept folders recreated")
|
|
return 0
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description="apply or undo a media relayout plan")
|
|
ap.add_argument("cmd", choices=["apply", "rollback", "verify", "sweep"])
|
|
ap.add_argument("--plan")
|
|
ap.add_argument("--root", default=DEFAULT_ROOT)
|
|
ap.add_argument("--journal")
|
|
ap.add_argument("--apply", action="store_true", help="actually change the library")
|
|
ap.add_argument("--force", action="store_true", help="relax the opinion guards")
|
|
args = ap.parse_args()
|
|
|
|
root = os.path.abspath(args.root)
|
|
if not os.path.isdir(root):
|
|
raise Refuse(f"no library at {root}")
|
|
journal_path = args.journal or journal_default(root)
|
|
journal = Journal(journal_path, args.apply)
|
|
try:
|
|
if args.cmd in ("rollback", "verify", "sweep") and not os.path.exists(journal_path):
|
|
raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
|
|
if args.cmd in ("rollback", "sweep") and not args.force and journal.rolled_back:
|
|
raise Refuse("this journal has already been rolled back"
|
|
" (--force to replay it)")
|
|
if args.cmd in ("apply", "verify") and not args.plan:
|
|
raise Refuse(f"{args.cmd} needs --plan")
|
|
mkdirs, keeps, moves = (read_plan(args.plan, root, args.force)
|
|
if args.plan else ([], [], []))
|
|
if args.cmd == "apply":
|
|
return do_apply(args, root, journal, mkdirs, keeps, moves)
|
|
if args.cmd == "verify":
|
|
return do_verify(root, journal, moves, keeps)
|
|
if args.cmd == "sweep":
|
|
return do_sweep(args, root, journal)
|
|
return do_rollback(args, root, journal)
|
|
finally:
|
|
journal.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.exit(main())
|
|
except Refuse as r:
|
|
print(f"refusing: {r}")
|
|
sys.exit(2)
|