Frank
9eb85ea1f9
Every apply and rollback printed "next: scripts/media-view.sh reap", but on Dankserver there is nothing to reap: the metadata view has never been installed (`media-view.sh status` -> NOT MOUNTED, `/var/lib/media-meta` absent) and the live quadlets still bind the array read-only. All three relayout arms tonight issued that order — 242,084 moves — and it pointed at a directory that does not exist. Where no overlay is mounted the servers bind the array itself, so the sidecars moved with the media. `--metadata-view` (default /var/lib/media-meta/view) names the overlay and the note now rides only when that path is a mountpoint. Red first: two new cases in tests/media_relayout.bats (apply and rollback); the shared helper names a real mountpoint so the pre-existing presence assertion keeps its meaning. 37 ok.
455 lines
19 KiB
Python
Executable File
455 lines
19 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""media-apply.py — the only thing that moves movie-library files.
|
|
|
|
Reads the plan written by scripts/media-plan.py (rows: `action src dst kind`,
|
|
plus `mkdir` rows) and performs it with rename(2) — bytes are never copied, so
|
|
an interrupted run has destroyed nothing. Every run is resumable and every run
|
|
is undoable: the journal is a write-ahead log, one JSON line per intent, fsynced
|
|
BEFORE the rename it authorises. That ordering is the whole design: a crash
|
|
leaves either an intent whose move happened or an intent whose move never ran,
|
|
and both are recognised on the next run. A file that moved with no journal line
|
|
cannot have been moved by this tool, and saying so is better than guessing.
|
|
|
|
Four subcommands:
|
|
|
|
apply --plan PLAN [--root ROOT] [--journal J] [--apply]
|
|
no --apply => report only and write NOTHING, not even folders
|
|
rollback --journal J [--root ROOT] [--apply]
|
|
back to the paths the census saw; folders this run created are
|
|
removed only when empty, never anything else
|
|
verify --plan PLAN --journal J [--root ROOT]
|
|
every planned file is at its target with the journalled byte count
|
|
sweep --journal J [--root ROOT] [--apply]
|
|
removes the folders THIS RUN emptied — the journal has to show a
|
|
file that moved out of one, so a folder someone made by hand can
|
|
never be removed by this tool, and rollback puts them back
|
|
|
|
Refusals (exit 2, before the first rename, with nothing touched): a target
|
|
already occupied by a foreign file, a source that has gone, a duplicate target,
|
|
a path outside --root, a journal that was already rolled back, a move that would
|
|
cross a filesystem (rename cannot do it and a silent copy at 24 TB would not).
|
|
|
|
--force relaxes only the guards that are opinion (journal already rolled back,
|
|
paths outside --root); it never allows overwriting a file.
|
|
|
|
--metadata-view names the overlay the media servers read their library through
|
|
(default /var/lib/media-meta/view). The sidecar follow-up printed after an apply
|
|
or rollback is conditioned on that path being a mountpoint: where no view is
|
|
mounted the servers bind the array itself, the sidecars moved with the media,
|
|
and a reap order would point at a directory that is not there.
|
|
|
|
The journal defaults to `media-relayout.jsonl` BESIDE the library root, never
|
|
inside it: a log inside the library is a file Jellyfin will index.
|
|
"""
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
DEFAULT_ROOT = os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies")
|
|
|
|
|
|
class Refuse(Exception):
|
|
"""A gate the operator has to see; every refusal leaves the library alone."""
|
|
|
|
|
|
def journal_default(root):
|
|
return os.path.join(os.path.dirname(os.path.abspath(root)), "media-relayout.jsonl")
|
|
|
|
|
|
class Journal:
|
|
"""Append-only JSONL of intents. Writes are fsynced before the tool acts."""
|
|
|
|
def __init__(self, path, writing):
|
|
self.path, self.writing = path, writing
|
|
self.records = []
|
|
if os.path.exists(path):
|
|
for line in open(path):
|
|
line = line.strip()
|
|
if line:
|
|
self.records.append(json.loads(line))
|
|
self._fh = None
|
|
|
|
def already(self, dst, src):
|
|
"""An intent for this exact (src, dst) was written by an earlier run."""
|
|
return any(r.get("op") == "move" and r.get("dst") == dst
|
|
and r.get("src") == src for r in self.records)
|
|
|
|
@property
|
|
def rolled_back(self):
|
|
return any(r.get("op") == "rollback" for r in self.records)
|
|
|
|
def record(self, **fields):
|
|
self.records.append(fields)
|
|
if not self.writing:
|
|
return
|
|
if self._fh is None:
|
|
self._fh = open(self.path, "a")
|
|
self._fh.write(json.dumps(fields) + "\n")
|
|
self._fh.flush()
|
|
os.fsync(self._fh.fileno())
|
|
|
|
def close(self):
|
|
if self._fh:
|
|
self._fh.close()
|
|
|
|
|
|
def read_plan(path, root, force):
|
|
"""Plan rows: (mkdirs, keeps, moves), header dropped, all paths under root."""
|
|
mkdirs, keeps, moves = [], [], []
|
|
with open(path) as fh:
|
|
header = fh.readline()
|
|
if not header.startswith("action\t"):
|
|
raise Refuse(f"{path} is not a plan (first column is "
|
|
f"{header.split(chr(9))[0]!r}, expected 'action') —"
|
|
" regenerate it with scripts/media-plan.py")
|
|
for line in fh:
|
|
f = line.rstrip("\n").split("\t")
|
|
if f[0] == "mkdir":
|
|
mkdirs.append(f[2])
|
|
elif f[0] == "keep":
|
|
keeps.append(f[1])
|
|
elif f[0] == "move":
|
|
moves.append((f[1], f[2], f[3] if len(f) > 3 else ""))
|
|
if not force:
|
|
for p in mkdirs + keeps + [p for m in moves for p in m[:2]]:
|
|
if p and not (p == root or p.startswith(root + os.sep)):
|
|
raise Refuse(f"{p} is outside --root {root}"
|
|
" (a plan built for another library?)")
|
|
return mkdirs, keeps, moves
|
|
|
|
|
|
# The media servers may read the library through the SSD overlay view
|
|
# (scripts/media-view.sh). A relayout moves bytes on the raw array, so sidecars
|
|
# the servers wrote through that view stay at the old coordinates — naming the
|
|
# follow-up here is the point of use for that fact, but only where the view
|
|
# exists: where it is not mounted the servers bind the array itself, the
|
|
# sidecars moved with the media, and a reap order points at a directory that
|
|
# is not there. See the Media Metadata View page for the host's state.
|
|
METADATA_VIEW = "/var/lib/media-meta/view"
|
|
METADATA_HINT = ("note: sidecars written through the metadata view did not follow"
|
|
" these moves — next: scripts/media-view.sh reap")
|
|
|
|
|
|
def metadata_note(args):
|
|
"""Print the sidecar follow-up only when the overlay is really mounted."""
|
|
if os.path.ismount(args.metadata_view):
|
|
print(METADATA_HINT)
|
|
|
|
|
|
def do_apply(args, root, journal, mkdirs, keeps, moves):
|
|
"""Preflight everything, then folders, then renames. Never a partial start."""
|
|
problems = []
|
|
for p in keeps:
|
|
if not os.path.exists(p):
|
|
problems.append(f"missing: keep row source is gone: {p}")
|
|
seen = {}
|
|
for src, dst, _kind in moves:
|
|
if src == dst:
|
|
problems.append(f"no-op: move row whose target is its source: {src}")
|
|
if dst in seen:
|
|
problems.append(f"double-claim: {dst} claimed by {seen[dst]} and {src}")
|
|
seen[dst] = src
|
|
if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
|
|
continue # this intent already ran
|
|
if not os.path.exists(src):
|
|
if os.path.exists(dst):
|
|
problems.append(f"missing: {src} is gone and {dst} carries no journal"
|
|
" line — it moved outside this tool, refusing to guess")
|
|
else:
|
|
problems.append(f"missing: {src} is gone and {dst} is not there either")
|
|
continue
|
|
if os.path.exists(dst):
|
|
problems.append(f"occupied: target already holds another file: {dst}")
|
|
if problems:
|
|
print(f"refusing: {len(problems)} problem(s), nothing moved")
|
|
for p in sorted(problems)[:20]:
|
|
print(" " + p)
|
|
if len(problems) > 20:
|
|
print(f" ... and {len(problems) - 20} more")
|
|
return 2
|
|
|
|
if not args.apply:
|
|
print(f"dry run: {len(moves)} moves, {len(mkdirs)} folders, {len(keeps)} files stay"
|
|
" — nothing written (pass --apply to run)")
|
|
for src, dst, _k in moves[:5]:
|
|
print(f" would move {src} -> {dst}")
|
|
return 0
|
|
|
|
made = {r.get("dst") for r in journal.records if r.get("op") == "mkdir"}
|
|
dirs_new = 0
|
|
for d in sorted(mkdirs, key=lambda q: q.count(os.sep)):
|
|
if d in made or os.path.isdir(d):
|
|
continue
|
|
journal.record(op="mkdir", dst=d)
|
|
os.makedirs(d, exist_ok=True)
|
|
dirs_new += 1
|
|
|
|
# same-filesystem gate: rename cannot cross a device, and a silent copy of a
|
|
# library this size is not an option anyone would choose.
|
|
cross = []
|
|
for src, dst, _k in moves:
|
|
if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
|
|
continue
|
|
parent = os.path.dirname(dst)
|
|
if os.path.exists(parent) and os.stat(parent).st_dev != os.stat(src).st_dev:
|
|
cross.append(dst)
|
|
if cross:
|
|
print(f"refusing: {len(cross)} moves would cross a filesystem"
|
|
f" (first: {cross[0]}) — this tool only renames")
|
|
return 2
|
|
|
|
moved = done = 0
|
|
for src, dst, _kind in moves:
|
|
if journal.already(dst, src):
|
|
if os.path.exists(dst) and not os.path.exists(src):
|
|
done += 1
|
|
continue
|
|
if os.path.exists(src) and not os.path.exists(dst):
|
|
os.rename(src, dst) # the intent ran, the move did not
|
|
moved += 1
|
|
continue
|
|
st = os.stat(src)
|
|
journal.record(op="move", src=src, dst=dst, bytes=st.st_size, mode=st.st_mode,
|
|
mtime=st.st_mtime, ino=st.st_ino, dev=st.st_dev)
|
|
os.rename(src, dst)
|
|
moved += 1
|
|
print(f"applied: {moved} moved, {done} already done in an earlier run,"
|
|
f" {dirs_new} folders created, {len(keeps)} files stayed put")
|
|
print(f"journal: {journal.path} — undo with: media-apply.py rollback"
|
|
f" --journal {journal.path} --root {root} --apply")
|
|
metadata_note(args)
|
|
return 0
|
|
|
|
|
|
def do_verify(root, journal, moves, keeps):
|
|
"""Every planned file sits at its target carrying the journalled bytes."""
|
|
journalled = {(r["src"], r["dst"]): r for r in journal.records
|
|
if r.get("op") == "move"}
|
|
todo = moves or [(r["src"], r["dst"], "") for r in journal.records
|
|
if r.get("op") == "move"]
|
|
bad, missing, bytes_total = [], [], 0
|
|
for src, dst, _kind in todo:
|
|
rec = journalled.get((src, dst))
|
|
if not os.path.exists(dst):
|
|
missing.append(dst)
|
|
continue
|
|
size = os.path.getsize(dst)
|
|
bytes_total += size
|
|
if rec and rec.get("bytes") not in (None, size):
|
|
bad.append(f"{dst}: {size} bytes on disk, {rec['bytes']} at move time")
|
|
if os.path.exists(src) and src != dst:
|
|
bad.append(f"{src}: still in place as well as at {dst}")
|
|
missing += [p for p in keeps if not os.path.exists(p)]
|
|
if bad or missing:
|
|
print(f"verify FAILED: {len(missing)} absent, {len(bad)} wrong")
|
|
for line in (missing + bad)[:20]:
|
|
print(" " + line)
|
|
return 2
|
|
empty = [r["dst"] for r in journal.records
|
|
if r.get("op") == "mkdir" and os.path.isdir(r["dst"])
|
|
and not os.listdir(r["dst"])]
|
|
print(f"verified: {len(todo)} files at their targets"
|
|
f" ({bytes_total / 2**40:.2f} TiB), {len(keeps)} kept")
|
|
if empty:
|
|
print(f"note: {len(empty)} folders are empty (left in place — see the trash law)")
|
|
return 0
|
|
|
|
|
|
def sweep_targets(root, journal):
|
|
"""The folders this run emptied, deepest first — never one it did not touch.
|
|
|
|
Eligibility is evidence, not appearance. A folder is a target only when the
|
|
journal shows a file moved out of it or out of something beneath it, it holds
|
|
nothing but empty directories now, it is not a folder this run created, and no
|
|
journalled destination lives under it. A symlink anywhere inside disqualifies
|
|
the whole subtree: this tool never follows a link, and deleting the folder
|
|
that holds one would delete someone's mount point.
|
|
|
|
Proof carries downward: once a folder qualifies, every empty directory inside
|
|
it qualifies too. A release's junk (``Screens/``, ``Sample/``) is never itself
|
|
a move source, so without that rule it holds its emptied parent on disk
|
|
forever — the parent can never be removed while the child stands.
|
|
"""
|
|
moves = [r for r in journal.records if r.get("op") == "move"]
|
|
made = {r["dst"] for r in journal.records if r.get("op") == "mkdir"}
|
|
# An rmdir INTENT that failed left its folder standing, and the next sweep
|
|
# must be able to finish it: only a folder that is really gone is out of
|
|
# scope. Counting every intent as done made 146 live survivors permanent.
|
|
gone = {r["path"] for r in journal.records if r.get("op") == "rmdir"
|
|
and not os.path.exists(r["path"])}
|
|
dsts = [r["dst"] for r in moves]
|
|
marked = set()
|
|
for r in moves:
|
|
d = os.path.dirname(r["src"])
|
|
while d.startswith(root + os.sep):
|
|
marked.add(d)
|
|
d = os.path.dirname(d)
|
|
|
|
def removable(d):
|
|
if d in made or d in gone or os.path.islink(d):
|
|
return False
|
|
if any(p == d or p.startswith(d + os.sep) for p in dsts):
|
|
return False
|
|
try:
|
|
entries = os.listdir(d)
|
|
except OSError:
|
|
return False
|
|
for e in entries:
|
|
c = os.path.join(d, e)
|
|
if os.path.islink(c) or not os.path.isdir(c) or not removable(c):
|
|
return False
|
|
return True
|
|
|
|
roots = [d for d in marked if removable(d)]
|
|
targets = set(roots)
|
|
for d in roots:
|
|
for dirpath, dirnames, _ in os.walk(d):
|
|
for c in dirnames:
|
|
p = os.path.join(dirpath, c)
|
|
if removable(p):
|
|
targets.add(p)
|
|
return sorted(targets, key=lambda q: (-q.count(os.sep), q))
|
|
|
|
|
|
def do_sweep(args, root, journal):
|
|
"""Remove the empty husks an apply left standing at the library root."""
|
|
moves = [r for r in journal.records if r.get("op") == "move"]
|
|
if not moves:
|
|
raise Refuse("the journal holds no moves — a sweep follows an apply, it"
|
|
" does not guess at emptiness")
|
|
outside = sorted({p for r in moves for p in (r["src"], r["dst"])
|
|
if not (p == root or p.startswith(root + os.sep))})
|
|
if outside and not args.force:
|
|
raise Refuse(f"{len(outside)} journalled paths are outside --root {root}"
|
|
f" (first: {outside[0]}) — this journal belongs to another"
|
|
" library")
|
|
targets = sweep_targets(root, journal)
|
|
if not targets:
|
|
print("swept: 0 folders — nothing this run emptied is still standing empty")
|
|
return 0
|
|
if not args.apply:
|
|
print(f"dry run: {len(targets)} folders this run emptied would be removed"
|
|
" (pass --apply to run)")
|
|
for d in targets[:5]:
|
|
print(f" would remove {d}")
|
|
return 0
|
|
removed = 0
|
|
for d in targets:
|
|
journal.record(op="rmdir", path=d)
|
|
try:
|
|
os.rmdir(d)
|
|
except OSError as e:
|
|
print(f"left in place: {d} ({e})")
|
|
continue
|
|
removed += 1
|
|
print(f"swept: {removed} folders removed, {len(targets) - removed} left in place")
|
|
print(f"journal: {journal.path} — rollback recreates them with their files")
|
|
return 0
|
|
|
|
|
|
def do_rollback(args, root, journal):
|
|
"""Undo every journalled move, newest first, after a full preflight."""
|
|
moves = [r for r in journal.records if r.get("op") == "move"]
|
|
made = [r["dst"] for r in journal.records if r.get("op") == "mkdir"]
|
|
swept = [r["path"] for r in journal.records if r.get("op") == "rmdir"]
|
|
problems = []
|
|
for r in reversed(moves):
|
|
src, dst = r["src"], r["dst"]
|
|
if not args.force and not (src.startswith(root + os.sep)
|
|
or dst.startswith(root + os.sep)):
|
|
problems.append(f"outside --root: {dst}")
|
|
continue
|
|
if os.path.exists(dst) and os.path.exists(src):
|
|
problems.append(f"occupied: both paths exist, refusing to choose:"
|
|
f" {src} | {dst}")
|
|
elif os.path.exists(dst):
|
|
pass
|
|
elif os.path.exists(src):
|
|
continue # already undone
|
|
else:
|
|
problems.append(f"missing: {dst} is gone and {src} never came back")
|
|
if problems:
|
|
print(f"refusing: {len(problems)} problem(s), nothing moved back")
|
|
for p in problems[:20]:
|
|
print(" " + p)
|
|
return 2
|
|
if not args.apply:
|
|
print(f"dry run: would move {len(moves)} files back to their census paths"
|
|
f" and recreate {len(set(swept))} swept folders (pass --apply)")
|
|
return 0
|
|
undone = 0
|
|
recreated = 0
|
|
for d in sorted(set(swept), key=lambda q: q.count(os.sep)):
|
|
if not os.path.isdir(d):
|
|
os.makedirs(d, exist_ok=True)
|
|
recreated += 1
|
|
for r in reversed(moves):
|
|
if os.path.exists(r["dst"]) and not os.path.exists(r["src"]):
|
|
os.makedirs(os.path.dirname(r["src"]), exist_ok=True)
|
|
os.rename(r["dst"], r["src"])
|
|
undone += 1
|
|
dirs_gone = 0
|
|
for d in sorted(set(made), key=lambda q: -q.count(os.sep)):
|
|
if d == root or not os.path.isdir(d) or os.listdir(d):
|
|
continue
|
|
try:
|
|
os.rmdir(d)
|
|
dirs_gone += 1
|
|
except OSError as e:
|
|
print(f"left in place: {d} ({e})")
|
|
journal.record(op="rollback", undone=undone, dirs_removed=dirs_gone,
|
|
t=datetime.datetime.now(datetime.timezone.utc).isoformat())
|
|
print(f"rolled back: {undone} files back where the census found them,"
|
|
f" {dirs_gone} empty folders removed, {recreated} swept folders recreated")
|
|
metadata_note(args)
|
|
return 0
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description="apply or undo a media relayout plan")
|
|
ap.add_argument("cmd", choices=["apply", "rollback", "verify", "sweep"])
|
|
ap.add_argument("--plan")
|
|
ap.add_argument("--root", default=DEFAULT_ROOT)
|
|
ap.add_argument("--journal")
|
|
ap.add_argument("--apply", action="store_true", help="actually change the library")
|
|
ap.add_argument("--force", action="store_true", help="relax the opinion guards")
|
|
ap.add_argument("--metadata-view", default=METADATA_VIEW,
|
|
help="overlay the media servers read through; the sidecar"
|
|
" note prints only when this path is a mountpoint")
|
|
args = ap.parse_args()
|
|
|
|
root = os.path.abspath(args.root)
|
|
if not os.path.isdir(root):
|
|
raise Refuse(f"no library at {root}")
|
|
journal_path = args.journal or journal_default(root)
|
|
journal = Journal(journal_path, args.apply)
|
|
try:
|
|
if args.cmd in ("rollback", "verify", "sweep") and not os.path.exists(journal_path):
|
|
raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
|
|
if args.cmd in ("rollback", "sweep") and not args.force and journal.rolled_back:
|
|
raise Refuse("this journal has already been rolled back"
|
|
" (--force to replay it)")
|
|
if args.cmd in ("apply", "verify") and not args.plan:
|
|
raise Refuse(f"{args.cmd} needs --plan")
|
|
mkdirs, keeps, moves = (read_plan(args.plan, root, args.force)
|
|
if args.plan else ([], [], []))
|
|
if args.cmd == "apply":
|
|
return do_apply(args, root, journal, mkdirs, keeps, moves)
|
|
if args.cmd == "verify":
|
|
return do_verify(root, journal, moves, keeps)
|
|
if args.cmd == "sweep":
|
|
return do_sweep(args, root, journal)
|
|
return do_rollback(args, root, journal)
|
|
finally:
|
|
journal.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.exit(main())
|
|
except Refuse as r:
|
|
print(f"refusing: {r}")
|
|
sys.exit(2)
|