dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
bd4ee78811f4b705f149f2c5494bad960b123d2d
serverAppConfigs/scripts/media-apply.py
T
Frank e83162a2ac feat(media): a TV mover that will not file a show the library already has 
Two fixes found by one live sweep, shipped together because the second is
what makes the first finishable.

1. scripts/media-tv.py — the other half of media-plan.py's refusal to file
   TV. Groups episode-shaped files by the series name before the marker
   (media-titles.ep_of, imported not restated), writes a plan in the
   executor's own format: <library>/<Series> (<year>)/Season <NN>/<own
   name>.<ext>. Refuses a group below --min, an episode naming no season,
   and an episode the destination library already holds at the same byte
   count — the movie share carries a second copy of all 124 Monk episodes
   the TV library has had for years; a duplicate is Frank's call, never a
   tool's copy. Applied by media-apply.py with its own journal and
   --root /mnt/entropy.

2. scripts/media-apply.py — gone meant "rmdir intended", so a folder whose
   rmdir failed was blacklisted forever and no later sweep could finish it
   (live movie sweep: 4517 targeted, 146 permanently stranded). gone now
   means "actually gone": a failed intent is retried by the next sweep.

tests: 9 new tests in media_relayout.bats, each red against the previous
revision — 8 TV + 1 sweep retry behind a chmod 500 parent. ./tests/run.sh
633 green. No live move run yet.
2026-10-30 20:30:24 -07:00

426 lines
18 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
"""media-apply.py — the only thing that moves movie-library files.
Reads the plan written by scripts/media-plan.py (rows: `action src dst kind`,
plus `mkdir` rows) and performs it with rename(2) — bytes are never copied, so
an interrupted run has destroyed nothing. Every run is resumable and every run
is undoable: the journal is a write-ahead log, one JSON line per intent, fsynced
BEFORE the rename it authorises. That ordering is the whole design: a crash
leaves either an intent whose move happened or an intent whose move never ran,
and both are recognised on the next run. A file that moved with no journal line
cannot have been moved by this tool, and saying so is better than guessing.
Four subcommands:
apply --plan PLAN [--root ROOT] [--journal J] [--apply]
no --apply => report only and write NOTHING, not even folders
rollback --journal J [--root ROOT] [--apply]
back to the paths the census saw; folders this run created are
removed only when empty, never anything else
verify --plan PLAN --journal J [--root ROOT]
every planned file is at its target with the journalled byte count
sweep --journal J [--root ROOT] [--apply]
removes the folders THIS RUN emptied — the journal has to show a
file that moved out of one, so a folder someone made by hand can
never be removed by this tool, and rollback puts them back
Refusals (exit 2, before the first rename, with nothing touched): a target
already occupied by a foreign file, a source that has gone, a duplicate target,
a path outside --root, a journal that was already rolled back, a move that would
cross a filesystem (rename cannot do it and a silent copy at 24 TB would not).
--force relaxes only the guards that are opinion (journal already rolled back,
paths outside --root); it never allows overwriting a file.
The journal defaults to `media-relayout.jsonl` BESIDE the library root, never
inside it: a log inside the library is a file Jellyfin will index.
"""
import argparse
import datetime
import json
import os
import sys
DEFAULT_ROOT = os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies")
class Refuse(Exception):
"""A gate the operator has to see; every refusal leaves the library alone."""
def journal_default(root):
return os.path.join(os.path.dirname(os.path.abspath(root)), "media-relayout.jsonl")
class Journal:
"""Append-only JSONL of intents. Writes are fsynced before the tool acts."""
def __init__(self, path, writing):
self.path, self.writing = path, writing
self.records = []
if os.path.exists(path):
for line in open(path):
line = line.strip()
if line:
self.records.append(json.loads(line))
self._fh = None
def already(self, dst, src):
"""An intent for this exact (src, dst) was written by an earlier run."""
return any(r.get("op") == "move" and r.get("dst") == dst
and r.get("src") == src for r in self.records)
@property
def rolled_back(self):
return any(r.get("op") == "rollback" for r in self.records)
def record(self, **fields):
self.records.append(fields)
if not self.writing:
return
if self._fh is None:
self._fh = open(self.path, "a")
self._fh.write(json.dumps(fields) + "\n")
self._fh.flush()
os.fsync(self._fh.fileno())
def close(self):
if self._fh:
self._fh.close()
def read_plan(path, root, force):
"""Plan rows: (mkdirs, keeps, moves), header dropped, all paths under root."""
mkdirs, keeps, moves = [], [], []
with open(path) as fh:
header = fh.readline()
if not header.startswith("action\t"):
raise Refuse(f"{path} is not a plan (first column is "
f"{header.split(chr(9))[0]!r}, expected 'action') —"
" regenerate it with scripts/media-plan.py")
for line in fh:
f = line.rstrip("\n").split("\t")
if f[0] == "mkdir":
mkdirs.append(f[2])
elif f[0] == "keep":
keeps.append(f[1])
elif f[0] == "move":
moves.append((f[1], f[2], f[3] if len(f) > 3 else ""))
if not force:
for p in mkdirs + keeps + [p for m in moves for p in m[:2]]:
if p and not (p == root or p.startswith(root + os.sep)):
raise Refuse(f"{p} is outside --root {root}"
" (a plan built for another library?)")
return mkdirs, keeps, moves
def do_apply(args, root, journal, mkdirs, keeps, moves):
"""Preflight everything, then folders, then renames. Never a partial start."""
problems = []
for p in keeps:
if not os.path.exists(p):
problems.append(f"missing: keep row source is gone: {p}")
seen = {}
for src, dst, _kind in moves:
if src == dst:
problems.append(f"no-op: move row whose target is its source: {src}")
if dst in seen:
problems.append(f"double-claim: {dst} claimed by {seen[dst]} and {src}")
seen[dst] = src
if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
continue # this intent already ran
if not os.path.exists(src):
if os.path.exists(dst):
problems.append(f"missing: {src} is gone and {dst} carries no journal"
" line — it moved outside this tool, refusing to guess")
else:
problems.append(f"missing: {src} is gone and {dst} is not there either")
continue
if os.path.exists(dst):
problems.append(f"occupied: target already holds another file: {dst}")
if problems:
print(f"refusing: {len(problems)} problem(s), nothing moved")
for p in sorted(problems)[:20]:
print(" " + p)
if len(problems) > 20:
print(f" ... and {len(problems) - 20} more")
return 2
if not args.apply:
print(f"dry run: {len(moves)} moves, {len(mkdirs)} folders, {len(keeps)} files stay"
" — nothing written (pass --apply to run)")
for src, dst, _k in moves[:5]:
print(f" would move {src} -> {dst}")
return 0
made = {r.get("dst") for r in journal.records if r.get("op") == "mkdir"}
dirs_new = 0
for d in sorted(mkdirs, key=lambda q: q.count(os.sep)):
if d in made or os.path.isdir(d):
continue
journal.record(op="mkdir", dst=d)
os.makedirs(d, exist_ok=True)
dirs_new += 1
# same-filesystem gate: rename cannot cross a device, and a silent copy of a
# library this size is not an option anyone would choose.
cross = []
for src, dst, _k in moves:
if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
continue
parent = os.path.dirname(dst)
if os.path.exists(parent) and os.stat(parent).st_dev != os.stat(src).st_dev:
cross.append(dst)
if cross:
print(f"refusing: {len(cross)} moves would cross a filesystem"
f" (first: {cross[0]}) — this tool only renames")
return 2
moved = done = 0
for src, dst, _kind in moves:
if journal.already(dst, src):
if os.path.exists(dst) and not os.path.exists(src):
done += 1
continue
if os.path.exists(src) and not os.path.exists(dst):
os.rename(src, dst) # the intent ran, the move did not
moved += 1
continue
st = os.stat(src)
journal.record(op="move", src=src, dst=dst, bytes=st.st_size, mode=st.st_mode,
mtime=st.st_mtime, ino=st.st_ino, dev=st.st_dev)
os.rename(src, dst)
moved += 1
print(f"applied: {moved} moved, {done} already done in an earlier run,"
f" {dirs_new} folders created, {len(keeps)} files stayed put")
print(f"journal: {journal.path} — undo with: media-apply.py rollback"
f" --journal {journal.path} --root {root} --apply")
return 0
def do_verify(root, journal, moves, keeps):
"""Every planned file sits at its target carrying the journalled bytes."""
journalled = {(r["src"], r["dst"]): r for r in journal.records
if r.get("op") == "move"}
todo = moves or [(r["src"], r["dst"], "") for r in journal.records
if r.get("op") == "move"]
bad, missing, bytes_total = [], [], 0
for src, dst, _kind in todo:
rec = journalled.get((src, dst))
if not os.path.exists(dst):
missing.append(dst)
continue
size = os.path.getsize(dst)
bytes_total += size
if rec and rec.get("bytes") not in (None, size):
bad.append(f"{dst}: {size} bytes on disk, {rec['bytes']} at move time")
if os.path.exists(src) and src != dst:
bad.append(f"{src}: still in place as well as at {dst}")
missing += [p for p in keeps if not os.path.exists(p)]
if bad or missing:
print(f"verify FAILED: {len(missing)} absent, {len(bad)} wrong")
for line in (missing + bad)[:20]:
print(" " + line)
return 2
empty = [r["dst"] for r in journal.records
if r.get("op") == "mkdir" and os.path.isdir(r["dst"])
and not os.listdir(r["dst"])]
print(f"verified: {len(todo)} files at their targets"
f" ({bytes_total / 2**40:.2f} TiB), {len(keeps)} kept")
if empty:
print(f"note: {len(empty)} folders are empty (left in place — see the trash law)")
return 0
def sweep_targets(root, journal):
"""The folders this run emptied, deepest first — never one it did not touch.
Eligibility is evidence, not appearance. A folder is a target only when the
journal shows a file moved out of it or out of something beneath it, it holds
nothing but empty directories now, it is not a folder this run created, and no
journalled destination lives under it. A symlink anywhere inside disqualifies
the whole subtree: this tool never follows a link, and deleting the folder
that holds one would delete someone's mount point.
Proof carries downward: once a folder qualifies, every empty directory inside
it qualifies too. A release's junk (``Screens/``, ``Sample/``) is never itself
a move source, so without that rule it holds its emptied parent on disk
forever — the parent can never be removed while the child stands.
"""
moves = [r for r in journal.records if r.get("op") == "move"]
made = {r["dst"] for r in journal.records if r.get("op") == "mkdir"}
# An rmdir INTENT that failed left its folder standing, and the next sweep
# must be able to finish it: only a folder that is really gone is out of
# scope. Counting every intent as done made 146 live survivors permanent.
gone = {r["path"] for r in journal.records if r.get("op") == "rmdir"
and not os.path.exists(r["path"])}
dsts = [r["dst"] for r in moves]
marked = set()
for r in moves:
d = os.path.dirname(r["src"])
while d.startswith(root + os.sep):
marked.add(d)
d = os.path.dirname(d)
def removable(d):
if d in made or d in gone or os.path.islink(d):
return False
if any(p == d or p.startswith(d + os.sep) for p in dsts):
return False
try:
entries = os.listdir(d)
except OSError:
return False
for e in entries:
c = os.path.join(d, e)
if os.path.islink(c) or not os.path.isdir(c) or not removable(c):
return False
return True
roots = [d for d in marked if removable(d)]
targets = set(roots)
for d in roots:
for dirpath, dirnames, _ in os.walk(d):
for c in dirnames:
p = os.path.join(dirpath, c)
if removable(p):
targets.add(p)
return sorted(targets, key=lambda q: (-q.count(os.sep), q))
def do_sweep(args, root, journal):
"""Remove the empty husks an apply left standing at the library root."""
moves = [r for r in journal.records if r.get("op") == "move"]
if not moves:
raise Refuse("the journal holds no moves — a sweep follows an apply, it"
" does not guess at emptiness")
outside = sorted({p for r in moves for p in (r["src"], r["dst"])
if not (p == root or p.startswith(root + os.sep))})
if outside and not args.force:
raise Refuse(f"{len(outside)} journalled paths are outside --root {root}"
f" (first: {outside[0]}) — this journal belongs to another"
" library")
targets = sweep_targets(root, journal)
if not targets:
print("swept: 0 folders — nothing this run emptied is still standing empty")
return 0
if not args.apply:
print(f"dry run: {len(targets)} folders this run emptied would be removed"
" (pass --apply to run)")
for d in targets[:5]:
print(f" would remove {d}")
return 0
removed = 0
for d in targets:
journal.record(op="rmdir", path=d)
try:
os.rmdir(d)
except OSError as e:
print(f"left in place: {d} ({e})")
continue
removed += 1
print(f"swept: {removed} folders removed, {len(targets) - removed} left in place")
print(f"journal: {journal.path} — rollback recreates them with their files")
return 0
def do_rollback(args, root, journal):
"""Undo every journalled move, newest first, after a full preflight."""
moves = [r for r in journal.records if r.get("op") == "move"]
made = [r["dst"] for r in journal.records if r.get("op") == "mkdir"]
swept = [r["path"] for r in journal.records if r.get("op") == "rmdir"]
problems = []
for r in reversed(moves):
src, dst = r["src"], r["dst"]
if not args.force and not (src.startswith(root + os.sep)
or dst.startswith(root + os.sep)):
problems.append(f"outside --root: {dst}")
continue
if os.path.exists(dst) and os.path.exists(src):
problems.append(f"occupied: both paths exist, refusing to choose:"
f" {src} | {dst}")
elif os.path.exists(dst):
pass
elif os.path.exists(src):
continue # already undone
else:
problems.append(f"missing: {dst} is gone and {src} never came back")
if problems:
print(f"refusing: {len(problems)} problem(s), nothing moved back")
for p in problems[:20]:
print(" " + p)
return 2
if not args.apply:
print(f"dry run: would move {len(moves)} files back to their census paths"
f" and recreate {len(set(swept))} swept folders (pass --apply)")
return 0
undone = 0
recreated = 0
for d in sorted(set(swept), key=lambda q: q.count(os.sep)):
if not os.path.isdir(d):
os.makedirs(d, exist_ok=True)
recreated += 1
for r in reversed(moves):
if os.path.exists(r["dst"]) and not os.path.exists(r["src"]):
os.makedirs(os.path.dirname(r["src"]), exist_ok=True)
os.rename(r["dst"], r["src"])
undone += 1
dirs_gone = 0
for d in sorted(set(made), key=lambda q: -q.count(os.sep)):
if d == root or not os.path.isdir(d) or os.listdir(d):
continue
try:
os.rmdir(d)
dirs_gone += 1
except OSError as e:
print(f"left in place: {d} ({e})")
journal.record(op="rollback", undone=undone, dirs_removed=dirs_gone,
t=datetime.datetime.now(datetime.timezone.utc).isoformat())
print(f"rolled back: {undone} files back where the census found them,"
f" {dirs_gone} empty folders removed, {recreated} swept folders recreated")
return 0
def main():
ap = argparse.ArgumentParser(description="apply or undo a media relayout plan")
ap.add_argument("cmd", choices=["apply", "rollback", "verify", "sweep"])
ap.add_argument("--plan")
ap.add_argument("--root", default=DEFAULT_ROOT)
ap.add_argument("--journal")
ap.add_argument("--apply", action="store_true", help="actually change the library")
ap.add_argument("--force", action="store_true", help="relax the opinion guards")
args = ap.parse_args()
root = os.path.abspath(args.root)
if not os.path.isdir(root):
raise Refuse(f"no library at {root}")
journal_path = args.journal or journal_default(root)
journal = Journal(journal_path, args.apply)
try:
if args.cmd in ("rollback", "verify", "sweep") and not os.path.exists(journal_path):
raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
if args.cmd in ("rollback", "sweep") and not args.force and journal.rolled_back:
raise Refuse("this journal has already been rolled back"
" (--force to replay it)")
if args.cmd in ("apply", "verify") and not args.plan:
raise Refuse(f"{args.cmd} needs --plan")
mkdirs, keeps, moves = (read_plan(args.plan, root, args.force)
if args.plan else ([], [], []))
if args.cmd == "apply":
return do_apply(args, root, journal, mkdirs, keeps, moves)
if args.cmd == "verify":
return do_verify(root, journal, moves, keeps)
if args.cmd == "sweep":
return do_sweep(args, root, journal)
return do_rollback(args, root, journal)
finally:
journal.close()
if __name__ == "__main__":
try:
sys.exit(main())
except Refuse as r:
print(f"refusing: {r}")
sys.exit(2)
Reference in new issue View Git Blame Copy Permalink