From 1cb28095a76366ea8899a94307fe847f50c19395 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 30 Oct 2026 20:03:16 -0700 Subject: [PATCH] feat(media): sweep removes the folders a completed run emptied An apply leaves every release folder it emptied standing where it was: the live library run left 1675 husks at the root. sweep takes the same journal and deletes only folders it can prove this run emptied - a journalled move out of the folder or something under it, nothing but empty directories left, no journalled destination beneath it, not a folder the run created. A folder nobody moved cannot be selected at all, so a hand-made placeholder survives, and a symlink anywhere inside disqualifies the subtree. Every rmdir is journalled, so rollback recreates what a sweep removed. --- scripts/media-apply.py | 100 +++++++++++++++-- .../Dankserver/Media Relayout.md | 71 ++++++++++-- tests/media_relayout.bats | 102 ++++++++++++++++++ 3 files changed, 256 insertions(+), 17 deletions(-) diff --git a/scripts/media-apply.py b/scripts/media-apply.py index 150d171..751d77c 100755 --- a/scripts/media-apply.py +++ b/scripts/media-apply.py @@ -10,7 +10,7 @@ leaves either an intent whose move happened or an intent whose move never ran, and both are recognised on the next run. A file that moved with no journal line cannot have been moved by this tool, and saying so is better than guessing. -Three subcommands: +Four subcommands: apply --plan PLAN [--root ROOT] [--journal J] [--apply] no --apply => report only and write NOTHING, not even folders @@ -19,6 +19,10 @@ Three subcommands: removed only when empty, never anything else verify --plan PLAN --journal J [--root ROOT] every planned file is at its target with the journalled byte count + sweep --journal J [--root ROOT] [--apply] + removes the folders THIS RUN emptied — the journal has to show a + file that moved out of one, so a folder someone made by hand can + never be removed by this tool, and rollback puts them back Refusals (exit 2, before the first rename, with nothing touched): a target already occupied by a foreign file, a source that has gone, a duplicate target, @@ -228,10 +232,87 @@ def do_verify(root, journal, moves, keeps): return 0 +def sweep_targets(root, journal): + """The folders this run emptied, deepest first — never one it did not touch. + + Eligibility is evidence, not appearance. A folder is a target only when the + journal shows a file moved out of it or out of something beneath it, it holds + nothing but empty directories now, it is not a folder this run created, and no + journalled destination lives under it. A symlink anywhere inside disqualifies + the whole subtree: this tool never follows a link, and deleting the folder + that holds one would delete someone's mount point. + """ + moves = [r for r in journal.records if r.get("op") == "move"] + made = {r["dst"] for r in journal.records if r.get("op") == "mkdir"} + gone = {r["path"] for r in journal.records if r.get("op") == "rmdir"} + dsts = [r["dst"] for r in moves] + marked = set() + for r in moves: + d = os.path.dirname(r["src"]) + while d.startswith(root + os.sep): + marked.add(d) + d = os.path.dirname(d) + + def removable(d): + if d in made or d in gone or os.path.islink(d): + return False + if any(p == d or p.startswith(d + os.sep) for p in dsts): + return False + try: + entries = os.listdir(d) + except OSError: + return False + for e in entries: + c = os.path.join(d, e) + if os.path.islink(c) or not os.path.isdir(c) or not removable(c): + return False + return True + + return sorted((d for d in marked if removable(d)), + key=lambda q: (-q.count(os.sep), q)) + + +def do_sweep(args, root, journal): + """Remove the empty husks an apply left standing at the library root.""" + moves = [r for r in journal.records if r.get("op") == "move"] + if not moves: + raise Refuse("the journal holds no moves — a sweep follows an apply, it" + " does not guess at emptiness") + outside = sorted({p for r in moves for p in (r["src"], r["dst"]) + if not (p == root or p.startswith(root + os.sep))}) + if outside and not args.force: + raise Refuse(f"{len(outside)} journalled paths are outside --root {root}" + f" (first: {outside[0]}) — this journal belongs to another" + " library") + targets = sweep_targets(root, journal) + if not targets: + print("swept: 0 folders — nothing this run emptied is still standing empty") + return 0 + if not args.apply: + print(f"dry run: {len(targets)} folders this run emptied would be removed" + " (pass --apply to run)") + for d in targets[:5]: + print(f" would remove {d}") + return 0 + removed = 0 + for d in targets: + journal.record(op="rmdir", path=d) + try: + os.rmdir(d) + except OSError as e: + print(f"left in place: {d} ({e})") + continue + removed += 1 + print(f"swept: {removed} folders removed, {len(targets) - removed} left in place") + print(f"journal: {journal.path} — rollback recreates them with their files") + return 0 + + def do_rollback(args, root, journal): """Undo every journalled move, newest first, after a full preflight.""" moves = [r for r in journal.records if r.get("op") == "move"] made = [r["dst"] for r in journal.records if r.get("op") == "mkdir"] + swept = [r["path"] for r in journal.records if r.get("op") == "rmdir"] problems = [] for r in reversed(moves): src, dst = r["src"], r["dst"] @@ -255,9 +336,14 @@ def do_rollback(args, root, journal): return 2 if not args.apply: print(f"dry run: would move {len(moves)} files back to their census paths" - " (pass --apply)") + f" and recreate {len(set(swept))} swept folders (pass --apply)") return 0 undone = 0 + recreated = 0 + for d in sorted(set(swept), key=lambda q: q.count(os.sep)): + if not os.path.isdir(d): + os.makedirs(d, exist_ok=True) + recreated += 1 for r in reversed(moves): if os.path.exists(r["dst"]) and not os.path.exists(r["src"]): os.makedirs(os.path.dirname(r["src"]), exist_ok=True) @@ -275,13 +361,13 @@ def do_rollback(args, root, journal): journal.record(op="rollback", undone=undone, dirs_removed=dirs_gone, t=datetime.datetime.now(datetime.timezone.utc).isoformat()) print(f"rolled back: {undone} files back where the census found them," - f" {dirs_gone} empty folders removed") + f" {dirs_gone} empty folders removed, {recreated} swept folders recreated") return 0 def main(): ap = argparse.ArgumentParser(description="apply or undo a media relayout plan") - ap.add_argument("cmd", choices=["apply", "rollback", "verify"]) + ap.add_argument("cmd", choices=["apply", "rollback", "verify", "sweep"]) ap.add_argument("--plan") ap.add_argument("--root", default=DEFAULT_ROOT) ap.add_argument("--journal") @@ -295,9 +381,9 @@ def main(): journal_path = args.journal or journal_default(root) journal = Journal(journal_path, args.apply) try: - if args.cmd in ("rollback", "verify") and not os.path.exists(journal_path): + if args.cmd in ("rollback", "verify", "sweep") and not os.path.exists(journal_path): raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}") - if args.cmd == "rollback" and not args.force and journal.rolled_back: + if args.cmd in ("rollback", "sweep") and not args.force and journal.rolled_back: raise Refuse("this journal has already been rolled back" " (--force to replay it)") if args.cmd in ("apply", "verify") and not args.plan: @@ -308,6 +394,8 @@ def main(): return do_apply(args, root, journal, mkdirs, keeps, moves) if args.cmd == "verify": return do_verify(root, journal, moves, keeps) + if args.cmd == "sweep": + return do_sweep(args, root, journal) return do_rollback(args, root, journal) finally: journal.close() diff --git a/serverAppConfigs-docs/Dankserver/Media Relayout.md b/serverAppConfigs-docs/Dankserver/Media Relayout.md index 38056d1..d476277 100644 --- a/serverAppConfigs-docs/Dankserver/Media Relayout.md +++ b/serverAppConfigs-docs/Dankserver/Media Relayout.md @@ -19,7 +19,7 @@ file. |---|---|---| | `scripts/media-census.py` | reads headers only | `enum`, `probed`, `plex` TSVs | | `scripts/media-plan.py` | no — TSVs in, TSVs out | the plan + the exception report | -| `scripts/media-apply.py` | **renames, never copies** | the moves, plus a journal | +| `scripts/media-apply.py` | **renames, never copies**; sweeps only what a run emptied | the moves, plus a journal | The seam is `tests/media_relayout.bats`; the target contract is quoted from [jellyfin.org/docs/general/server/media/movies](https://jellyfin.org/docs/general/server/media/movies) @@ -61,17 +61,58 @@ that would be filed wrong, and the plan is not run until it is understood. ```bash scripts/media-apply.py apply --plan /tmp/jf-plan.tsv --root /mnt/entropy/Movies # dry, writes nothing scripts/media-apply.py apply --plan /tmp/jf-plan.tsv --root /mnt/entropy/Movies --apply -scripts/media-apply.py verify --plan /tmp/jf-plan.tsv --journal …/media-relayout.jsonl -scripts/media-apply.py rollback --journal …/media-relayout.jsonl --apply +scripts/media-apply.py verify --plan /tmp/jf-plan.tsv --journal /mnt/entropy/media-relayout.jsonl +scripts/media-apply.py sweep --journal /mnt/entropy/media-relayout.jsonl --root /mnt/entropy/Movies # dry +scripts/media-apply.py sweep --journal /mnt/entropy/media-relayout.jsonl --root /mnt/entropy/Movies --apply +scripts/media-apply.py rollback --journal /mnt/entropy/media-relayout.jsonl --root /mnt/entropy/Movies --apply ``` -After it lands, both servers need a scan or the change is invisible: -[[Jellyfin]] *Library → Scan All Libraries*, and Plex's library scan -(`podman exec plex …` or the web UI). `verify` reporting empty folders is normal -for the folders the piles own; they are left in place per the deletion policy. +An apply also leaves the release folders it emptied standing where they were — +the live run left 1675 of them at the library root — which is what `sweep` +removes, dry-run first, from the same journal. + +After it lands, both servers need a scan or the change is invisible. + +```bash +# Run this from the host shell through `podman exec`, never from a neighbour +# container: Plex's HTTPD closes without a single byte on any request that +# arrives from another container's netns (10.89.x.x reads as "empty reply", not +# as a 401). The token rides stdin, never argv, where any process on the host +# could read it out of /proc//cmdline. +umask 077 +podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \ + '/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok +podman exec -i plex sh -c 'read -r T + for k in $(curl -s -H "X-Plex-Token: $T" -H "Accept: application/json" \ + http://127.0.0.1:32400/library/sections | grep -o "\"key\":\"[0-9]*\"" | cut -d\" -f4); do + curl -s -o /dev/null -w "plex section $k: %{http_code}\n" -X POST \ + -H "X-Plex-Token: $T" "http://127.0.0.1:32400/library/sections/$k/refresh" + done' /tmp/jf.hdr +curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n" \ + -X POST http://127.0.0.1:8096/Library/Refresh +``` ## Laws +- **The journal lives beside the library, never in `/tmp`.** The live run's + `/mnt/entropy/media-relayout.jsonl` is the only rollback instrument; a journal + a reboot clears is not an instrument. +- **An apply breaks [[Plex]] playback for every moved item until its section is + rescanned.** The database keeps the release-folder path, so streams fail + `Couldn't find the file to stream` while the scanner works through the section. + Trigger the scans in the same window as the apply, and never read a 404 in that + window as a lost file — the file is where the journal says it is. - **Plex's matched title outranks the filename**, and a Plex year of `0` means *indexed but never matched* — trusting it named folders `Title (0)`, which is a Jellyfin item with no year, no poster and no identity. A year that is not a @@ -110,14 +151,22 @@ for the folders the piles own; they are left in place per the deletion policy. duplicate destination stops the run *before the first rename*, so the operator fixes one thing and re-runs instead of untangling a half-applied plan. `--force` relaxes only the opinion guards; it never permits overwriting. +- **A sweep deletes only what the journal proves the run emptied.** A folder is + eligible when a journalled file moved out of it or out of something under it, + it now holds nothing but empty directories, no journalled destination lives + beneath it, and it is not a folder the run created. A folder nobody moved — + someone's placeholder — cannot be selected at all, and a symlink anywhere + inside disqualifies the subtree, because the thing behind a link is not this + tool's to delete. Every removal is journalled as `rmdir` and `rollback` + recreates the folder, so the sweep is undone by the same command as the moves. ## Rollback One command, from the journal of the run: -`media-apply.py rollback --journal …/media-relayout.jsonl --apply`. It restores -every source path, removes only the folders that run created *and that are now -empty*, and refuses everything if any moved file cannot be accounted for. A -rolled-back journal refuses to roll back twice. +`media-apply.py rollback --journal …/media-relayout.jsonl --apply`. It recreates +any folder a sweep removed, restores every source path, removes only the folders +that run created *and that are now empty*, and refuses everything if any moved +file cannot be accounted for. A rolled-back journal refuses to roll back twice. ## See Also diff --git a/tests/media_relayout.bats b/tests/media_relayout.bats index de6e284..eba23d9 100755 --- a/tests/media_relayout.bats +++ b/tests/media_relayout.bats @@ -331,3 +331,105 @@ PY # the refusal is all-or-nothing: another film did not move back [ -e "$LIB/Marnie (1964)/Marnie (1964).mkv" ] } + +# ---------------------------------------------------------------- sweep +# +# A relayout leaves the release folders it emptied standing at the library root: +# 1675 of them after the live run, each holding nothing but empty subfolders. The +# sweep pass removes only folders THIS RUN emptied — proven from the journal, +# never from "it looks empty" — so a folder someone made by hand is untouchable, +# and every removal rides the same rollback as the moves. + +@test "sweep removes only the folders this run emptied" { + plan + apply_plan + mkdir -p "$LIB/A Empty Placeholder" # someone's folder, never a move source + run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply + [ "$status" -eq 0 ] + [[ "$output" == *"swept"* ]] + [ ! -e "$LIB/AMERICAN POP" ] + [ ! -e "$LIB/VHS 2 (2013)" ] + [ ! -e "$LIB/Alien (1979)/Featurettes" ] + [ ! -e "$LIB/Alien (1979)/subs" ] + [ ! -e "$LIB/Marnie (1964)/Exrtas" ] + # a folder the run never moved a file out of survives, empty or not + [ -d "$LIB/A Empty Placeholder" ] +} + +@test "sweep without --apply reports the husks and changes nothing" { + plan + apply_plan + local before jlines + before=$(find "$LIB" -type d | wc -l) + jlines=$(wc -l < "$JOURNAL") + run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" + [ "$status" -eq 0 ] + [[ "$output" == *"--apply"* ]] + [ "$(find "$LIB" -type d | wc -l)" -eq "$before" ] + [ "$(wc -l < "$JOURNAL")" -eq "$jlines" ] + [ -e "$LIB/AMERICAN POP" ] +} + +@test "a folder that still holds content is never swept" { + plan + apply_plan + run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply + [ "$status" -eq 0 ] + # the film folder, the folder the run created, and the piles all stay + [ -d "$LIB/Marnie (1964)" ] + [ -e "$LIB/Marnie (1964)/Marnie (1964).mkv" ] + [ -d "$LIB/V-H-S-2 (2013)" ] + [ -d "$LIB/_unsorted bonus/extras" ] +} + +@test "rollback after a sweep brings the folders and the files back" { + plan + apply_plan + run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply + [ "$status" -eq 0 ] + [ ! -e "$LIB/AMERICAN POP" ] + run python3 "$APPLY" rollback --root "$LIB" --journal "$JOURNAL" --apply + [ "$status" -eq 0 ] + [ -e "$LIB/AMERICAN POP/AMERICAN.POP.2023.1080p.WEB-DL.mkv" ] + [ -e "$LIB/VHS 2 (2013)/VHS.2.2013.1080p.BluRay.x265.mkv" ] + [ -e "$LIB/Alien (1979)/Featurettes/A Bug And A Bite - Making Alien.mkv" ] + [ -e "$LIB/Alien (1979)/subs/3_Eng.srt" ] +} + +@test "a second sweep has nothing left to do and says so" { + plan + apply_plan + run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply + [ "$status" -eq 0 ] + local before + before=$(find "$LIB" -type d | wc -l) + run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply + [ "$status" -eq 0 ] + [[ "$output" == *"swept: 0"* ]] + [ "$(find "$LIB" -type d | wc -l)" -eq "$before" ] +} + +@test "sweep never follows a symlink and never removes the folder it guards" { + plan + apply_plan + mkdir -p "$LIB/Elsewhere" + at "$LIB/Elsewhere/keep.txt" 10 + ln -s "$LIB/Elsewhere" "$LIB/AMERICAN POP/pointer" + run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply + [ "$status" -eq 0 ] + # a link is content, not nothing: the husk it sits in survives untouched + [ -d "$LIB/AMERICAN POP" ] + [ -L "$LIB/AMERICAN POP/pointer" ] + [ -e "$LIB/Elsewhere/keep.txt" ] +} + +@test "sweep refuses a journal that belongs to another library" { + plan + apply_plan + mkdir -p "$BATS_TEST_TMPDIR/Other" + run python3 "$APPLY" sweep --root "$BATS_TEST_TMPDIR/Other" \ + --journal "$JOURNAL" --apply + [ "$status" -ne 0 ] + [[ "$output" == *"outside"* ]] + [ -d "$LIB/AMERICAN POP" ] +} -- 2.54.0