Merge pull request 'feat(media): sweep removes the folders a completed run emptied' (#94) from feat/media-sweep into main

scripts/media-apply.py

@@ -10,7 +10,7 @@ leaves either an intent whose move happened or an intent whose move never ran,
 and both are recognised on the next run. A file that moved with no journal line
 cannot have been moved by this tool, and saying so is better than guessing.
 
-Three subcommands:
+Four subcommands:
 
     apply    --plan PLAN [--root ROOT] [--journal J] [--apply]
              no --apply => report only and write NOTHING, not even folders
@@ -19,6 +19,10 @@ Three subcommands:
              removed only when empty, never anything else
     verify   --plan PLAN --journal J [--root ROOT]
              every planned file is at its target with the journalled byte count
+    sweep    --journal J [--root ROOT] [--apply]
+             removes the folders THIS RUN emptied — the journal has to show a
+             file that moved out of one, so a folder someone made by hand can
+             never be removed by this tool, and rollback puts them back
 
 Refusals (exit 2, before the first rename, with nothing touched): a target
 already occupied by a foreign file, a source that has gone, a duplicate target,
@@ -228,10 +232,87 @@ def do_verify(root, journal, moves, keeps):
     return 0
 
 
+def sweep_targets(root, journal):
+    """The folders this run emptied, deepest first — never one it did not touch.
+
+    Eligibility is evidence, not appearance. A folder is a target only when the
+    journal shows a file moved out of it or out of something beneath it, it holds
+    nothing but empty directories now, it is not a folder this run created, and no
+    journalled destination lives under it. A symlink anywhere inside disqualifies
+    the whole subtree: this tool never follows a link, and deleting the folder
+    that holds one would delete someone's mount point.
+    """
+    moves = [r for r in journal.records if r.get("op") == "move"]
+    made = {r["dst"] for r in journal.records if r.get("op") == "mkdir"}
+    gone = {r["path"] for r in journal.records if r.get("op") == "rmdir"}
+    dsts = [r["dst"] for r in moves]
+    marked = set()
+    for r in moves:
+        d = os.path.dirname(r["src"])
+        while d.startswith(root + os.sep):
+            marked.add(d)
+            d = os.path.dirname(d)
+
+    def removable(d):
+        if d in made or d in gone or os.path.islink(d):
+            return False
+        if any(p == d or p.startswith(d + os.sep) for p in dsts):
+            return False
+        try:
+            entries = os.listdir(d)
+        except OSError:
+            return False
+        for e in entries:
+            c = os.path.join(d, e)
+            if os.path.islink(c) or not os.path.isdir(c) or not removable(c):
+                return False
+        return True
+
+    return sorted((d for d in marked if removable(d)),
+                  key=lambda q: (-q.count(os.sep), q))
+
+
+def do_sweep(args, root, journal):
+    """Remove the empty husks an apply left standing at the library root."""
+    moves = [r for r in journal.records if r.get("op") == "move"]
+    if not moves:
+        raise Refuse("the journal holds no moves — a sweep follows an apply, it"
+                     " does not guess at emptiness")
+    outside = sorted({p for r in moves for p in (r["src"], r["dst"])
+                      if not (p == root or p.startswith(root + os.sep))})
+    if outside and not args.force:
+        raise Refuse(f"{len(outside)} journalled paths are outside --root {root}"
+                     f" (first: {outside[0]}) — this journal belongs to another"
+                     " library")
+    targets = sweep_targets(root, journal)
+    if not targets:
+        print("swept: 0 folders — nothing this run emptied is still standing empty")
+        return 0
+    if not args.apply:
+        print(f"dry run: {len(targets)} folders this run emptied would be removed"
+              " (pass --apply to run)")
+        for d in targets[:5]:
+            print(f"  would remove {d}")
+        return 0
+    removed = 0
+    for d in targets:
+        journal.record(op="rmdir", path=d)
+        try:
+            os.rmdir(d)
+        except OSError as e:
+            print(f"left in place: {d} ({e})")
+            continue
+        removed += 1
+    print(f"swept: {removed} folders removed, {len(targets) - removed} left in place")
+    print(f"journal: {journal.path} — rollback recreates them with their files")
+    return 0
+
+
 def do_rollback(args, root, journal):
     """Undo every journalled move, newest first, after a full preflight."""
     moves = [r for r in journal.records if r.get("op") == "move"]
     made = [r["dst"] for r in journal.records if r.get("op") == "mkdir"]
+    swept = [r["path"] for r in journal.records if r.get("op") == "rmdir"]
     problems = []
     for r in reversed(moves):
         src, dst = r["src"], r["dst"]
@@ -255,9 +336,14 @@ def do_rollback(args, root, journal):
         return 2
     if not args.apply:
         print(f"dry run: would move {len(moves)} files back to their census paths"
-              " (pass --apply)")
+              f" and recreate {len(set(swept))} swept folders (pass --apply)")
         return 0
     undone = 0
+    recreated = 0
+    for d in sorted(set(swept), key=lambda q: q.count(os.sep)):
+        if not os.path.isdir(d):
+            os.makedirs(d, exist_ok=True)
+            recreated += 1
     for r in reversed(moves):
         if os.path.exists(r["dst"]) and not os.path.exists(r["src"]):
             os.makedirs(os.path.dirname(r["src"]), exist_ok=True)
@@ -275,13 +361,13 @@ def do_rollback(args, root, journal):
     journal.record(op="rollback", undone=undone, dirs_removed=dirs_gone,
                    t=datetime.datetime.now(datetime.timezone.utc).isoformat())
     print(f"rolled back: {undone} files back where the census found them,"
-          f" {dirs_gone} empty folders removed")
+          f" {dirs_gone} empty folders removed, {recreated} swept folders recreated")
     return 0
 
 
 def main():
     ap = argparse.ArgumentParser(description="apply or undo a media relayout plan")
-    ap.add_argument("cmd", choices=["apply", "rollback", "verify"])
+    ap.add_argument("cmd", choices=["apply", "rollback", "verify", "sweep"])
     ap.add_argument("--plan")
     ap.add_argument("--root", default=DEFAULT_ROOT)
     ap.add_argument("--journal")
@@ -295,9 +381,9 @@ def main():
     journal_path = args.journal or journal_default(root)
     journal = Journal(journal_path, args.apply)
     try:
-        if args.cmd in ("rollback", "verify") and not os.path.exists(journal_path):
+        if args.cmd in ("rollback", "verify", "sweep") and not os.path.exists(journal_path):
             raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
-        if args.cmd == "rollback" and not args.force and journal.rolled_back:
+        if args.cmd in ("rollback", "sweep") and not args.force and journal.rolled_back:
             raise Refuse("this journal has already been rolled back"
                          " (--force to replay it)")
         if args.cmd in ("apply", "verify") and not args.plan:
@@ -308,6 +394,8 @@ def main():
             return do_apply(args, root, journal, mkdirs, keeps, moves)
         if args.cmd == "verify":
             return do_verify(root, journal, moves, keeps)
+        if args.cmd == "sweep":
+            return do_sweep(args, root, journal)
         return do_rollback(args, root, journal)
     finally:
         journal.close()

serverAppConfigs-docs/Dankserver/Media Relayout.md

@@ -19,7 +19,7 @@ file.
 |---|---|---|
 | `scripts/media-census.py` | reads headers only | `enum`, `probed`, `plex` TSVs |
 | `scripts/media-plan.py` | no — TSVs in, TSVs out | the plan + the exception report |
-| `scripts/media-apply.py` | **renames, never copies** | the moves, plus a journal |
+| `scripts/media-apply.py` | **renames, never copies**; sweeps only what a run emptied | the moves, plus a journal |
 
 The seam is `tests/media_relayout.bats`; the target contract is quoted from
 [jellyfin.org/docs/general/server/media/movies](https://jellyfin.org/docs/general/server/media/movies)
@@ -61,17 +61,58 @@ that would be filed wrong, and the plan is not run until it is understood.
 ```bash
 scripts/media-apply.py apply    --plan /tmp/jf-plan.tsv --root /mnt/entropy/Movies   # dry, writes nothing
 scripts/media-apply.py apply    --plan /tmp/jf-plan.tsv --root /mnt/entropy/Movies --apply
-scripts/media-apply.py verify   --plan /tmp/jf-plan.tsv --journal …/media-relayout.jsonl
+scripts/media-apply.py verify   --plan /tmp/jf-plan.tsv --journal /mnt/entropy/media-relayout.jsonl
-scripts/media-apply.py rollback --journal …/media-relayout.jsonl --apply
+scripts/media-apply.py sweep    --journal /mnt/entropy/media-relayout.jsonl --root /mnt/entropy/Movies   # dry
+scripts/media-apply.py sweep    --journal /mnt/entropy/media-relayout.jsonl --root /mnt/entropy/Movies --apply
+scripts/media-apply.py rollback --journal /mnt/entropy/media-relayout.jsonl --root /mnt/entropy/Movies --apply
 ```
 
-After it lands, both servers need a scan or the change is invisible:
+An apply also leaves the release folders it emptied standing where they were —
-[[Jellyfin]] *Library → Scan All Libraries*, and Plex's library scan
+the live run left 1675 of them at the library root — which is what `sweep`
-(`podman exec plex …` or the web UI). `verify` reporting empty folders is normal
+removes, dry-run first, from the same journal.
-for the folders the piles own; they are left in place per the deletion policy.
+
+After it lands, both servers need a scan or the change is invisible.
+
+```bash
+# Run this from the host shell through `podman exec`, never from a neighbour
+# container: Plex's HTTPD closes without a single byte on any request that
+# arrives from another container's netns (10.89.x.x reads as "empty reply", not
+# as a 401). The token rides stdin, never argv, where any process on the host
+# could read it out of /proc/<pid>/cmdline.
+umask 077
+podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \
+  '/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok
+podman exec -i plex sh -c 'read -r T
+  for k in $(curl -s -H "X-Plex-Token: $T" -H "Accept: application/json" \
+      http://127.0.0.1:32400/library/sections | grep -o "\"key\":\"[0-9]*\"" | cut -d\" -f4); do
+    curl -s -o /dev/null -w "plex section $k: %{http_code}\n" -X POST \
+      -H "X-Plex-Token: $T" "http://127.0.0.1:32400/library/sections/$k/refresh"
+  done' </tmp/px.tok
+```
+
+[[Jellyfin]] has no credential on this box that can trigger a scan: the two keys
+in its database are Seerr's, unused since 2026-09-01, and answer 401 to every
+call. Mint one (*Dashboard → Advanced → API Keys*, Administrator) and store it in
+`~/.config/secrets/jellyfin`, or press *Library → Scan All Libraries* in the web
+UI. A scripted scan passes the key in a mode-600 curl config, never in a header
+argument:
+
+```bash
+umask 077; printf 'header = "X-Emby-Token: %s"\n' "$(cat ~/.config/secrets/jellyfin)" >/tmp/jf.hdr
+curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n" \
+  -X POST http://127.0.0.1:8096/Library/Refresh
+```
 
 ## Laws
 
+- **The journal lives beside the library, never in `/tmp`.** The live run's
+  `/mnt/entropy/media-relayout.jsonl` is the only rollback instrument; a journal
+  a reboot clears is not an instrument.
+- **An apply breaks [[Plex]] playback for every moved item until its section is
+  rescanned.** The database keeps the release-folder path, so streams fail
+  `Couldn't find the file to stream` while the scanner works through the section.
+  Trigger the scans in the same window as the apply, and never read a 404 in that
+  window as a lost file — the file is where the journal says it is.
 - **Plex's matched title outranks the filename**, and a Plex year of `0` means
   *indexed but never matched* — trusting it named folders `Title (0)`, which is
   a Jellyfin item with no year, no poster and no identity. A year that is not a
@@ -110,14 +151,22 @@ for the folders the piles own; they are left in place per the deletion policy.
   duplicate destination stops the run *before the first rename*, so the operator
   fixes one thing and re-runs instead of untangling a half-applied plan.
   `--force` relaxes only the opinion guards; it never permits overwriting.
+- **A sweep deletes only what the journal proves the run emptied.** A folder is
+  eligible when a journalled file moved out of it or out of something under it,
+  it now holds nothing but empty directories, no journalled destination lives
+  beneath it, and it is not a folder the run created. A folder nobody moved —
+  someone's placeholder — cannot be selected at all, and a symlink anywhere
+  inside disqualifies the subtree, because the thing behind a link is not this
+  tool's to delete. Every removal is journalled as `rmdir` and `rollback`
+  recreates the folder, so the sweep is undone by the same command as the moves.
 
 ## Rollback
 
 One command, from the journal of the run:
-`media-apply.py rollback --journal …/media-relayout.jsonl --apply`. It restores
+`media-apply.py rollback --journal …/media-relayout.jsonl --apply`. It recreates
-every source path, removes only the folders that run created *and that are now
+any folder a sweep removed, restores every source path, removes only the folders
-empty*, and refuses everything if any moved file cannot be accounted for. A
+that run created *and that are now empty*, and refuses everything if any moved
-rolled-back journal refuses to roll back twice.
+file cannot be accounted for. A rolled-back journal refuses to roll back twice.
 
 ## See Also
 

tests/media_relayout.bats

@@ -331,3 +331,105 @@ PY
     # the refusal is all-or-nothing: another film did not move back
     [ -e "$LIB/Marnie (1964)/Marnie (1964).mkv" ]
 }
+
+# ---------------------------------------------------------------- sweep
+#
+# A relayout leaves the release folders it emptied standing at the library root:
+# 1675 of them after the live run, each holding nothing but empty subfolders. The
+# sweep pass removes only folders THIS RUN emptied — proven from the journal,
+# never from "it looks empty" — so a folder someone made by hand is untouchable,
+# and every removal rides the same rollback as the moves.
+
+@test "sweep removes only the folders this run emptied" {
+    plan
+    apply_plan
+    mkdir -p "$LIB/A Empty Placeholder"      # someone's folder, never a move source
+    run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"swept"* ]]
+    [ ! -e "$LIB/AMERICAN POP" ]
+    [ ! -e "$LIB/VHS 2 (2013)" ]
+    [ ! -e "$LIB/Alien (1979)/Featurettes" ]
+    [ ! -e "$LIB/Alien (1979)/subs" ]
+    [ ! -e "$LIB/Marnie (1964)/Exrtas" ]
+    # a folder the run never moved a file out of survives, empty or not
+    [ -d "$LIB/A Empty Placeholder" ]
+}
+
+@test "sweep without --apply reports the husks and changes nothing" {
+    plan
+    apply_plan
+    local before jlines
+    before=$(find "$LIB" -type d | wc -l)
+    jlines=$(wc -l < "$JOURNAL")
+    run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL"
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"--apply"* ]]
+    [ "$(find "$LIB" -type d | wc -l)" -eq "$before" ]
+    [ "$(wc -l < "$JOURNAL")" -eq "$jlines" ]
+    [ -e "$LIB/AMERICAN POP" ]
+}
+
+@test "a folder that still holds content is never swept" {
+    plan
+    apply_plan
+    run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
+    [ "$status" -eq 0 ]
+    # the film folder, the folder the run created, and the piles all stay
+    [ -d "$LIB/Marnie (1964)" ]
+    [ -e "$LIB/Marnie (1964)/Marnie (1964).mkv" ]
+    [ -d "$LIB/V-H-S-2 (2013)" ]
+    [ -d "$LIB/_unsorted bonus/extras" ]
+}
+
+@test "rollback after a sweep brings the folders and the files back" {
+    plan
+    apply_plan
+    run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
+    [ "$status" -eq 0 ]
+    [ ! -e "$LIB/AMERICAN POP" ]
+    run python3 "$APPLY" rollback --root "$LIB" --journal "$JOURNAL" --apply
+    [ "$status" -eq 0 ]
+    [ -e "$LIB/AMERICAN POP/AMERICAN.POP.2023.1080p.WEB-DL.mkv" ]
+    [ -e "$LIB/VHS 2 (2013)/VHS.2.2013.1080p.BluRay.x265.mkv" ]
+    [ -e "$LIB/Alien (1979)/Featurettes/A Bug And A Bite - Making Alien.mkv" ]
+    [ -e "$LIB/Alien (1979)/subs/3_Eng.srt" ]
+}
+
+@test "a second sweep has nothing left to do and says so" {
+    plan
+    apply_plan
+    run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
+    [ "$status" -eq 0 ]
+    local before
+    before=$(find "$LIB" -type d | wc -l)
+    run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"swept: 0"* ]]
+    [ "$(find "$LIB" -type d | wc -l)" -eq "$before" ]
+}
+
+@test "sweep never follows a symlink and never removes the folder it guards" {
+    plan
+    apply_plan
+    mkdir -p "$LIB/Elsewhere"
+    at "$LIB/Elsewhere/keep.txt" 10
+    ln -s "$LIB/Elsewhere" "$LIB/AMERICAN POP/pointer"
+    run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
+    [ "$status" -eq 0 ]
+    # a link is content, not nothing: the husk it sits in survives untouched
+    [ -d "$LIB/AMERICAN POP" ]
+    [ -L "$LIB/AMERICAN POP/pointer" ]
+    [ -e "$LIB/Elsewhere/keep.txt" ]
+}
+
+@test "sweep refuses a journal that belongs to another library" {
+    plan
+    apply_plan
+    mkdir -p "$BATS_TEST_TMPDIR/Other"
+    run python3 "$APPLY" sweep --root "$BATS_TEST_TMPDIR/Other" \
+        --journal "$JOURNAL" --apply
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"outside"* ]]
+    [ -d "$LIB/AMERICAN POP" ]
+}