dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(media): sweep removes the folders a completed run emptied #94

Merged
dank merged 1 commits from feat/media-sweep into main 2026-10-30 20:04:24 -07:00
Conversation 0 Commits 1 Files Changed 3
+256 -17
3 changed files with 256 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/media-apply.py
+94 -6
View File
@@ -10,7 +10,7 @@ leaves either an intent whose move happened or an intent whose move never ran,
and both are recognised on the next run. A file that moved with no journal line
cannot have been moved by this tool, and saying so is better than guessing.
Three subcommands:
Four subcommands:
apply --plan PLAN [--root ROOT] [--journal J] [--apply]
no --apply => report only and write NOTHING, not even folders
@@ -19,6 +19,10 @@ Three subcommands:
removed only when empty, never anything else
verify --plan PLAN --journal J [--root ROOT]
every planned file is at its target with the journalled byte count
sweep --journal J [--root ROOT] [--apply]
removes the folders THIS RUN emptied — the journal has to show a
file that moved out of one, so a folder someone made by hand can
never be removed by this tool, and rollback puts them back
Refusals (exit 2, before the first rename, with nothing touched): a target
already occupied by a foreign file, a source that has gone, a duplicate target,
@@ -228,10 +232,87 @@ def do_verify(root, journal, moves, keeps):
return 0
def sweep_targets(root, journal):
"""The folders this run emptied, deepest first — never one it did not touch.
Eligibility is evidence, not appearance. A folder is a target only when the
journal shows a file moved out of it or out of something beneath it, it holds
nothing but empty directories now, it is not a folder this run created, and no
journalled destination lives under it. A symlink anywhere inside disqualifies
the whole subtree: this tool never follows a link, and deleting the folder
that holds one would delete someone's mount point.
"""
moves = [r for r in journal.records if r.get("op") == "move"]
made = {r["dst"] for r in journal.records if r.get("op") == "mkdir"}
gone = {r["path"] for r in journal.records if r.get("op") == "rmdir"}
dsts = [r["dst"] for r in moves]
marked = set()
for r in moves:
d = os.path.dirname(r["src"])
while d.startswith(root + os.sep):
marked.add(d)
d = os.path.dirname(d)
def removable(d):
if d in made or d in gone or os.path.islink(d):
return False
if any(p == d or p.startswith(d + os.sep) for p in dsts):
return False
try:
entries = os.listdir(d)
except OSError:
return False
for e in entries:
c = os.path.join(d, e)
if os.path.islink(c) or not os.path.isdir(c) or not removable(c):
return False
return True
return sorted((d for d in marked if removable(d)),
key=lambda q: (-q.count(os.sep), q))
def do_sweep(args, root, journal):
"""Remove the empty husks an apply left standing at the library root."""
moves = [r for r in journal.records if r.get("op") == "move"]
if not moves:
raise Refuse("the journal holds no moves — a sweep follows an apply, it"
" does not guess at emptiness")
outside = sorted({p for r in moves for p in (r["src"], r["dst"])
if not (p == root or p.startswith(root + os.sep))})
if outside and not args.force:
raise Refuse(f"{len(outside)} journalled paths are outside --root {root}"
f" (first: {outside[0]}) — this journal belongs to another"
" library")
targets = sweep_targets(root, journal)
if not targets:
print("swept: 0 folders — nothing this run emptied is still standing empty")
return 0
if not args.apply:
print(f"dry run: {len(targets)} folders this run emptied would be removed"
" (pass --apply to run)")
for d in targets[:5]:
print(f" would remove {d}")
return 0
removed = 0
for d in targets:
journal.record(op="rmdir", path=d)
try:
os.rmdir(d)
except OSError as e:
print(f"left in place: {d} ({e})")
continue
removed += 1
print(f"swept: {removed} folders removed, {len(targets) - removed} left in place")
print(f"journal: {journal.path} — rollback recreates them with their files")
return 0
def do_rollback(args, root, journal):
"""Undo every journalled move, newest first, after a full preflight."""
moves = [r for r in journal.records if r.get("op") == "move"]
made = [r["dst"] for r in journal.records if r.get("op") == "mkdir"]
swept = [r["path"] for r in journal.records if r.get("op") == "rmdir"]
problems = []
for r in reversed(moves):
src, dst = r["src"], r["dst"]
@@ -255,9 +336,14 @@ def do_rollback(args, root, journal):
return 2
if not args.apply:
print(f"dry run: would move {len(moves)} files back to their census paths"
" (pass --apply)")
f" and recreate {len(set(swept))} swept folders (pass --apply)")
return 0
undone = 0
recreated = 0
for d in sorted(set(swept), key=lambda q: q.count(os.sep)):
if not os.path.isdir(d):
os.makedirs(d, exist_ok=True)
recreated += 1
for r in reversed(moves):
if os.path.exists(r["dst"]) and not os.path.exists(r["src"]):
os.makedirs(os.path.dirname(r["src"]), exist_ok=True)
@@ -275,13 +361,13 @@ def do_rollback(args, root, journal):
journal.record(op="rollback", undone=undone, dirs_removed=dirs_gone,
t=datetime.datetime.now(datetime.timezone.utc).isoformat())
print(f"rolled back: {undone} files back where the census found them,"
f" {dirs_gone} empty folders removed")
f" {dirs_gone} empty folders removed, {recreated} swept folders recreated")
return 0
def main():
ap = argparse.ArgumentParser(description="apply or undo a media relayout plan")
ap.add_argument("cmd", choices=["apply", "rollback", "verify"])
ap.add_argument("cmd", choices=["apply", "rollback", "verify", "sweep"])
ap.add_argument("--plan")
ap.add_argument("--root", default=DEFAULT_ROOT)
ap.add_argument("--journal")
@@ -295,9 +381,9 @@ def main():
journal_path = args.journal or journal_default(root)
journal = Journal(journal_path, args.apply)
try:
if args.cmd in ("rollback", "verify") and not os.path.exists(journal_path):
if args.cmd in ("rollback", "verify", "sweep") and not os.path.exists(journal_path):
raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
if args.cmd == "rollback" and not args.force and journal.rolled_back:
if args.cmd in ("rollback", "sweep") and not args.force and journal.rolled_back:
raise Refuse("this journal has already been rolled back"
" (--force to replay it)")
if args.cmd in ("apply", "verify") and not args.plan:
@@ -308,6 +394,8 @@ def main():
return do_apply(args, root, journal, mkdirs, keeps, moves)
if args.cmd == "verify":
return do_verify(root, journal, moves, keeps)
if args.cmd == "sweep":
return do_sweep(args, root, journal)
return do_rollback(args, root, journal)
finally:
journal.close()
serverAppConfigs-docs/Dankserver/Media Relayout.md
+60 -11
View File
@@ -19,7 +19,7 @@ file.
|---|---|---|
| `scripts/media-census.py` | reads headers only | `enum`, `probed`, `plex` TSVs |
| `scripts/media-plan.py` | no — TSVs in, TSVs out | the plan + the exception report |
| `scripts/media-apply.py` | **renames, never copies** | the moves, plus a journal |
| `scripts/media-apply.py` | **renames, never copies**; sweeps only what a run emptied | the moves, plus a journal |
The seam is `tests/media_relayout.bats`; the target contract is quoted from
[jellyfin.org/docs/general/server/media/movies](https://jellyfin.org/docs/general/server/media/movies)
@@ -61,17 +61,58 @@ that would be filed wrong, and the plan is not run until it is understood.
```bash
scripts/media-apply.py apply --plan /tmp/jf-plan.tsv --root /mnt/entropy/Movies # dry, writes nothing
scripts/media-apply.py apply --plan /tmp/jf-plan.tsv --root /mnt/entropy/Movies --apply
scripts/media-apply.py verify --plan /tmp/jf-plan.tsv --journal …/media-relayout.jsonl
scripts/media-apply.py rollback --journal …/media-relayout.jsonl --apply
scripts/media-apply.py verify --plan /tmp/jf-plan.tsv --journal /mnt/entropy/media-relayout.jsonl
scripts/media-apply.py sweep --journal /mnt/entropy/media-relayout.jsonl --root /mnt/entropy/Movies # dry
scripts/media-apply.py sweep --journal /mnt/entropy/media-relayout.jsonl --root /mnt/entropy/Movies --apply
scripts/media-apply.py rollback --journal /mnt/entropy/media-relayout.jsonl --root /mnt/entropy/Movies --apply
```
After it lands, both servers need a scan or the change is invisible:
[[Jellyfin]] *Library → Scan All Libraries*, and Plex's library scan
(`podman exec plex …` or the web UI). `verify` reporting empty folders is normal
for the folders the piles own; they are left in place per the deletion policy.
An apply also leaves the release folders it emptied standing where they were —
the live run left 1675 of them at the library root — which is what `sweep`
removes, dry-run first, from the same journal.
After it lands, both servers need a scan or the change is invisible.
```bash
# Run this from the host shell through `podman exec`, never from a neighbour
# container: Plex's HTTPD closes without a single byte on any request that
# arrives from another container's netns (10.89.x.x reads as "empty reply", not
# as a 401). The token rides stdin, never argv, where any process on the host
# could read it out of /proc/<pid>/cmdline.
umask 077
podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \
'/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok
podman exec -i plex sh -c 'read -r T
for k in $(curl -s -H "X-Plex-Token: $T" -H "Accept: application/json" \
http://127.0.0.1:32400/library/sections | grep -o "\"key\":\"[0-9]*\"" | cut -d\" -f4); do
curl -s -o /dev/null -w "plex section $k: %{http_code}\n" -X POST \
-H "X-Plex-Token: $T" "http://127.0.0.1:32400/library/sections/$k/refresh"
done' </tmp/px.tok
```
[[Jellyfin]] has no credential on this box that can trigger a scan: the two keys
in its database are Seerr's, unused since 2026-09-01, and answer 401 to every
call. Mint one (*Dashboard → Advanced → API Keys*, Administrator) and store it in
`~/.config/secrets/jellyfin`, or press *Library → Scan All Libraries* in the web
UI. A scripted scan passes the key in a mode-600 curl config, never in a header
argument:
```bash
umask 077; printf 'header = "X-Emby-Token: %s"\n' "$(cat ~/.config/secrets/jellyfin)" >/tmp/jf.hdr
curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n" \
-X POST http://127.0.0.1:8096/Library/Refresh
```
## Laws
- **The journal lives beside the library, never in `/tmp`.** The live run's
`/mnt/entropy/media-relayout.jsonl` is the only rollback instrument; a journal
a reboot clears is not an instrument.
- **An apply breaks [[Plex]] playback for every moved item until its section is
rescanned.** The database keeps the release-folder path, so streams fail
`Couldn't find the file to stream` while the scanner works through the section.
Trigger the scans in the same window as the apply, and never read a 404 in that
window as a lost file — the file is where the journal says it is.
- **Plex's matched title outranks the filename**, and a Plex year of `0` means
*indexed but never matched* — trusting it named folders `Title (0)`, which is
a Jellyfin item with no year, no poster and no identity. A year that is not a
@@ -110,14 +151,22 @@ for the folders the piles own; they are left in place per the deletion policy.
duplicate destination stops the run *before the first rename*, so the operator
fixes one thing and re-runs instead of untangling a half-applied plan.
`--force` relaxes only the opinion guards; it never permits overwriting.
- **A sweep deletes only what the journal proves the run emptied.** A folder is
eligible when a journalled file moved out of it or out of something under it,
it now holds nothing but empty directories, no journalled destination lives
beneath it, and it is not a folder the run created. A folder nobody moved —
someone's placeholder — cannot be selected at all, and a symlink anywhere
inside disqualifies the subtree, because the thing behind a link is not this
tool's to delete. Every removal is journalled as `rmdir` and `rollback`
recreates the folder, so the sweep is undone by the same command as the moves.
## Rollback
One command, from the journal of the run:
`media-apply.py rollback --journal …/media-relayout.jsonl --apply`. It restores
every source path, removes only the folders that run created *and that are now
empty*, and refuses everything if any moved file cannot be accounted for. A
rolled-back journal refuses to roll back twice.
`media-apply.py rollback --journal …/media-relayout.jsonl --apply`. It recreates
any folder a sweep removed, restores every source path, removes only the folders
that run created *and that are now empty*, and refuses everything if any moved
file cannot be accounted for. A rolled-back journal refuses to roll back twice.
## See Also
tests/media_relayout.bats
+102
View File
@@ -331,3 +331,105 @@ PY
# the refusal is all-or-nothing: another film did not move back
[ -e "$LIB/Marnie (1964)/Marnie (1964).mkv" ]
}
# ---------------------------------------------------------------- sweep
#
# A relayout leaves the release folders it emptied standing at the library root:
# 1675 of them after the live run, each holding nothing but empty subfolders. The
# sweep pass removes only folders THIS RUN emptied — proven from the journal,
# never from "it looks empty" — so a folder someone made by hand is untouchable,
# and every removal rides the same rollback as the moves.
@test "sweep removes only the folders this run emptied" {
plan
apply_plan
mkdir -p "$LIB/A Empty Placeholder" # someone's folder, never a move source
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
[ "$status" -eq 0 ]
[[ "$output" == *"swept"* ]]
[ ! -e "$LIB/AMERICAN POP" ]
[ ! -e "$LIB/VHS 2 (2013)" ]
[ ! -e "$LIB/Alien (1979)/Featurettes" ]
[ ! -e "$LIB/Alien (1979)/subs" ]
[ ! -e "$LIB/Marnie (1964)/Exrtas" ]
# a folder the run never moved a file out of survives, empty or not
[ -d "$LIB/A Empty Placeholder" ]
}
@test "sweep without --apply reports the husks and changes nothing" {
plan
apply_plan
local before jlines
before=$(find "$LIB" -type d | wc -l)
jlines=$(wc -l < "$JOURNAL")
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL"
[ "$status" -eq 0 ]
[[ "$output" == *"--apply"* ]]
[ "$(find "$LIB" -type d | wc -l)" -eq "$before" ]
[ "$(wc -l < "$JOURNAL")" -eq "$jlines" ]
[ -e "$LIB/AMERICAN POP" ]
}
@test "a folder that still holds content is never swept" {
plan
apply_plan
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
[ "$status" -eq 0 ]
# the film folder, the folder the run created, and the piles all stay
[ -d "$LIB/Marnie (1964)" ]
[ -e "$LIB/Marnie (1964)/Marnie (1964).mkv" ]
[ -d "$LIB/V-H-S-2 (2013)" ]
[ -d "$LIB/_unsorted bonus/extras" ]
}
@test "rollback after a sweep brings the folders and the files back" {
plan
apply_plan
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
[ "$status" -eq 0 ]
[ ! -e "$LIB/AMERICAN POP" ]
run python3 "$APPLY" rollback --root "$LIB" --journal "$JOURNAL" --apply
[ "$status" -eq 0 ]
[ -e "$LIB/AMERICAN POP/AMERICAN.POP.2023.1080p.WEB-DL.mkv" ]
[ -e "$LIB/VHS 2 (2013)/VHS.2.2013.1080p.BluRay.x265.mkv" ]
[ -e "$LIB/Alien (1979)/Featurettes/A Bug And A Bite - Making Alien.mkv" ]
[ -e "$LIB/Alien (1979)/subs/3_Eng.srt" ]
}
@test "a second sweep has nothing left to do and says so" {
plan
apply_plan
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
[ "$status" -eq 0 ]
local before
before=$(find "$LIB" -type d | wc -l)
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
[ "$status" -eq 0 ]
[[ "$output" == *"swept: 0"* ]]
[ "$(find "$LIB" -type d | wc -l)" -eq "$before" ]
}
@test "sweep never follows a symlink and never removes the folder it guards" {
plan
apply_plan
mkdir -p "$LIB/Elsewhere"
at "$LIB/Elsewhere/keep.txt" 10
ln -s "$LIB/Elsewhere" "$LIB/AMERICAN POP/pointer"
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
[ "$status" -eq 0 ]
# a link is content, not nothing: the husk it sits in survives untouched
[ -d "$LIB/AMERICAN POP" ]
[ -L "$LIB/AMERICAN POP/pointer" ]
[ -e "$LIB/Elsewhere/keep.txt" ]
}
@test "sweep refuses a journal that belongs to another library" {
plan
apply_plan
mkdir -p "$BATS_TEST_TMPDIR/Other"
run python3 "$APPLY" sweep --root "$BATS_TEST_TMPDIR/Other" \
--journal "$JOURNAL" --apply
[ "$status" -ne 0 ]
[[ "$output" == *"outside"* ]]
[ -d "$LIB/AMERICAN POP" ]
}