diff --git a/AGENTS.md b/AGENTS.md index c7c952f..eb0b9ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -807,6 +807,7 @@ per-attempt page and no per-swap ledger anywhere else. | `frank-tts` — cloned-voice speech, Dankclaw | `serverAppConfigs-docs/Dankclaw/Frank TTS.md` | | Dankclaw kernel posture (`iommu=pt` since 2026-10-18) | `serverAppConfigs-docs/LLM/dankclaw-iommu-translation-off.md` | | Deployment flow | `serverAppConfigs-docs/Architecture/Deployment Flow.md` | +| Movie-library relayout (Jellyfin format, Dankserver) | `serverAppConfigs-docs/Dankserver/Media Relayout.md` | There is no page for a retired endpoint and no change diary. A payload swap appends one row to the live endpoint's **Eras** table; when the endpoint itself @@ -848,5 +849,6 @@ history. | Writing a doc | **One owner per fact** — edit the endpoint/service page; everywhere else links. Never restate a number. | | Benchmarking (only when ordered) | **Stop and mask the service first**, bench, unmask, restart, verify health. | | Deleting a file | **Move it to trash** — `.AGENTTRASH/` (in-repo) or `~/.AGENTTRASH/YYYYMMDD-topic`. Never `rm`. | +| Relayout the movie library | **Census → plan → read the exceptions → `scripts/media-apply.py`** — renames only, journal before every move; never hand-rename, never copy. | | Tests exist | **Run them** — verify tests pass after every change. Do not commit failing tests. | | Real secrets | **Never commit** — host `.env` files; `.env.example` holds placeholders only. | diff --git a/scripts/media-apply.py b/scripts/media-apply.py new file mode 100755 index 0000000..150d171 --- /dev/null +++ b/scripts/media-apply.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +"""media-apply.py — the only thing that moves movie-library files. + +Reads the plan written by scripts/media-plan.py (rows: `action src dst kind`, +plus `mkdir` rows) and performs it with rename(2) — bytes are never copied, so +an interrupted run has destroyed nothing. Every run is resumable and every run +is undoable: the journal is a write-ahead log, one JSON line per intent, fsynced +BEFORE the rename it authorises. That ordering is the whole design: a crash +leaves either an intent whose move happened or an intent whose move never ran, +and both are recognised on the next run. A file that moved with no journal line +cannot have been moved by this tool, and saying so is better than guessing. + +Three subcommands: + + apply --plan PLAN [--root ROOT] [--journal J] [--apply] + no --apply => report only and write NOTHING, not even folders + rollback --journal J [--root ROOT] [--apply] + back to the paths the census saw; folders this run created are + removed only when empty, never anything else + verify --plan PLAN --journal J [--root ROOT] + every planned file is at its target with the journalled byte count + +Refusals (exit 2, before the first rename, with nothing touched): a target +already occupied by a foreign file, a source that has gone, a duplicate target, +a path outside --root, a journal that was already rolled back, a move that would +cross a filesystem (rename cannot do it and a silent copy at 24 TB would not). + +--force relaxes only the guards that are opinion (journal already rolled back, +paths outside --root); it never allows overwriting a file. + +The journal defaults to `media-relayout.jsonl` BESIDE the library root, never +inside it: a log inside the library is a file Jellyfin will index. +""" +import argparse +import datetime +import json +import os +import sys + +DEFAULT_ROOT = os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies") + + +class Refuse(Exception): + """A gate the operator has to see; every refusal leaves the library alone.""" + + +def journal_default(root): + return os.path.join(os.path.dirname(os.path.abspath(root)), "media-relayout.jsonl") + + +class Journal: + """Append-only JSONL of intents. Writes are fsynced before the tool acts.""" + + def __init__(self, path, writing): + self.path, self.writing = path, writing + self.records = [] + if os.path.exists(path): + for line in open(path): + line = line.strip() + if line: + self.records.append(json.loads(line)) + self._fh = None + + def already(self, dst, src): + """An intent for this exact (src, dst) was written by an earlier run.""" + return any(r.get("op") == "move" and r.get("dst") == dst + and r.get("src") == src for r in self.records) + + @property + def rolled_back(self): + return any(r.get("op") == "rollback" for r in self.records) + + def record(self, **fields): + self.records.append(fields) + if not self.writing: + return + if self._fh is None: + self._fh = open(self.path, "a") + self._fh.write(json.dumps(fields) + "\n") + self._fh.flush() + os.fsync(self._fh.fileno()) + + def close(self): + if self._fh: + self._fh.close() + + +def read_plan(path, root, force): + """Plan rows: (mkdirs, keeps, moves), header dropped, all paths under root.""" + mkdirs, keeps, moves = [], [], [] + with open(path) as fh: + header = fh.readline() + if not header.startswith("action\t"): + raise Refuse(f"{path} is not a plan (first column is " + f"{header.split(chr(9))[0]!r}, expected 'action') —" + " regenerate it with scripts/media-plan.py") + for line in fh: + f = line.rstrip("\n").split("\t") + if f[0] == "mkdir": + mkdirs.append(f[2]) + elif f[0] == "keep": + keeps.append(f[1]) + elif f[0] == "move": + moves.append((f[1], f[2], f[3] if len(f) > 3 else "")) + if not force: + for p in mkdirs + keeps + [p for m in moves for p in m[:2]]: + if p and not (p == root or p.startswith(root + os.sep)): + raise Refuse(f"{p} is outside --root {root}" + " (a plan built for another library?)") + return mkdirs, keeps, moves + + +def do_apply(args, root, journal, mkdirs, keeps, moves): + """Preflight everything, then folders, then renames. Never a partial start.""" + problems = [] + for p in keeps: + if not os.path.exists(p): + problems.append(f"missing: keep row source is gone: {p}") + seen = {} + for src, dst, _kind in moves: + if src == dst: + problems.append(f"no-op: move row whose target is its source: {src}") + if dst in seen: + problems.append(f"double-claim: {dst} claimed by {seen[dst]} and {src}") + seen[dst] = src + if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src): + continue # this intent already ran + if not os.path.exists(src): + if os.path.exists(dst): + problems.append(f"missing: {src} is gone and {dst} carries no journal" + " line — it moved outside this tool, refusing to guess") + else: + problems.append(f"missing: {src} is gone and {dst} is not there either") + continue + if os.path.exists(dst): + problems.append(f"occupied: target already holds another file: {dst}") + if problems: + print(f"refusing: {len(problems)} problem(s), nothing moved") + for p in sorted(problems)[:20]: + print(" " + p) + if len(problems) > 20: + print(f" ... and {len(problems) - 20} more") + return 2 + + if not args.apply: + print(f"dry run: {len(moves)} moves, {len(mkdirs)} folders, {len(keeps)} files stay" + " — nothing written (pass --apply to run)") + for src, dst, _k in moves[:5]: + print(f" would move {src} -> {dst}") + return 0 + + made = {r.get("dst") for r in journal.records if r.get("op") == "mkdir"} + dirs_new = 0 + for d in sorted(mkdirs, key=lambda q: q.count(os.sep)): + if d in made or os.path.isdir(d): + continue + journal.record(op="mkdir", dst=d) + os.makedirs(d, exist_ok=True) + dirs_new += 1 + + # same-filesystem gate: rename cannot cross a device, and a silent copy of a + # library this size is not an option anyone would choose. + cross = [] + for src, dst, _k in moves: + if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src): + continue + parent = os.path.dirname(dst) + if os.path.exists(parent) and os.stat(parent).st_dev != os.stat(src).st_dev: + cross.append(dst) + if cross: + print(f"refusing: {len(cross)} moves would cross a filesystem" + f" (first: {cross[0]}) — this tool only renames") + return 2 + + moved = done = 0 + for src, dst, _kind in moves: + if journal.already(dst, src): + if os.path.exists(dst) and not os.path.exists(src): + done += 1 + continue + if os.path.exists(src) and not os.path.exists(dst): + os.rename(src, dst) # the intent ran, the move did not + moved += 1 + continue + st = os.stat(src) + journal.record(op="move", src=src, dst=dst, bytes=st.st_size, mode=st.st_mode, + mtime=st.st_mtime, ino=st.st_ino, dev=st.st_dev) + os.rename(src, dst) + moved += 1 + print(f"applied: {moved} moved, {done} already done in an earlier run," + f" {dirs_new} folders created, {len(keeps)} files stayed put") + print(f"journal: {journal.path} — undo with: media-apply.py rollback" + f" --journal {journal.path} --root {root} --apply") + return 0 + + +def do_verify(root, journal, moves, keeps): + """Every planned file sits at its target carrying the journalled bytes.""" + journalled = {(r["src"], r["dst"]): r for r in journal.records + if r.get("op") == "move"} + todo = moves or [(r["src"], r["dst"], "") for r in journal.records + if r.get("op") == "move"] + bad, missing, bytes_total = [], [], 0 + for src, dst, _kind in todo: + rec = journalled.get((src, dst)) + if not os.path.exists(dst): + missing.append(dst) + continue + size = os.path.getsize(dst) + bytes_total += size + if rec and rec.get("bytes") not in (None, size): + bad.append(f"{dst}: {size} bytes on disk, {rec['bytes']} at move time") + if os.path.exists(src) and src != dst: + bad.append(f"{src}: still in place as well as at {dst}") + missing += [p for p in keeps if not os.path.exists(p)] + if bad or missing: + print(f"verify FAILED: {len(missing)} absent, {len(bad)} wrong") + for line in (missing + bad)[:20]: + print(" " + line) + return 2 + empty = [r["dst"] for r in journal.records + if r.get("op") == "mkdir" and os.path.isdir(r["dst"]) + and not os.listdir(r["dst"])] + print(f"verified: {len(todo)} files at their targets" + f" ({bytes_total / 2**40:.2f} TiB), {len(keeps)} kept") + if empty: + print(f"note: {len(empty)} folders are empty (left in place — see the trash law)") + return 0 + + +def do_rollback(args, root, journal): + """Undo every journalled move, newest first, after a full preflight.""" + moves = [r for r in journal.records if r.get("op") == "move"] + made = [r["dst"] for r in journal.records if r.get("op") == "mkdir"] + problems = [] + for r in reversed(moves): + src, dst = r["src"], r["dst"] + if not args.force and not (src.startswith(root + os.sep) + or dst.startswith(root + os.sep)): + problems.append(f"outside --root: {dst}") + continue + if os.path.exists(dst) and os.path.exists(src): + problems.append(f"occupied: both paths exist, refusing to choose:" + f" {src} | {dst}") + elif os.path.exists(dst): + pass + elif os.path.exists(src): + continue # already undone + else: + problems.append(f"missing: {dst} is gone and {src} never came back") + if problems: + print(f"refusing: {len(problems)} problem(s), nothing moved back") + for p in problems[:20]: + print(" " + p) + return 2 + if not args.apply: + print(f"dry run: would move {len(moves)} files back to their census paths" + " (pass --apply)") + return 0 + undone = 0 + for r in reversed(moves): + if os.path.exists(r["dst"]) and not os.path.exists(r["src"]): + os.makedirs(os.path.dirname(r["src"]), exist_ok=True) + os.rename(r["dst"], r["src"]) + undone += 1 + dirs_gone = 0 + for d in sorted(set(made), key=lambda q: -q.count(os.sep)): + if d == root or not os.path.isdir(d) or os.listdir(d): + continue + try: + os.rmdir(d) + dirs_gone += 1 + except OSError as e: + print(f"left in place: {d} ({e})") + journal.record(op="rollback", undone=undone, dirs_removed=dirs_gone, + t=datetime.datetime.now(datetime.timezone.utc).isoformat()) + print(f"rolled back: {undone} files back where the census found them," + f" {dirs_gone} empty folders removed") + return 0 + + +def main(): + ap = argparse.ArgumentParser(description="apply or undo a media relayout plan") + ap.add_argument("cmd", choices=["apply", "rollback", "verify"]) + ap.add_argument("--plan") + ap.add_argument("--root", default=DEFAULT_ROOT) + ap.add_argument("--journal") + ap.add_argument("--apply", action="store_true", help="actually change the library") + ap.add_argument("--force", action="store_true", help="relax the opinion guards") + args = ap.parse_args() + + root = os.path.abspath(args.root) + if not os.path.isdir(root): + raise Refuse(f"no library at {root}") + journal_path = args.journal or journal_default(root) + journal = Journal(journal_path, args.apply) + try: + if args.cmd in ("rollback", "verify") and not os.path.exists(journal_path): + raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}") + if args.cmd == "rollback" and not args.force and journal.rolled_back: + raise Refuse("this journal has already been rolled back" + " (--force to replay it)") + if args.cmd in ("apply", "verify") and not args.plan: + raise Refuse(f"{args.cmd} needs --plan") + mkdirs, keeps, moves = (read_plan(args.plan, root, args.force) + if args.plan else ([], [], [])) + if args.cmd == "apply": + return do_apply(args, root, journal, mkdirs, keeps, moves) + if args.cmd == "verify": + return do_verify(root, journal, moves, keeps) + return do_rollback(args, root, journal) + finally: + journal.close() + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Refuse as r: + print(f"refusing: {r}") + sys.exit(2) diff --git a/scripts/media-census.py b/scripts/media-census.py new file mode 100755 index 0000000..3063476 --- /dev/null +++ b/scripts/media-census.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +r"""Read-only census of a movie library — the inputs a relayout plan is built from. + + media-census.py enumerate [--root DIR] [--out TSV] + media-census.py probe [--enum TSV] [--out TSV] [--jobs N] + media-census.py plex [--root DIR] [--db PATH] [--out TSV] + +Run it on the machine that mounts the library. Nothing here writes inside the +library and nothing is deleted; the only outputs are the three TSVs below, and +scripts/media-plan.py reads them instead of touching media. + + enum bytes ext path every file under the root + probed bytes path dur vcodec WxH fps vbit + pix audio subs spare format + one row per video-container file; `audio` is "codec:lang+codec:lang", + `subs` the subtitle-stream count, `spare` the audio streams carrying + neither a default nor a forced tag + plex file leaf_title … (see PLEX_COLS) + the titles Plex already matched — the authority over any filename guess + +`probe` is the slow step (every container is opened for its header, ~10 min for +4,800 files on a spindle array at 6 workers) so it is `nice`d: a census beside a +live service must not starve it. `enumerate` is a stat pass, so re-censusing +before a plan is cheap — and a plan is only ever as good as these three files. +""" +import argparse +import csv +import json +import os +import re +import sqlite3 +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor + +# Containers ffprobe can read. .iso/.img are deliberately NOT here: a disc image is +# a mounted volume, not a stream, and the planner treats it as a movie file instead. +PROBE_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg", + ".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".bdmv", ".vob"} +# Trash and thumbnail caches are not library content, and this repo's trash policy +# means .AGENTTRASH can sit inside a movie folder at any time. +SKIP_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir|lost\+found)(/|$)") +PLEX_DB = ("/var/lib/plexmediaserver/Library/Application Support/Plex Media Server" + "/Plug-in Support/Databases/com.plexapp.plugins.library.db") +PLEX_COLS = ["file", "leaf_title", "leaf_orig", "leaf_edition", "leaf_year", + "parent_title", "parent_orig", "parent_year", "parent_edition", + "container", "vcodec", "acodec", "width", "height"] +PLEX_SQL = """ +select p.file, coalesce(mi.title,''), coalesce(mi.original_title,''), + coalesce(mi.edition_title,''), coalesce(mi.year,0), + coalesce(par.title,''), coalesce(par.original_title,''), + coalesce(par.year,0), coalesce(par.edition_title,''), + m.container, m.video_codec, m.audio_codec, m.width, m.height +from media_parts p +join media_items m on m.id = p.media_item_id +join metadata_items mi on mi.id = m.metadata_item_id +left join metadata_items par on par.id = mi.parent_id +where p.file like ? and p.deleted_at is null and mi.deleted_at is null +""" + + +def enumerate_lib(root, out): + """One row per real file under root: size, extension, path.""" + rows = [] + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames + if not SKIP_DIR.search(os.path.join(dirpath, d))] + for fn in filenames: + p = os.path.join(dirpath, fn) + if SKIP_DIR.search(p): + continue + try: + st = os.lstat(p) + except OSError: + continue + if not os.path.isfile(p) or os.path.islink(p): + continue # links and sockets are not library content + rows.append((st.st_size, os.path.splitext(fn)[1].lower(), p)) + with open(out, "w") as fh: + for size, ext, p in rows: + fh.write(f"{size}\t{ext}\t{p}\n") + vid = [r for r in rows if r[1] in PROBE_EXT] + print(f"files={len(rows)} bytes={sum(r[0] for r in rows)} " + f"probes={len(vid)} probe_bytes={sum(r[0] for r in vid)} -> {out}") + + +def probe_one(path): + """ffprobe header read -> the probed TSV row (never the whole file).""" + try: + os.nice(19) # a census must never outrank the media server + except OSError: + pass + try: + size = os.stat(path).st_size + except OSError: + return None + cmd = ["ffprobe", "-v", "error", "-print_format", "json", + "-show_format", "-show_streams", path] + try: + done = subprocess.run(cmd, capture_output=True, text=True, timeout=180) + data = json.loads(done.stdout or "{}") + except (subprocess.TimeoutExpired, subprocess.SubprocessError, + json.JSONDecodeError, OSError): + data = {} + st = data.get("streams", []) + fmt = data.get("format", {}) + v = next((s for s in st if s.get("codec_type") == "video"), None) + aud = [s for s in st if s.get("codec_type") == "audio"] + subs = [s for s in st if s.get("codec_type") == "subtitle"] + fps = "-" + if v and v.get("r_frame_rate"): + try: + n, d = (int(x) for x in v["r_frame_rate"].split("/", 1)) + fps = f"{round(n / d * 100) / 100:g}" if d else "0" + except ValueError: + pass + + def num(x): + try: + return round(float(x)) + except (TypeError, ValueError): + return 0 + + return [size, path, num(fmt.get("duration")), + (v or {}).get("codec_name", "-"), + f"{v['width']}x{v['height']}" if v else "-", + fps, num(fmt.get("bit_rate")), (v or {}).get("pix_fmt", "-"), + "+".join(f"{a.get('codec_name', '?')}:{a.get('language', '?')}" for a in aud), + len(subs), + sum(1 for a in aud if not re.search( + r"default|forced", json.dumps(a.get("tags", {})), re.I)), + fmt.get("format_name", "-") or "-"] + + +def probe(enum, out, jobs): + """Probe every video-container row of the enum TSV, in parallel.""" + paths = [] + with open(enum) as fh: + for line in fh: + f = line.rstrip("\n").split("\t", 2) + if len(f) == 3 and f[1] in PROBE_EXT: + paths.append(f[2]) + print(f"probing {len(paths)} files with {jobs} workers", flush=True) + done = unreadable = 0 + with open(out, "w") as fh, ThreadPoolExecutor(max_workers=jobs) as pool: + for row in pool.map(probe_one, paths): + if row is None: + continue + if row[3] == "-": + unreadable += 1 # header unreadable: the row still gets written + done += 1 + fh.write("\t".join(str(x) for x in row) + "\n") + if done % 250 == 0: + print(f" {done}/{len(paths)}", flush=True) + print(f"probed={done} unreadable={unreadable} -> {out}") + + +def plex(root, db, out): + """Plex's matched titles for files under root, from its DB, read-only.""" + uri = f"file:{db}?mode=ro&immutable=1" + try: + con = sqlite3.connect(uri, uri=True) + except sqlite3.Error as e: + sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n" + "Plex runs rootless here: run this as the container's owner, never " + "with sudo — rootless storage is per-user, so root sees nothing.") + with open(out, "w", newline="") as fh: + w = csv.writer(fh, delimiter="\t") + w.writerow(PLEX_COLS) + n = 0 + for r in con.execute(PLEX_SQL, (root + "/%",)): + w.writerow([str(x).replace("\t", " ").replace("\n", " ") for x in r]) + n += 1 + print(f"plex-matched={n} -> {out}") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + root = {"default": os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies"), + "help": "library root"} + e = sub.add_parser("enumerate", help="stat every file under the root") + e.add_argument("--root", **root) + e.add_argument("--out", default="/tmp/media-enum.tsv") + p = sub.add_parser("probe", help="ffprobe every video container") + p.add_argument("--enum", default="/tmp/media-enum.tsv") + p.add_argument("--out", default="/tmp/media-probed.tsv") + p.add_argument("--jobs", type=int, default=6) + x = sub.add_parser("plex", help="export Plex's matched titles for the root") + x.add_argument("--root", **root) + x.add_argument("--db", default=PLEX_DB) + x.add_argument("--out", default="/tmp/plex-movies.tsv") + a = ap.parse_args() + if a.cmd == "enumerate": + enumerate_lib(a.root, a.out) + elif a.cmd == "probe": + probe(a.enum, a.out, a.jobs) + else: + plex(a.root, a.db, a.out) + + +if __name__ == "__main__": + main() diff --git a/scripts/media-plan.py b/scripts/media-plan.py new file mode 100755 index 0000000..acf0823 --- /dev/null +++ b/scripts/media-plan.py @@ -0,0 +1,882 @@ +#!/usr/bin/env python3 +r"""Plan a Jellyfin-format relayout of a movie library. DRY RUN ONLY: nothing moves. + + media-plan.py [--root DIR] [--enum TSV] [--probed TSV] [--plex TSV] + [--out TSV] [--exceptions TXT] + +Inputs are the TSVs scripts/media-census.py exports, plus the optional Plex +metadata-cache export whose clean titles outrank the filename parser; the title +authority for everything Plex never matched is scripts/media-titles.py beside this +file. No media file is opened, so planning the whole library costs seconds. + +PLAN is a TSV the executor (scripts/media-apply.py) replays verbatim — +`actionsrcdstkind` with action in mkdir | move | keep, ordered so +parents exist before children. EXCEPTIONS is the human review list: every file the +plan could not name a film for, every collision it refused, the READMEs the +executor must write into the piles it created. + +Target contract (jellyfin.org/docs/general/server/media/movies): + * one folder per movie, named Title (year) + * the video inside is named exactly like the folder + * extra copies of one film are versions: " -