From 441b908937a8b3e7a6af4f7596b0f9ef46a1539e Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 30 Oct 2026 23:15:32 -0700 Subject: [PATCH] feat(media): plan a TV relayout from the numbering Jellyfin already has The Movies relayout planned from Plex; the TV library has no Plex rows and 58k files whose names lie about which episode they hold. Jellyfin has already matched and numbered 47,676 of them, so the planner treats that numbering as the authority and the file name as the fallback: an episode number the name states, or a provider row, or nothing - and "nothing" parks the file under _unsorted with a reason instead of inventing a number. - media-census.py jellyfin: one TSV row per matched episode, its container path mapped back to the host, series name/year/TMDb id from the Series row (the episode row carries none of them). Both TSV writers now end lines with \n so a shell consumer sees the last field clean. - media-shows.py: the plan. Series folders take the provider's own spelling; one file claiming two episodes keeps both numbers; two files claiming one episode - the bigger keeps the standard name, the other parks; bonus material files under the series' extras folder, never as an episode; a long file inside the TV library leaves for Movies only when TMDb answers with a matching title, otherwise it parks. Re-running the plan on a library it already filed moves nothing. - tests/media_shows.bats: 15 behaviors, including the executor preflight and the re-plan stability law. --- scripts/media-census.py | 113 ++++++- scripts/media-shows.py | 656 ++++++++++++++++++++++++++++++++++++++++ tests/media_shows.bats | 271 +++++++++++++++++ 3 files changed, 1034 insertions(+), 6 deletions(-) create mode 100755 scripts/media-shows.py create mode 100755 tests/media_shows.bats diff --git a/scripts/media-census.py b/scripts/media-census.py index 3063476..e9e09ab 100755 --- a/scripts/media-census.py +++ b/scripts/media-census.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -r"""Read-only census of a movie library — the inputs a relayout plan is built from. +r"""Read-only census of a media library — the inputs a relayout plan is built from. media-census.py enumerate [--root DIR] [--out TSV] media-census.py probe [--enum TSV] [--out TSV] [--jobs N] media-census.py plex [--root DIR] [--db PATH] [--out TSV] + media-census.py jellyfin [--root DIR] [--db PATH] [--container PREFIX] [--out TSV] Run it on the machine that mounts the library. Nothing here writes inside the -library and nothing is deleted; the only outputs are the three TSVs below, and -scripts/media-plan.py reads them instead of touching media. +library and nothing is deleted; the only outputs are the TSVs below, and the +scripts/media-plan.py and scripts/media-shows.py read them instead of touching media. enum bytes ext path every file under the root probed bytes path dur vcodec WxH fps vbit @@ -18,10 +19,17 @@ scripts/media-plan.py reads them instead of touching media. plex file leaf_title … (see PLEX_COLS) the titles Plex already matched — the authority over any filename guess + jellyfin file series year season episode extra + tmdb (see JF_COLS) + one row per episode Jellyfin has matched and numbered, its path mapped + from the container's view back to the host; `year` and `tmdb` are the + SERIES' values, `extra` is Jellyfin's own extra type when the row is + bonus material rather than an episode + `probe` is the slow step (every container is opened for its header, ~10 min for 4,800 files on a spindle array at 6 workers) so it is `nice`d: a census beside a live service must not starve it. `enumerate` is a stat pass, so re-censusing -before a plan is cheap — and a plan is only ever as good as these three files. +before a plan is cheap — and a plan is only ever as good as these files. """ import argparse import csv @@ -57,6 +65,14 @@ join metadata_items mi on mi.id = m.metadata_item_id left join metadata_items par on par.id = mi.parent_id where p.file like ? and p.deleted_at is null and mi.deleted_at is null """ +JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db" +JF_COLS = ["file", "series", "year", "season", "episode", "extra", "tmdb"] +# Jellyfin stores the entity's .NET class as its type; the TV tree is these three. +JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode", + "MediaBrowser.Controller.Entities.TV.Season", + "MediaBrowser.Controller.Entities.TV.Series") +JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, IsVirtualItem + from BaseItems where Type = ? and Path like ?""") def enumerate_lib(root, out): @@ -165,7 +181,7 @@ def plex(root, db, out): "Plex runs rootless here: run this as the container's owner, never " "with sudo — rootless storage is per-user, so root sees nothing.") with open(out, "w", newline="") as fh: - w = csv.writer(fh, delimiter="\t") + w = csv.writer(fh, delimiter="\t", lineterminator="\n") w.writerow(PLEX_COLS) n = 0 for r in con.execute(PLEX_SQL, (root + "/%",)): @@ -174,6 +190,79 @@ def plex(root, db, out): print(f"plex-matched={n} -> {out}") +def jellyfin(root, db, container, out): + """The episodes Jellyfin has already numbered, for the library at root. + + Jellyfin stores each path as the CONTAINER sees it (its quadlet binds the + share at /data), so `container` is the in-container prefix that stands for + root and every exported path is mapped back to the host before it is written + — the planner reads host paths, because it plans moves on the array. + + The numbering is what matters, not the name: a release group's file name says + nothing about which episode a file is, and Jellyfin has already answered that + question against a provider. Season/episode come from the episode row; the + series name, its year and its TMDb id come from the Series row the episode + climbs to, because those are the only place Jellyfin keeps them. + """ + try: + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + except sqlite3.Error as e: + sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n" + "It is a WAL database: copy it with its -wal and -shm beside it and" + " point --db at the copy. Never sudo it — the data dir belongs to" + " the container's owner, and rootless storage is per-user.") + have = {r[1] for r in con.execute("pragma table_info(BaseItems)")} + need = {"Id", "ParentId", "Type", "Path", "IndexNumber", "ParentIndexNumber", + "ExtraType", "IsVirtualItem", "ProductionYear", "Data"} + if not need <= have: + sys.exit(f"{db} is not a Jellyfin library database (BaseItems has no" + f" {','.join(sorted(need - have))})") + node = {} + for i, pid, typ, name, year, data in con.execute( + "select Id, ParentId, Type, Name, ProductionYear, Data from BaseItems" + " where Type in (?,?,?)", JF_TV): + node[i] = (pid, typ, name, year, data) + + def series_of(item_id): + """Climb episode -> season -> series; empty when the chain is broken.""" + for _ in range(4): + if item_id not in node: + return "", "", "" + pid, typ, name, year, data = node[item_id] + if typ.endswith(".Series"): + try: + tmdb = (json.loads(data or "{}") + .get("ProviderIds") or {}).get("Tmdb", "") + except ValueError: + tmdb = "" + return name or "", year or "", tmdb + item_id = pid + return "", "", "" + + # The fleet's quadlets bind the share at /data, so the container's view of + # this library is /data/ unless told otherwise. + if not container: + container = "/data/" + os.path.basename(os.path.normpath(root)) + prefix = container.rstrip("/") + "/" + written = skipped = 0 + with open(out, "w", newline="") as fh: + w = csv.writer(fh, delimiter="\t", lineterminator="\n") + w.writerow(JF_COLS) + for iid, path, idx, pidx, extra, virt in con.execute(JF_SQL, + (JF_TV[0], prefix + "%")): + if virt or not path or not path.startswith(prefix): + skipped += 1 # a virtual item is a variant, not a file + continue + sname, syear, tmdb = series_of(iid) + w.writerow([root + "/" + path[len(prefix):], + sname.replace("\t", " "), syear, + "" if pidx is None else pidx, + "" if idx is None else idx, + (extra or "").replace("\t", " "), tmdb]) + written += 1 + print(f"episodes={written} virtual-or-blank={skipped} -> {out}") + + def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) sub = ap.add_subparsers(dest="cmd", required=True) @@ -190,13 +279,25 @@ def main(): x.add_argument("--root", **root) x.add_argument("--db", default=PLEX_DB) x.add_argument("--out", default="/tmp/plex-movies.tsv") + j = sub.add_parser("jellyfin", + help="export the episodes Jellyfin has numbered for the root") + j.add_argument("--root", default=os.environ.get("MEDIA_ROOT", + "/mnt/entropy/TVShows"), + help="TV library root on the host") + j.add_argument("--db", default=JF_DB) + j.add_argument("--container", default=None, + help="the in-container prefix that stands for --root" + " (default /data/)") + j.add_argument("--out", default="/tmp/jellyfin-tv.tsv") a = ap.parse_args() if a.cmd == "enumerate": enumerate_lib(a.root, a.out) elif a.cmd == "probe": probe(a.enum, a.out, a.jobs) - else: + elif a.cmd == "plex": plex(a.root, a.db, a.out) + else: + jellyfin(a.root, a.db, a.container, a.out) if __name__ == "__main__": diff --git a/scripts/media-shows.py b/scripts/media-shows.py new file mode 100755 index 0000000..44d52bb --- /dev/null +++ b/scripts/media-shows.py @@ -0,0 +1,656 @@ +#!/usr/bin/env python3 +r"""Plan a Jellyfin-format relayout of a TV library. DRY RUN ONLY: nothing moves. + + media-shows.py [--root DIR] [--enum TSV] [--titles TSV] [--probed TSV] + [--tmdb-cache JSON] [--tmdb] [--movies DIR] [--park DIR] + [--out TSV] [--exceptions TXT] + +Inputs are the TSVs scripts/media-census.py exports: `enumerate` lists what +exists, `jellyfin` says which episode each file IS, and `probe` is optional +duration evidence for the movie rule. The TMDb cache is a plain JSON file so a +plan stays reproducible: answers are read from it, and a miss is looked up only +with --tmdb and TMDB_API_KEY in the environment (the key never enters argv, and +never the cache). + +Target contract (jellyfin.org/docs/general/server/media/shows): + + / ()/Season / SxxEyy.ext + one file holding two episodes SxxE01-E02 specials Season 00 + a documented part of an episode SxxE03 Part 1 sidecars ..srt + bonus material /trailers|featurettes|… — never a season + +PLAN is the TSV scripts/media-apply.py replays: `action src dst kind`, every +mkdir row before any move, and `keep` for a file already where the contract puts +it. EXCEPTIONS is the review list: everything parked, and why. + +The authority chain, strongest first, for one video file: + + 1. a provider row — Jellyfin matched this exact path and numbered it. + 2. the file's own SxxEyy / 1x04 marker, the series taken from a sibling's + provider row, else from the nearest ancestor folder that has one, else from + the release folder's own words. + 3. nothing. An unnumbered file is PARKED under --park with its relative path + kept: an invented `Season 01` steals an episode number from the file that + really is S01E01, and a wrong number is worse than no number. + +Guards, and why each is here: + + * One series, one folder, one year: the year ANY provider row of a series + states is shared by all of them, so a box set split across five release + folders lands inside ONE series folder instead of five. + * Two files claiming one episode: the bigger keeps the standard name, the other + parks with the reason. Nothing is overwritten and nothing is deleted — which + encode survives is Frank's call, and a parked file is one rename from home. + * A file with no episode evidence that runs long and that the MOVIE provider + names leaves the TV library for --movies under the movie naming contract. + Long and unnamed parks; short and unnumbered is bonus material. + * Series art and the show nfo follow the SERIES, a season image follows its + SEASON, a subtitle or episode nfo follows the episode's new base — an episode + moved without its subtitles is a half-move. + * Hidden files, macOS forks, qBittorrent resume parts and the park pile itself + are never moved. Skipping the park is what makes re-planning stable. + +The plan crosses three trees (the library, its park, the movie library), so the +executor runs with --root at the share that holds them, on its own journal — +one journal is one undo: + + python3 scripts/media-shows.py --root /mnt/entropy/TVShows \ + --enum /tmp/tv-enum.tsv --titles /tmp/jellyfin-tv.tsv \ + --out /mnt/entropy/media-tv-plan.tsv + python3 scripts/media-apply.py apply --plan /mnt/entropy/media-tv-plan.tsv \ + --root /mnt/entropy --journal /mnt/entropy/media-tv-relayout.jsonl +""" +import argparse +import collections +import importlib.util +import json +import os +import re +import sys +import urllib.parse +import urllib.request + +ARGS = argparse.ArgumentParser(description=__doc__.splitlines()[0], + formatter_class=argparse.RawDescriptionHelpFormatter) +ARGS.add_argument("--root", default=os.environ.get("TV_ROOT", "/mnt/entropy/TVShows"), + help="the TV library the plan reorganises") +ARGS.add_argument("--enum", default="/tmp/tv-enum.tsv", + help="file census (media-census.py enumerate)") +ARGS.add_argument("--titles", default="/tmp/jellyfin-tv.tsv", + help="the provider's numbering (media-census.py jellyfin)") +ARGS.add_argument("--probed", default="", + help="ffprobe census for the movie rule; absent = no durations") +ARGS.add_argument("--tmdb-cache", default="/tmp/tmdb-cache.json", + help="TMDb answers as an input file, so a plan is reproducible") +ARGS.add_argument("--tmdb", action="store_true", + help="look up cache misses (needs TMDB_API_KEY)") +ARGS.add_argument("--movies", default="/mnt/entropy/Movies", + help="movie library a mis-filed film moves into") +ARGS.add_argument("--park", default="", + help="review pile (default: /_unsorted)") +ARGS.add_argument("--out", default="/tmp/tv-plan.tsv") +ARGS.add_argument("--exceptions", default="/tmp/tv-exceptions.txt") +ARGS = ARGS.parse_args() + +ROOT = os.path.abspath(ARGS.root) +PARK = os.path.abspath(ARGS.park or os.path.join(ROOT, "_unsorted")) +MOVIES = os.path.abspath(ARGS.movies) +ENGINE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "media-titles.py") +_spec = importlib.util.spec_from_file_location("eng", ENGINE) +eng = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(eng) + +try: + CACHE = json.load(open(ARGS.tmdb_cache)) +except (OSError, ValueError): + CACHE = {} +API_KEY = os.environ.get("TMDB_API_KEY", "") + +VIDEO_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg", + ".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".flv", ".vob", ".rm", + ".rmvb", ".iso", ".img"} +SUB_EXT = {".srt", ".ass", ".ssa", ".sub", ".vtt"} +IMG_EXT = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif", ".tbn"} +NFO_EXT = {".nfo", ".xml"} +# Files that are never anybody's business: hidden, forks, resume state, thumbnails. +SKIP_NAME = re.compile(r"^(\.|#|dl_state|@eaDir|Thumbs\.db|desktop\.ini)|\.!q$") +# Show-level metadata keeps its own name at the series root; season-level at the +# season root; folder.jpg stays with the folder it was written for. +SERIES_META = re.compile(r"^(tvshow|series|show)(\.nfo|\.xml)?$" + r"|^(poster|tvposter|banner|clearlogo|clearart|logo|art" + r"|landscape|fanart|showbackdrop|backdrop)\d*\." + r"(jpg|jpeg|png|webp|bmp)$", re.I) +SEASON_META = re.compile(r"^season\.nfo$", re.I) +FOLDER_IMAGE = re.compile(r"^folder\.(jpg|jpeg|png|webp|bmp)$", re.I) +SEASON_DIR_RE = re.compile(r"(?i)^\s*(?:season[\s._-]*(\d{1,3})|s(\d{1,2}))\s*$") +# Release words a folder name carries that are not part of the show's name. +FOLDER_JUNK = re.compile(r"(?i)^(complete|collection|box|set|batch|pack|series|season" + r"|multigroup|dvd|remux|uncut|extended|dual|audio|multi|dub" + r"|dubbed|sub|subbed|internal|repack|proper|webrip)$") +STOP_WORDS = {"the", "a", "an", "of", "and", "to", "in", "on", "for", "with", "de", + "la", "le", "il", "des", "der", "das", "y", "e", "el", "les"} +BAD_CHARS = re.compile(r'[<>:"/|?*]') +# SxxEyy with an optional second episode, plus the 1x04 form. The second number +# is guarded so a resolution never reads as an episode: "S01E01-1080p" is ONE +# episode, and the longest-match backtracking of \d{1,3} is what the (?![\dpP]) +# after it closes off. +EP_RANGE = re.compile(r"(?i)(? 2: + try: + dur[os.path.abspath(f[1])] = float(f[2] or 0) + except ValueError: + pass + +video = [r for r in rows if r["ext"] in VIDEO_EXT] +other = [r for r in rows if r["ext"] not in VIDEO_EXT] + +# What a folder already knows: one provider row names the series for every sibling +# that has no row of its own — that is how an anime pack gets its show. +by_dir = collections.defaultdict(list) +for r in video: + by_dir[os.path.dirname(r["path"])].append(r) +dir_series = {} +for d, rs in by_dir.items(): + names = collections.Counter(titles[r["path"]]["series"] for r in rs + if r["path"] in titles and titles[r["path"]]["series"]) + if names: + dir_series[d] = names.most_common(1)[0][0] + + +# A series folder that sits above a folder of provider rows names its own show, and +# that is what keeps an already-filed library stable when it is re-planned: extras in +# "The Examples (2010)/trailers" belong to "The Examples", not to a series invented +# out of the word "trailers". +for d, name in sorted(dir_series.items()): + a = os.path.dirname(d) + while a and a != ROOT and len(a) > len(ROOT): + if eng.norm(folder_series(os.path.basename(a))) == eng.norm(name): + dir_series.setdefault(a, name) + a = os.path.dirname(a) + + +def series_of_dir(d): + """The series an ancestor folder belongs to, nearest first.""" + while d and d != ROOT and len(d) > len(ROOT): + if d in dir_series: + return dir_series[d] + d = os.path.dirname(d) + return "" + + +def head_series(path): + """The folder evidence for a file's series: its own folder's words, then the + top-level release folder's.""" + parts = rel(path).split("/")[:-1] + for d in reversed(parts): + name = folder_series(d) + if name: + return name + return "" + + +def extras_kind(path, extra_type): + """The canonical bonus subfolder, from Jellyfin's own extra type first.""" + if extra_type: + return EXTRA_KIND.get(eng.norm(extra_type).replace(" ", ""), "other") + for d in rel(path).split("/")[:-1]: + if eng.folder_is_bonus(d): + for name, rx in EXTRA_DIRS: + if re.search(rx, d, re.I): + return name + return "extras" + stem = os.path.splitext(os.path.basename(path))[0] + for name, rx in EXTRA_DIRS: + if re.search(rx, stem, re.I): + return name + return "extras" + + +# ----------------------------------------------------------- classify every file +series = {} # norm(name) -> {name, year, tmdb} +claimants = [] # (row, name, season, [episodes]) +extras = [] # (row, name, bonus folder) +films = [] # film-shaped, no episode evidence +unknown = [] # (row, reason) + + +def note(name, year="", tmdb_id=""): + """Register a series; the first name wins, the first year and id stick.""" + k = eng.norm(name) + s = series.setdefault(k, {"name": clean(name), "year": "", "tmdb": ""}) + if year and not s["year"]: + s["year"] = str(year) + if tmdb_id and not s["tmdb"]: + s["tmdb"] = str(tmdb_id) + + + +def as_extra(r, owner, extra_type): + """Bonus material needs a series to file under; with none it is parked.""" + if not owner: + unknown.append((r, "bonus material with no series to own it")) + return + note(owner) + extras.append((r, owner, extras_kind(r["path"], extra_type))) + +for r in video: + p, t = r["path"], titles.get(r["path"]) + marked = markers(r["base"]) + if t and t["extra"]: + as_extra(r, t["series"] or series_of_dir(os.path.dirname(p)) or head_series(p), + t["extra"]) + continue + if t and t["series"] and t["season"] and t["episode"]: + season, eps = int(t["season"]), [int(t["episode"])] + # One file can hold two episodes; the row gives the first, the name the range. + if marked and marked[0] == season and len(marked[1]) > 1: + eps = marked[1] + claimants.append((r, t["series"], season, eps)) + note(t["series"], t["year"], t["tmdb"]) + continue + if marked: + owner = (series_of_dir(os.path.dirname(p)) or head_series(p) + or title_of(r["base"][:marked[2]])[0]) + if not eng.norm(owner).replace(" ", ""): + unknown.append((r, "an episode marker with no series name to hang it on")) + continue + claimants.append((r, owner, marked[0], marked[1])) + note(owner) + continue + if eng.is_extra(p, title_of(r["base"])[0], dur.get(p, 0.0), ROOT): + as_extra(r, series_of_dir(os.path.dirname(p)) or head_series(p), "") + continue + if dur.get(p, 0.0) >= SHORT_MAX or (p not in dur and title_of(r["base"])[1] != "?"): + films.append(r) + continue + unknown.append((r, "no episode marker, no provider row, nothing long enough to be a film")) + +# A provider that never states a year still has one: the search is exact, so a +# miss parks nothing — the folder simply carries no year, which Jellyfin accepts. +for k, s in series.items(): + if not s["year"]: + got = tmdb("tv", s["name"]) + if got and eng.norm(got["name"]) == k: + s["year"] = got["year"] + +if ARGS.tmdb: + with open(ARGS.tmdb_cache, "w") as fh: + json.dump(CACHE, fh, indent=1, sort_keys=True) + + +def series_dir(name): + s = series.get(eng.norm(name)) + if not s: + s = {"name": clean(name), "year": ""} + label = f"{s['name']} ({s['year']})" if s["year"] else s["name"] + return f"{ROOT}/{clean(label)}" + + +# ------------------------------------------------------------------- the plan +moves, keeps, parked = [], [], [] +reasons = collections.defaultdict(list) +used = {} # dst -> the file that claimed it + + +def book(r, dst_dir, name, kind): + """Claim one destination for one file, or park the file when it is taken.""" + dst = f"{dst_dir}/{name}" + if dst == r["path"]: + keeps.append((r["path"], dst, kind)) + return + if dst in used: + park(r, "something else in the plan already files there") + return + if dst in held: + park(r, f"{rel(dst)} already occupies that name") + return + used[dst] = r["path"] + moves.append((r["path"], dst, kind)) + + +def park(r, reason): + parked.append((r, reason)) + reasons[reason].append(r) + dst = f"{PARK}/{rel(r['path'])}" + if dst in used or dst in held: + dst = f"{PARK}/{rel(r['path'])}.{len(reasons)}" + used[dst] = r["path"] + moves.append((r["path"], dst, "park")) + + +# Episodes first: two files claiming one number is a decision Frank makes, not one +# the plan sneaks — the bigger copy takes the standard name, the other waits. +by_ep = collections.defaultdict(list) +for r, name, season, eps in claimants: + for e in eps: + by_ep[(eng.norm(name), season, e)].append(r) +winners, losers = set(), {} +for (key, season, e), rs in by_ep.items(): + rs.sort(key=lambda r: (-r["bytes"], r["path"])) + winners.add(rs[0]["path"]) + for r in rs[1:]: + losers.setdefault(r["path"], + f"duplicate of {series[key]['name']} S{season:02d}E{e:02d}" + " (the bigger copy keeps the standard name)") +for r, name, season, eps in claimants: + if r["path"] in losers: + park(r, losers[r["path"]]) + continue + s = series[eng.norm(name)] + label = f" S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps) + part = PART_RE.search(os.path.splitext(r["base"])[0]) + tail = f" Part {part.group(1).upper()}" if part and len(eps) == 1 else "" + book(r, f"{series_dir(name)}/Season {season:02d}", s["name"] + label + tail + r["ext"], + "tv") + +for r, name, kind in extras: + book(r, f"{series_dir(name)}/{kind}", + clean(os.path.splitext(r["base"])[0]) + r["ext"], "extra") + +# A film inside the TV library leaves for the movie library only when the movie +# provider answers with a title that matches the name and, when the name states a +# year, that year too. +for r in films: + parsed, year = title_of(r["base"]) + got = tmdb("movie", parsed) + words = eng.norm(got["name"]).split() if got else [] + if not got or not eng.mentions(parsed.split(), words) \ + or (year != "?" and got["year"] and got["year"] != year): + park(r, "long file with no episode evidence, and no movie provider names it") + continue + label = f"{got['name']} ({got['year']})" if got["year"] else got["name"] + book(r, f"{MOVIES}/{clean(label)}", clean(label) + r["ext"], "movie") + +# ------------------------------------------------- sidecars and folder metadata +moved = {} # src -> dst, for every booked file +for src, dst, _kind in moves: + moved[src] = dst +for src, dst, _kind in keeps: + moved[src] = dst +dst_dirs = collections.defaultdict(collections.Counter) +video_dst = collections.defaultdict(list) +for src, dst in moved.items(): + if src in {r["path"] for r in video}: + video_dst[os.path.dirname(src)].append((src, dst)) + dst_dirs[os.path.dirname(src)][os.path.dirname(dst)] += 1 + + +def series_name_at(d): + """The series the files of a folder belong to, else the folder's own words.""" + names = [titles[src]["series"] for src, _ in video_dst.get(d, []) + if src in titles and titles[src]["series"]] + if names: + return collections.Counter(names).most_common(1)[0][0] + return series_of_dir(d) or head_series(d + "/x") + + +def season_at(d): + """The season number a folder carries: its own name, else its files' rows.""" + m = SEASON_DIR_RE.match(os.path.basename(d)) + if m: + return int(m.group(1) or m.group(2)) + nums = [int(titles[src]["season"]) for src, _ in video_dst.get(d, []) + if src in titles and titles[src]["season"]] + return collections.Counter(nums).most_common(1)[0][0] if nums else None + + +def folder_dst(d): + """Where a folder's own images go: the one place its files landed, or the + series root when they scattered across several seasons.""" + c = dst_dirs.get(d) + if not c: + return None + if len(c) == 1: + return c.most_common(1)[0][0] + owner = series_name_at(d) + return series_dir(owner) if owner else None + + +def anchor_of(path, stem): + """The video file this sidecar names: the longest file name it extends, in its + own folder first, then the parent's (a Subs/ or Extras/ folder beside them).""" + for d in (os.path.dirname(path), os.path.dirname(os.path.dirname(path))): + best = None + for src, dst in video_dst.get(d, []): + base = os.path.basename(src) + for prefix in (os.path.splitext(base)[0], base): + if stem == prefix or stem.startswith(prefix + "."): + if best is None or len(prefix) > best[1]: + best = (src, dst, len(prefix)) + if best: + return best[0], best[1] + return None + + +def sidecar_name(r, src, dst): + """The sidecar's own tail — language, flag, the media extension an nfo must + carry — hung on the video file's new base.""" + stem = os.path.splitext(os.path.basename(src))[0] + tail = os.path.splitext(r["base"])[0][len(stem):] + if r["ext"] in NFO_EXT: + tail = os.path.splitext(os.path.basename(src))[1] + tail + return clean(os.path.splitext(os.path.basename(dst))[0] + tail + r["ext"]) + + +for r in other: + p, base = r["path"], r["base"] + stem = os.path.splitext(base)[0] + d = os.path.dirname(p) + if r["ext"] in SUB_EXT or r["ext"] in NFO_EXT or r["ext"] in IMG_EXT: + a = anchor_of(p, stem) + if a: + book(r, os.path.dirname(a[1]), sidecar_name(r, a[0], a[1]), + "subtitle" if r["ext"] in SUB_EXT else + ("nfo" if r["ext"] in NFO_EXT else "image")) + continue + if SERIES_META.match(base): + owner = series_name_at(d) + if owner: + book(r, series_dir(owner), base, "meta") + continue + elif SEASON_META.match(base): + owner, season = series_name_at(d), season_at(d) + if owner and season is not None: + book(r, f"{series_dir(owner)}/Season {season:02d}", base, "meta") + continue + elif r["ext"] in IMG_EXT or r["ext"] in NFO_EXT: + dest = folder_dst(d) + if dest: + book(r, dest, base, "meta") + continue + unknown.append((r, "not media, or media with nothing beside it to belong to")) + +for r, reason in unknown: + park(r, reason) + + +# --------------------------------------------------------------------- output +def write_plan(): + """mkdir rows for every destination folder and its whole chain, then the moves. + Parents first: the executor creates directories in the order they are listed.""" + chain = set() + for _s, dst, _k in moves: + d = os.path.dirname(dst) + while d and d not in (ROOT, MOVIES, PARK, os.path.dirname(d)): + chain.add(d) + d = os.path.dirname(d) + dirs = sorted(chain, key=lambda p: (p.count("/"), p)) + with open(ARGS.out, "w") as fh: + fh.write("action\tsrc\tdst\tkind\n") + for d in dirs: + fh.write(f"mkdir\t\t{d}\tdir\n") + for src, dst, kind in sorted(moves, key=lambda m: (m[1], m[0])): + fh.write(f"move\t{src}\t{dst}\t{kind}\n") + for src, dst, kind in sorted(keeps, key=lambda m: (m[1], m[0])): + fh.write(f"keep\t{src}\t{dst}\t{kind}\n") + sized = sum(held[s]["bytes"] for s, _d, _k in moves if s in held) + print(f"plan: {len(moves) - len(parked)} moves, {len(parked)} parked" + f" ({sized / 2**40:.2f} TiB touched), {len(keeps)} already placed," + f" {len(dirs)} folders -> {ARGS.out}") + + +def write_exceptions(): + no_year = sorted(s["name"] for s in series.values() if not s["year"]) + with open(ARGS.exceptions, "w") as fh: + fh.write(f"TV relayout exceptions for {ROOT}\n\n") + fh.write(f"Everything below is parked under {PARK}, keeping the relative path it\n") + fh.write("has today. Nothing is deleted and nothing is overwritten: a parked file\n") + fh.write("is one rename from wherever the plan is pointed next.\n\n") + for reason in sorted(reasons, key=lambda r: -len(reasons[r])): + rs = reasons[reason] + fh.write(f"{len(rs):>6} files {reason}\n") + for r in sorted(rs, key=lambda r: rel(r["path"]))[:400]: + fh.write(f" {rel(r['path'])}\n") + if len(rs) > 400: + fh.write(f" … and {len(rs) - 400} more\n") + fh.write("\n") + if no_year: + fh.write(f"{len(no_year):>6} series carry no year from any provider, so their\n") + fh.write(" folders are named without one; Jellyfin still matches them.\n") + fh.write(" " + ", ".join(no_year[:300]) + "\n") + print(f"exceptions -> {ARGS.exceptions}") + + +write_plan() +write_exceptions() diff --git a/tests/media_shows.bats b/tests/media_shows.bats new file mode 100755 index 0000000..fd4029b --- /dev/null +++ b/tests/media_shows.bats @@ -0,0 +1,271 @@ +#!/usr/bin/env bats +# Seam: the TV-library relayout — scripts/media-census.py jellyfin (the metadata +# authority export) and scripts/media-shows.py (the plan). +# +# The TV library is 52k files written in five different release layouts, and the +# one thing that makes a rename of that size survivable is that the plan is +# derived from what a metadata provider already matched, not from a guess at the +# file name. So every test here states which authority a file had — name only, +# provider only, both, neither — and asserts where the PLAN files it, never how +# the planner decided. A fixture Jellyfin database stands in for the live one and +# the TMDb cache is an input file, so no test touches the network. +# +# The two outcomes that cost a re-run of a 50k-file plan are a file filed as an +# episode that is not one (it steals an episode number from its real owner) and a +# move that lands on an existing file. Everything below is written to catch +# exactly those, plus the gate that matters most: the plan must be replayable by +# scripts/media-apply.py, whose preflight is the fleet's only overwrite guard. + +load helpers + +PLAN="$REPO_ROOT/scripts/media-shows.py" +CENSUS="$REPO_ROOT/scripts/media-census.py" +APPLY="$REPO_ROOT/scripts/media-apply.py" + +# at PATH SIZE — a file with deterministic content. +at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; } + +# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] — one row of the authority +# export scripts/media-census.py jellyfin writes. +title_row() { + printf '%s\t%s\t%s\t%s\t%s\t%s\t\n' "$1" "$2" "$3" "${4:-}" "${5:-}" "${6:-}" \ + >> "$TITLES" +} + +# dst SRC — the destination the plan gives SRC, or nothing when it does not move. +dst() { + awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $3}' "$OUT" +} + +setup() { + LIB="$BATS_TEST_TMPDIR/TVShows" + PARK="$LIB/_unsorted" + MOVIES="$BATS_TEST_TMPDIR/Movies" + ENUM="$BATS_TEST_TMPDIR/enum.tsv" + PROBED="$BATS_TEST_TMPDIR/probed.tsv" + TITLES="$BATS_TEST_TMPDIR/titles.tsv" + CACHE="$BATS_TEST_TMPDIR/tmdb.json" + OUT="$BATS_TEST_TMPDIR/plan.tsv" + EXC="$BATS_TEST_TMPDIR/exceptions.txt" + : > "$PROBED" + + SEASON="$LIB/The Examples (2010) Season 1 S01 (1080p BluRay x265 Silence)" + # An episode its own name states, with a subtitle, a season image and a + # trailer: four things that have to end up in four different places. + at "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" 9000 + at "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt" 200 + at "$SEASON/folder.jpg" 300 + at "$SEASON/Trailers/teaser.mkv" 1000 + at "$SEASON/tvshow.nfo" 400 + # An anime pack: no episode marker anywhere in its name, numbered by the + # provider only. Its year is the one thing no authority states. + ANIME="$LIB/Anime Pack [grp] (1080p HEVC)" + at "$ANIME/[grp] Anime Pack - 05.mkv" 5000 + # Two releases claiming one episode, and a file holding two episodes. + at "$LIB/Dupes/Show Two S01E01.mkv" 4000 + at "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" 7000 + at "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" 7000 + # A file no authority numbers, and a long file the movie provider knows. + at "$LIB/Orphan Tapes/clip001.mp4" 3000 + at "$LIB/Longform/Some Documentary Special (2015).mkv" 8000 + # One episode-shaped file loose at the library root. + at "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" 4000 + + # The enum census lists every file; the planner reads that instead of walking + # the tree, so the fixture writes the same three columns. + python3 - "$LIB" "$ENUM" <<'PY' +import os, sys +root, out = sys.argv[1], sys.argv[2] +with open(out, "w") as fh: + for d, _, fs in os.walk(root): + for f in fs: + p = os.path.join(d, f) + fh.write(f"{os.stat(p).st_size}\t{os.path.splitext(f)[1]}\t{p}\n") +PY + + printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\n' > "$TITLES" + printf '{}\n' > "$CACHE" + title_row "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" \ + "The Examples" 2010 1 1 + title_row "$ANIME/[grp] Anime Pack - 05.mkv" "Anime Pack" "" 1 5 + title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1 + title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1 + title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2 + title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3 +} + +plan() { + run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --titles "$TITLES" \ + --probed "$PROBED" --tmdb-cache "$CACHE" --movies "$MOVIES" \ + --out "$OUT" --exceptions "$EXC" +} + +@test "the census exports the provider's episode numbering as a TSV" { + DB="$BATS_TEST_TMPDIR/jellyfin.db" + python3 - "$DB" "$SEASON" <<'PY' +import os, sqlite3, sys, uuid +db, season = sys.argv[1], sys.argv[2] +c = sqlite3.connect(db) +pre = "/data/TVShows/" + os.path.basename(season) +c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber, + ParentIndexNumber, ProductionYear, ExtraType, + IsVirtualItem, Data)""") +sid = str(uuid.uuid4()) +c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)", + (sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples", + pre, None, None, 2010, None, 0, + '{"ProviderIds":{"Tmdb":"1234"}}')) +c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)", + (str(uuid.uuid4()), sid, "MediaBrowser.Controller.Entities.TV.Episode", + "Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1, + None, None, 0, '{}')) +c.commit() +PY + run python3 "$CENSUS" jellyfin --root "$LIB" --db "$DB" \ + --out "$BATS_TEST_TMPDIR/exp.tsv" + [ "$status" -eq 0 ] + run head -1 "$BATS_TEST_TMPDIR/exp.tsv" + [ "${lines[0]}" = "file series year season episode extra tmdb" ] + tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \ + | grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234$" +} + +@test "an episode its own name states files as Show SxxEyy under Season NN" { + plan + [ "$status" -eq 0 ] + [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv")" \ + = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.mkv" ] +} + +@test "a subtitle follows its episode and keeps its language flag" { + plan + [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt")" \ + = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.en.srt" ] +} + +@test "a season image keeps its own name; a series nfo keeps the series folder" { + plan + [ "$(dst "$SEASON/folder.jpg")" = "$LIB/The Examples (2010)/Season 01/folder.jpg" ] + [ "$(dst "$SEASON/tvshow.nfo")" = "$LIB/The Examples (2010)/tvshow.nfo" ] +} + +@test "bonus material files as an extra, never as an episode" { + plan + [ "$(dst "$SEASON/Trailers/teaser.mkv")" = "$LIB/The Examples (2010)/trailers/teaser.mkv" ] + [ -z "$(awk -F '\t' '$3 ~ /Season .*teaser/ {print $3}' "$OUT")" ] +} + +@test "a file only the provider numbers files at the episode it is matched to" { + plan + [ "$(dst "$ANIME/[grp] Anime Pack - 05.mkv")" \ + = "$LIB/Anime Pack/Season 01/Anime Pack S01E05.mkv" ] +} + +@test "one file holding two episodes keeps both numbers in its name" { + plan + [ "$(dst "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv")" \ + = "$LIB/Show Two (2016)/Season 01/Show Two S01E02-E03.mkv" ] +} + +@test "two files claiming one episode: one keeps the name, the other parks" { + plan + n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two S01E01.mkv " "$OUT") + [ "$n" -eq 1 ] + run dst "$LIB/Dupes/Show Two S01E01.mkv" + [[ "$output" == "$PARK/"* ]] + run dst "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" + [[ "$output" == "$LIB/Show Two (2016)/Season 01/"* ]] +} + +@test "a file no authority numbers is parked, never renamed into the library" { + plan + run dst "$LIB/Orphan Tapes/clip001.mp4" + [[ "$output" == "$PARK/Orphan Tapes/clip001.mp4" ]] + grep -q "clip001.mp4" "$EXC" +} + +@test "a long file the movie provider knows leaves the TV library for the movie one" { + printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \ + "$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED" + python3 -c 'import json, sys +json.dump({"movie:some documentary": + {"name": "Some Documentary Special", "year": 2015, "id": 55}}, + open(sys.argv[1], "w"))' "$CACHE" + plan + [ "$(dst "$LIB/Longform/Some Documentary Special (2015).mkv")" \ + = "$MOVIES/Some Documentary Special (2015)/Some Documentary Special (2015).mkv" ] +} + +@test "a long file no movie provider knows is parked, not moved into Movies" { + printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \ + "$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED" + plan + run dst "$LIB/Longform/Some Documentary Special (2015).mkv" + [[ "$output" == "$PARK/"* ]] +} + +@test "the plan creates every folder before the move that needs it" { + plan + python3 - "$OUT" "$LIB" "$MOVIES" "$PARK" <<'PY' +import os, sys +plan, known = sys.argv[1], set(sys.argv[2:]) +made, bad = set(), [] +for line in open(plan).read().splitlines()[1:]: + action, _src, dest, _kind = line.split("\t") + if action == "mkdir": + made.add(dest) + elif action == "move": + d = os.path.dirname(dest) + while d not in known and d != os.path.dirname(d): + if d not in made: + bad.append(f"{dest}: {d} was never made") + d = os.path.dirname(d) +print("\n".join(bad[:5])) +sys.exit(1 if bad else 0) +PY +} + +@test "no move lands outside the library or the movie library" { + plan + run awk -F '\t' -v l="$LIB" -v m="$MOVIES" \ + '$1 == "move" && $3 !~ "^" l "/" && $3 !~ "^" m "/" {print $3}' "$OUT" + [ -z "$output" ] +} + +@test "the executor preflights the plan and no target is claimed twice" { + plan + run python3 "$APPLY" apply --plan "$OUT" --root "$BATS_TEST_TMPDIR" \ + --journal "$BATS_TEST_TMPDIR/journal.jsonl" + [ "$status" -eq 0 ] + dup=$(awk -F '\t' '$1 == "move" {print $3}' "$OUT" | sort | uniq -d) + [ -z "$dup" ] +} + +@test "re-planning a library the plan already satisfies moves nothing" { + plan + # Applying the plan and re-censusing is exactly what the operator does next: + # every authority file is keyed by path, so all three are replayed through the + # plan the way a fresh census would see them. (The enum has no header; the + # titles export does, and its header row is not a path so it survives.) + python3 - "$OUT" "$ENUM" "$TITLES" "$PROBED" <<'PY' +import os, sys +plan = sys.argv[1] +moves = {} +for line in open(plan).read().splitlines()[1:]: + action, src, dest, _kind = line.split("\t") + if action == "move": + moves[src] = dest + os.makedirs(os.path.dirname(dest), exist_ok=True) + os.rename(src, dest) +for path, col in ((sys.argv[2], 2), (sys.argv[3], 0), (sys.argv[4], 1)): + if not os.path.getsize(path): + continue + rows = [line.rstrip("\n").split("\t") for line in open(path)] + with open(path, "w") as fh: + for f in rows: + f[col] = moves.get(f[col], f[col]) + fh.write("\t".join(f) + "\n") +PY + plan + [ -z "$(awk -F '\t' '$1 == "move" {print $2}' "$OUT")" ] +}