diff --git a/scripts/media-census.py b/scripts/media-census.py index fd9c5b3..6a9db9e 100755 --- a/scripts/media-census.py +++ b/scripts/media-census.py @@ -21,11 +21,12 @@ scripts/media-plan.py and scripts/media-shows.py read them instead of touching m the titles Plex already matched — the authority over any filename guess jellyfin file series year season episode extra - tmdb (see JF_COLS) + tmdb title tvdb (see JF_COLS) one row per episode Jellyfin has matched and numbered, its path mapped - from the container's view back to the host; `year` and `tmdb` are the - SERIES' values, `extra` is Jellyfin's own extra type when the row is - bonus material rather than an episode + from the container's view back to the host; `year`, `tmdb` and `tvdb` + are the SERIES' values, `title` is the episode title the row carries, + `extra` is Jellyfin's own extra type when the row is bonus material + rather than an episode music file artist album year disc track title mb performer (see MUSIC_COLS) @@ -83,13 +84,14 @@ left join metadata_items par on par.id = mi.parent_id where p.file like ? and p.deleted_at is null and mi.deleted_at is null """ JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db" -JF_COLS = ["file", "series", "year", "season", "episode", "extra", "tmdb"] +JF_COLS = ["file", "series", "year", "season", "episode", "extra", + "tmdb", "title", "tvdb"] # Jellyfin stores the entity's .NET class as its type; the TV tree is these three. JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode", "MediaBrowser.Controller.Entities.TV.Season", "MediaBrowser.Controller.Entities.TV.Series") -JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, IsVirtualItem - from BaseItems where Type = ? and Path like ?""") +JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, + IsVirtualItem, Name from BaseItems where Type = ? and Path like ?""") # The music tree is these two entity classes. The track row carries the tags # Jellyfin read out of the file; the album row carries what the provider matched, @@ -254,9 +256,10 @@ def jellyfin(root, db, container, out): The numbering is what matters, not the name: a release group's file name says nothing about which episode a file is, and Jellyfin has already answered that - question against a provider. Season/episode come from the episode row; the - series name, its year and its TMDb id come from the Series row the episode - climbs to, because those are the only place Jellyfin keeps them. + question against a provider. Season/episode and the episode's own title come + from the episode row; the series name, its year and its TMDb and Tvdb ids come + from the Series row the episode climbs to, because those are the only place + Jellyfin keeps them. """ try: con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) @@ -277,21 +280,38 @@ def jellyfin(root, db, container, out): " where Type in (?,?,?)", JF_TV): node[i] = (pid, typ, name, year, data) + # Jellyfin 12 keeps provider ids in their own table; older builds kept them in + # the row's Data blob. Both shapes are read, because the tvdb id the series + # naming depends on is wherever this build happened to write it. + prov = {} + if "BaseItemProviders" in {r[0] for r in con.execute( + "select name from sqlite_master where type='table'")}: + for iid, pid, val in con.execute( + "select ItemId, ProviderId, ProviderValue from BaseItemProviders" + " where ProviderId in ('Tmdb', 'Tvdb')"): + prov[(iid, pid)] = val + + def provider_of(item_id, want): + got = prov.get((item_id, want)) + if got: + return str(got) + try: + blob = json.loads(node[item_id][4] or "{}").get("ProviderIds") or {} + except (KeyError, ValueError, TypeError): + return "" + return str(blob.get(want, "") or "") + def series_of(item_id): """Climb episode -> season -> series; empty when the chain is broken.""" for _ in range(4): if item_id not in node: - return "", "", "" - pid, typ, name, year, data = node[item_id] + return "", "", "", "" + pid, typ = node[item_id][0], node[item_id][1] if typ.endswith(".Series"): - try: - tmdb = (json.loads(data or "{}") - .get("ProviderIds") or {}).get("Tmdb", "") - except ValueError: - tmdb = "" - return name or "", year or "", tmdb + return (node[item_id][2] or "", node[item_id][3] or "", + provider_of(item_id, "Tmdb"), provider_of(item_id, "Tvdb")) item_id = pid - return "", "", "" + return "", "", "", "" # The fleet's quadlets bind the share at /data, so the container's view of # this library is /data/ unless told otherwise. @@ -302,17 +322,18 @@ def jellyfin(root, db, container, out): with open(out, "w", newline="") as fh: w = csv.writer(fh, delimiter="\t", lineterminator="\n") w.writerow(JF_COLS) - for iid, path, idx, pidx, extra, virt in con.execute(JF_SQL, - (JF_TV[0], prefix + "%")): + for iid, path, idx, pidx, extra, virt, epname in con.execute( + JF_SQL, (JF_TV[0], prefix + "%")): if virt or not path or not path.startswith(prefix): skipped += 1 # a virtual item is a variant, not a file continue - sname, syear, tmdb = series_of(iid) + sname, syear, tmdb, tvdb = series_of(iid) w.writerow([root + "/" + path[len(prefix):], sname.replace("\t", " "), syear, "" if pidx is None else pidx, "" if idx is None else idx, - (extra or "").replace("\t", " "), tmdb]) + (extra or "").replace("\t", " "), tmdb, + (epname or "").replace("\t", " "), tvdb]) written += 1 print(f"episodes={written} virtual-or-blank={skipped} -> {out}") diff --git a/scripts/media-shows.py b/scripts/media-shows.py index 9d55613..b722726 100755 --- a/scripts/media-shows.py +++ b/scripts/media-shows.py @@ -6,29 +6,40 @@ r"""Plan a Jellyfin-format relayout of a TV library. DRY RUN ONLY: nothing moves [--out TSV] [--exceptions TXT] Inputs are the TSVs scripts/media-census.py exports: `enumerate` lists what -exists, `jellyfin` says which episode each file IS, and `probe` is optional -duration evidence for the movie rule. The TMDb cache is a plain JSON file so a -plan stays reproducible: answers are read from it, and a miss is looked up only -with --tmdb and TMDB_API_KEY in the environment (the key never enters argv, and +exists, `jellyfin` says which episode each file IS (its series' TMDb and Tvdb +ids, and the episode title the row carries), and `probe` is optional duration +evidence for the movie rule. Every name and title answer lives in the cache JSON +so a plan stays reproducible: answers are read from it, and a miss is looked up +only with --tmdb/--tvdb and their credentials (a key never enters argv, and never the cache). Target contract (jellyfin.org/docs/general/server/media/shows): - / ()/Season / SxxEyy.ext + / ()/Season / () - SxxEyy - .ext one file holding two episodes SxxE01-E02 specials Season 00 a documented part of an episode SxxE03 Part 1 sidecars <base>.<lang>.srt bonus material <series>/trailers|featurettes|… — never a season +The series name and the episode title are the ones the IDENTITY authority — tvdb +— states for the id the metadata server matched, not the words a release folder +or a server's folder-derived Series name happens to carry. + PLAN is the TSV scripts/media-apply.py replays: `action src dst kind`, every mkdir row before any move, and `keep` for a file already where the contract puts it. EXCEPTIONS is the review list: everything parked, and why. The authority chain, strongest first, for one video file: - 1. a provider row — Jellyfin matched this exact path and numbered it. + 1. a provider row — Jellyfin matched this exact path and numbered it. The + row's NAME is only evidence once an id backs it: the server names a Series + after the folder it scanned, so `24 (2001) Season 1` is a folder echo, and + the series' tvdb id is what undoes it. 2. the file's own SxxEyy / 1x04 marker, the series taken from a sibling's provider row, else from the nearest ancestor folder that has one, else from - the release folder's own words. + a registered show its own words name, else — outside the review pile — from + the release folder's words. Inside a hidden top-level folder a file may + MATCH a registered show but never INVENT one: invention is what filled the + pile to begin with. 3. nothing. An unnumbered file is PARKED under --park with its relative path kept: an invented `Season 01` steals an episode number from the file that really is S01E01, and a wrong number is worse than no number. @@ -47,8 +58,11 @@ Guards, and why each is here: * Series art and the show nfo follow the SERIES, a season image follows its SEASON, a subtitle or episode nfo follows the episode's new base — an episode moved without its subtitles is a half-move. - * Hidden files, macOS forks, qBittorrent resume parts and the park pile itself - are never moved. Skipping the park is what makes re-planning stable. + * Hidden files, macOS forks, qBittorrent resume parts are never moved. A file + already inside --park is never moved either: that is what makes re-planning + stable. --park belongs OUTSIDE the library — parked files inside it get + indexed as one show named after the park folder, and a row like that is a + folder echo, never evidence (the guard in `row_names_the_park`). The plan crosses three trees (the library, its park, the movie library), so the executor runs with --root at the share that holds them, on its own journal — @@ -56,7 +70,8 @@ one journal is one undo: python3 scripts/media-shows.py --root /mnt/entropy/TVShows \ --enum /tmp/tv-enum.tsv --titles /tmp/jellyfin-tv.tsv \ - --out /mnt/entropy/media-tv-plan.tsv + --tvdb --tmdb --out /mnt/entropy/media-tv-plan.tsv \ + --park /mnt/entropy/media-park/TVShows python3 scripts/media-apply.py apply --plan /mnt/entropy/media-tv-plan.tsv \ --root /mnt/entropy --journal /mnt/entropy/media-tv-relayout.jsonl """ @@ -80,14 +95,21 @@ ARGS.add_argument("--titles", default="/tmp/jellyfin-tv.tsv", help="the provider's numbering (media-census.py jellyfin)") ARGS.add_argument("--probed", default="", help="ffprobe census for the movie rule; absent = no durations") -ARGS.add_argument("--tmdb-cache", default="/tmp/tmdb-cache.json", - help="TMDb answers as an input file, so a plan is reproducible") +ARGS.add_argument("--tmdb-cache", default="/tmp/tv-names.json", + help="name answers (TMDb and Tvdb) as ONE input file, so a plan " + "is reproducible; keys carry the provider they came from") ARGS.add_argument("--tmdb", action="store_true", - help="look up cache misses (needs TMDB_API_KEY)") + help="look up cache misses at TMDb (needs TMDB_API_KEY)") +ARGS.add_argument("--tvdb", action="store_true", + help="look up cache misses at Tvdb (needs the credentials file)") +ARGS.add_argument("--tvdb-credentials", + default=os.path.expanduser("~/.config/secrets/tvdb"), + help="file with an api_key= line (and a pin= line for a " + "user-issued key); mode 600, and never the cache") ARGS.add_argument("--movies", default="/mnt/entropy/Movies", help="movie library a mis-filed film moves into") ARGS.add_argument("--park", default="", - help="review pile (default: <root>/_unsorted)") + help="review pile OUTSIDE the library (default: <root>/_unsorted)") ARGS.add_argument("--out", default="/tmp/tv-plan.tsv") ARGS.add_argument("--exceptions", default="/tmp/tv-exceptions.txt") ARGS = ARGS.parse_args() @@ -251,6 +273,134 @@ def tmdb(kind, query): return CACHE[ck] +# ---------------------------------------------------------------- tvdb, the identity +# tvdb names the show and the episode: it is the only provider whose strings are +# the ones the operator asked for, so its answer replaces a folder-derived Series +# name. The login mints a JWT per run from a mode-600 file; neither the key nor +# the token reaches argv, the cache, or the plan. A provider that will not answer +# never invents a name: every lookup here returns None, and None keeps the +# provider's own words and lands in the exceptions. +TVDB = {"jwt": None, "dead": False} + + +def http_json(url, body=None, headers=None): + req = urllib.request.Request( + url, data=json.dumps(body).encode() if body is not None else None, + headers={"Accept": "application/json", "Content-Type": "application/json", + **(headers or {})}) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + + +def tvdb_login(): + """The run's JWT, or None: a project key needs only api_key=, a user-issued + key also needs the pin.""" + if TVDB["jwt"] is not None: + return TVDB["jwt"] or None + TVDB["jwt"] = "" + try: + lines = open(ARGS.tvdb_credentials).read().splitlines() + except OSError: + print(f"tvdb: no credentials at {ARGS.tvdb_credentials}", file=sys.stderr) + return None + sec = {} + for line in lines: + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + sec[k.strip()] = v.strip() + if not sec.get("api_key"): + print(f"tvdb: no api_key in {ARGS.tvdb_credentials}", file=sys.stderr) + return None + body = {"apiKey": sec["api_key"]} + if sec.get("pin"): + body["token"] = sec["pin"] + try: + TVDB["jwt"] = (http_json("https://api4.thetvdb.com/v4/login", body) + .get("data") or {}).get("token") or "" + except Exception as e: # noqa: BLE001 + print(f"tvdb login failed: {e}", file=sys.stderr) + return None + return TVDB["jwt"] or None + + +def tvdb(path): + """One Tvdb v4 GET, or None when it cannot be answered.""" + if not ARGS.tvdb or TVDB["dead"]: + return None + jwt = tvdb_login() + if not jwt: + TVDB["dead"] = True + return None + try: + return http_json("https://api4.thetvdb.com/v4" + path, + headers={"Authorization": "Bearer " + jwt}).get("data") + except Exception as e: # noqa: BLE001 + print(f"tvdb {path} failed: {e}", file=sys.stderr) + return None + + +def tvdb_series(sid): + """{name, year} for a tvdb series id.""" + ck = f"tvdb:series:{sid}" + if ck in CACHE: + return CACHE[ck] + data = tvdb(f"/series/{sid}") + if not data: + return None + CACHE[ck] = {"name": (data.get("name") or "").strip(), + "year": (data.get("firstAired") or "")[:4]} + return CACHE[ck] + + +def tvdb_id_for_tmdb(mid): + """The tvdb id TMDb records for a series id — the bridge for the bulk of this + library, which the server matched by TMDb.""" + ck = f"tmdb:external:{mid}" + if ck in CACHE: + return CACHE[ck] or "" + if not (ARGS.tmdb and API_KEY): + return "" + try: + got = http_json(f"https://api.themoviedb.org/3/tv/{mid}/external_ids", + headers={"Authorization": "Bearer " + API_KEY}) + except Exception as e: # noqa: BLE001 + print(f"tmdb external ids failed for {mid}: {e}", file=sys.stderr) + return "" + CACHE[ck] = str(got.get("tvdb_id") or "") + return CACHE[ck] + + +def tvdb_search(name): + """The tvdb series a provider name states, or None: only tvdb's own spelling + counts, so a near-miss never renames a show.""" + ck = f"tvdb:search:{name}" + if ck in CACHE: + return CACHE[ck] + for hit in tvdb("/search?query=" + urllib.parse.quote(name) + "&type=series") or []: + if eng.norm(hit.get("name") or "") != eng.norm(name): + continue + CACHE[ck] = {"name": (hit.get("name") or "").strip(), + "year": str(hit.get("year") or ""), + "id": str(hit.get("tvdb_id") or "")} + return CACHE[ck] + CACHE[ck] = None + return None + + +def tvdb_episodes(sid): + """`season/episode` -> title for one series, in the one call tvdb answers.""" + ck = f"tvdb:episodes:{sid}" + if ck in CACHE: + return CACHE[ck] + data = tvdb(f"/series/{sid}/episodes/default") + if not data: + return None + CACHE[ck] = {f"{e.get('seasonNumber')}/{e.get('number')}": + (e.get("name") or "").strip() + for e in data.get("episodes") or []} + return CACHE[ck] + # ------------------------------------------------------------------- the inputs rows = [] for line in open(ARGS.enum): @@ -264,6 +414,23 @@ for line in open(ARGS.enum): rows.append({"bytes": int(size), "ext": ext.lower(), "path": p, "base": base}) held = {r["path"]: r for r in rows} +def row_names_the_park(path, name): + """The library's own review pile is not a show. When the park sat INSIDE the + library the server indexed the whole folder as one series named after it, and + a row like that would rename a real show to `_unsorted` — so a row whose + series is the hidden top-level folder the file itself sits in is dropped.""" + top = rel(path).split("/")[0] + return top.startswith("_") and eng.norm(name) == eng.norm(top) + + +def in_review_pile(path): + """A file under the library's hidden top-level folder is review material by + that folder's own convention: its filename may MATCH a show a provider names + somewhere else, but it may never INVENT one — inventing one from a file's own + words is what filled the pile to begin with.""" + return rel(path).split("/")[0].startswith("_") + + titles = {} with open(ARGS.titles) as fh: cols = fh.readline().rstrip("\n").split("\t") @@ -272,7 +439,7 @@ with open(ARGS.titles) as fh: f += [""] * (len(cols) - len(f)) row = dict(zip(cols, f)) p = os.path.abspath(row.get("file", "")) - if p in held: + if p in held and not row_names_the_park(p, row.get("series", "")): titles[p] = row dur = {} @@ -379,14 +546,17 @@ def series_name(name): return clean(re.sub(r"\s{2,}", " ", PACK_IDX_RE.sub("", name)).strip(" .")) -def note(name, year="", tmdb_id=""): - """Register a series; the first name wins, the first year and id stick.""" +def note(name, year="", tmdb_id="", tvdb_id=""): + """Register a series; the first name wins, the first year and ids stick.""" k = eng.norm(name) - s = series.setdefault(k, {"name": series_name(name), "year": "", "tmdb": ""}) + s = series.setdefault(k, {"name": series_name(name), "year": "", "tmdb": "", + "tvdb": ""}) if year and not s["year"]: s["year"] = str(year) if tmdb_id and not s["tmdb"]: s["tmdb"] = str(tmdb_id) + if tvdb_id and not s["tvdb"]: + s["tvdb"] = str(tvdb_id) @@ -413,7 +583,9 @@ for r in video: # every file under SNL 16-20/Season 16..19 carries season 16 episode 20. owner = (t["series"] if t and t["series"] else series_of_dir(os.path.dirname(p)) - or title_of(r["base"][:marked[2]])[0]) + or known_in(r["base"]) + or ("" if in_review_pile(p) + else title_of(r["base"][:marked[2]])[0])) if not eng.norm(owner).replace(" ", ""): unknown.append((r, "an episode marker with no series name to hang it on")) continue @@ -427,7 +599,8 @@ for r in video: continue eps = [ep] claimants.append((r, owner, season, eps)) - note(owner, t["year"] if t else "", t["tmdb"] if t else "") + note(owner, t["year"] if t else "", t["tmdb"] if t else "", + t["tvdb"] if t else "") continue owner = series_of_dir(os.path.dirname(p)) if fs is not None and owner and leading_index(r["base"]): @@ -450,17 +623,80 @@ for k, s in series.items(): if got and eng.norm(got["name"]) == k: s["year"] = got["year"] -if ARGS.tmdb: +# tvdb is the identity authority. The row's own tvdb id is the door; a TMDb id +# crosses through TMDb's external ids; a bare name counts only when tvdb spells +# it the same and the years do not disagree. A series nothing resolves keeps the +# provider's words AND says so in the exceptions, which is where the operator +# finds the folders the plan did not rename to authority. +for s in series.values(): + got = tvdb_series(s["tvdb"]) if s["tvdb"] else None + if not got and s["tmdb"]: + sid = tvdb_id_for_tmdb(s["tmdb"]) + if sid: + s["tvdb"] = sid + got = tvdb_series(sid) + if not got: + hit = tvdb_search(s["name"]) + if hit and hit["id"] and (not s["year"] or not hit["year"] + or hit["year"] == s["year"]): + s["tvdb"] = hit["id"] + got = hit + if got and got["name"]: + s["canon"], s["canon_year"] = got["name"], got["year"] + +# One identity, one folder: the provider's spellings collapse to the shows tvdb +# says they are, so a box set split over five release folders is ONE series and +# two files claiming one episode MEET inside the duplicate rule below. +alias, canon = {}, {} +for k, s in series.items(): + key = f"id:{s['tvdb']}" if s["tvdb"] else k + alias[k] = key + keep = canon.setdefault(key, s) + if keep is not s: + for f in ("year", "tvdb", "canon", "canon_year"): + if not keep.get(f) and s.get(f): + keep[f] = s[f] + +if ARGS.tmdb or ARGS.tvdb: with open(ARGS.tmdb_cache, "w") as fh: json.dump(CACHE, fh, indent=1, sort_keys=True) +def identity(s): + """(name, year) a folder and its files carry: tvdb's whenever it answered, + the provider's own words only for a series nothing resolved.""" + if s.get("canon"): + return s["canon"], s["canon_year"] or s["year"] + return s["name"], s["year"] + + +def series_key(name): + """The merged identity a provider spelling belongs to.""" + return alias.get(eng.norm(name), eng.norm(name)) + + +def label_of(s): + """`<Series> (<year>)`, the form both matchers read.""" + nm, yr = identity(s) + nm = clean(nm) + return f"{nm} ({yr})" if yr else nm + + +def episode_title(r, s, season, ep): + """The title this episode carries: tvdb's for the id the series resolved to, + else the title the matched row states, else nothing at all — a dangling ` - ` + is not a title, and an invented one is worse than none.""" + if s["tvdb"]: + t = ((tvdb_episodes(s["tvdb"]) or {}).get(f"{season}/{ep}") or "").strip() + if t: + return clean(t) + t = ((titles.get(r["path"]) or {}).get("title") or "").strip() + return clean(t) if t else "" + + def series_dir(name): - s = series.get(eng.norm(name)) - if not s: - s = {"name": clean(name), "year": ""} - label = f"{s['name']} ({s['year']})" if s["year"] else s["name"] - return f"{ROOT}/{clean(label)}" + s = canon.get(series_key(name)) + return f"{ROOT}/{label_of(s)}" if s else f"{ROOT}/{clean(series_name(name))}" # ------------------------------------------------------------------- the plan @@ -500,25 +736,28 @@ def park(r, reason): by_ep = collections.defaultdict(list) for r, name, season, eps in claimants: for e in eps: - by_ep[(eng.norm(name), season, e)].append(r) + by_ep[(series_key(name), season, e)].append(r) winners, losers = set(), {} for (key, season, e), rs in by_ep.items(): rs.sort(key=lambda r: (-r["bytes"], r["path"])) winners.add(rs[0]["path"]) for r in rs[1:]: losers.setdefault(r["path"], - f"duplicate of {series[key]['name']} S{season:02d}E{e:02d}" + f"duplicate of {canon[key]['name']} S{season:02d}E{e:02d}" " (the bigger copy keeps the standard name)") for r, name, season, eps in claimants: if r["path"] in losers: park(r, losers[r["path"]]) continue - s = series[eng.norm(name)] - label = f" S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps) + s = canon[series_key(name)] + stem = f"{label_of(s)} - S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps) + title = episode_title(r, s, season, eps[0]) + if title: + stem += f" - {title}" part = PART_RE.search(os.path.splitext(r["base"])[0]) - tail = f" Part {part.group(1).upper()}" if part and len(eps) == 1 else "" - book(r, f"{series_dir(name)}/Season {season:02d}", s["name"] + label + tail + r["ext"], - "tv") + if part and len(eps) == 1: + stem += f" Part {part.group(1).upper()}" + book(r, f"{series_dir(name)}/Season {season:02d}", stem + r["ext"], "tv") for r, name, kind in extras: book(r, f"{series_dir(name)}/{kind}", @@ -680,7 +919,8 @@ def write_plan(): def write_exceptions(): - no_year = sorted(s["name"] for s in series.values() if not s["year"]) + no_year = sorted({s["name"] for s in canon.values() if not s["year"]}) + no_tvdb = sorted({identity(s)[0] for s in canon.values() if not s.get("canon")}) with open(ARGS.exceptions, "w") as fh: fh.write(f"TV relayout exceptions for {ROOT}\n\n") fh.write(f"Everything below is parked under {PARK}, keeping the relative path it\n") @@ -698,6 +938,12 @@ def write_exceptions(): fh.write(f"{len(no_year):>6} series carry no year from any provider, so their\n") fh.write(" folders are named without one; Jellyfin still matches them.\n") fh.write(" " + ", ".join(no_year[:300]) + "\n") + if no_tvdb: + fh.write(f"\n{len(no_tvdb):>6} series carry no tvdb identity, so their folders\n") + fh.write(" keep the name the metadata provider gave them and no episode\n") + fh.write(" title was taken. Nothing was invented: give these a resolvable\n") + fh.write(" id (--tvdb) and re-plan; a rename is idempotent.\n") + fh.write(" " + ", ".join(no_tvdb[:300]) + "\n") print(f"exceptions -> {ARGS.exceptions}") diff --git a/serverAppConfigs-docs/Dankserver/Media Relayout.md b/serverAppConfigs-docs/Dankserver/Media Relayout.md index ebf963b..1740802 100644 --- a/serverAppConfigs-docs/Dankserver/Media Relayout.md +++ b/serverAppConfigs-docs/Dankserver/Media Relayout.md @@ -187,12 +187,16 @@ show. A name whose marker is `Part 2` is a film and stays put. `/mnt/entropy/TVShows` is scene packs rather than a library, and Jellyfin has already matched and numbered most of it. `scripts/media-shows.py` re-files that -library into the same shape, and it never guesses: every series name and every -episode number it writes traces to an authority, and what has none is parked for -a human to read. +library into the shape Jellyfin reads, with **tvdb as the name authority**: + + <Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.ext + +It never guesses: every series name, episode number and title it writes traces to +an authority, and what has none is parked for a human to read. ```bash -# 1. The authority: Jellyfin's own matched rows (the DB opens read-only). +# 1. The authority: Jellyfin's own matched rows — each carries the series' TMDb +# and Tvdb ids and the title the row holds (the DB opens read-only). python3 scripts/media-census.py jellyfin --root /mnt/entropy/TVShows \ --out /tmp/tv_titles.tsv # 2. What is on disk (stat only), and the runtimes the film arm needs. @@ -201,10 +205,15 @@ python3 scripts/media-census.py enumerate --root /mnt/entropy/TVShows \ python3 scripts/media-census.py probe --root /mnt/entropy/TVShows \ --enum /tmp/tv_enum.tsv --jobs 6 --out /tmp/tv_probed.tsv # The probe is the slow step: ffprobe reads both ends of every file. -# 3. The plan. --tmdb needs TMDB_API_KEY; without it a series no provider dated -# is simply named without a year, which Jellyfin accepts. +# 3. The plan. --tmdb needs TMDB_API_KEY, --tvdb needs ~/.config/secrets/tvdb +# (`api_key=`, never argv); both write into the SAME cache file, so the plan is +# reproducible offline afterwards. Run it once to fill the cache, then again: +# the second run is the one whose numbers are quoted. python3 scripts/media-shows.py --root /mnt/entropy/TVShows --enum /tmp/tv_enum.tsv \ - --titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv --out /mnt/entropy/media-relayout-shows-plan.tsv \ + --titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv \ + --tmdb-cache /mnt/entropy/media-tv-names.json --tmdb --tvdb \ + --park /mnt/entropy/media-park/TVShows \ + --out /mnt/entropy/media-relayout-shows-plan.tsv \ --exceptions /mnt/entropy/media-relayout-shows-exceptions.txt # 4. Read the exceptions file FIRST — it is the review, not a log. Same executor # as above, but on this arm's OWN journal: a shared journal cannot be unwound. @@ -217,8 +226,11 @@ Evidence, strongest first — the order the planner believes things: - the **episode marker the file's own name states** (`S01E02-E03`, `1x04`); - the **`Season NN` folder it sits in and the number its name leads with inside it**: `Season 19/01 Charles Barkley, Nirvana.avi` is S19E01; -- the **provider row**: identity (the series name and its year), and the - numbering of a file whose own name says nothing; +- the **provider row**: which episode the file IS, and the numbering of a file + whose own name says nothing; +- the **row's tvdb id, else its TMDb id carried through TMDb's external ids, else + an exact tvdb name search**: that id names the SERIES FOLDER and supplies the + episode title, because a name the server read off a folder is not a show name; - a **bonus folder** anywhere above the file, or Jellyfin's `ExtraType` → the series' canonical extra folder; - a runtime over `SHORT_MAX` makes it film-shaped, and it leaves for `--movies` @@ -237,11 +249,22 @@ Two refusals carry the design, and a dry run on the real library found both: the file's own leading number, win; a row that still insists on season 1989 parks its file instead of creating `Season 1989`. -The provider's name is copied verbatim, warts included: Jellyfin carries a Series -named `Black Clover 001` (92 episodes, no year) because it matched a pack off a -filename. Renaming a show is Jellyfin's work — a folder rename never touches its -provider ids — so the mover keeps the name it is handed and strips only the pack -index a release leads with (`4a. Seed DESTINY` → `Seed DESTINY`). +**A provider name is evidence only when an id backs it.** Jellyfin names a Series +after the folder it scanned, so a library filed by release folders ends up with +one Series item per box set — `24 (2001) Season 1`, `Modern Family S05` — and +copying those names verbatim launders folder junk into the contract. The planner +resolves each series to its tvdb id and writes tvdb's name and year; the merged +identity is also what turns a box set split across five release folders into ONE +folder, and what makes two files claiming one episode MEET in the duplicate rule. +A series no id resolves keeps the provider's words and is listed under *no tvdb +identity* in the exceptions — that list, not a rename, is the review. + +**The review pile lives outside the library.** Files parked inside +`/mnt/entropy/TVShows` were indexed as one show named `_unsorted`, and that +show's rows then looked like evidence. So a row whose series is the hidden +top-level folder the file itself sits in is dropped, and `--park` outside the root +makes the next plan re-file the pile instead of skipping it. Inside a hidden +top-level folder a file's own words may MATCH a registered show, never invent one. ## The music library's own relayout diff --git a/tests/media_shows.bats b/tests/media_shows.bats index fd48a5f..6502013 100755 --- a/tests/media_shows.bats +++ b/tests/media_shows.bats @@ -8,7 +8,14 @@ # file name. So every test here states which authority a file had — name only, # provider only, both, neither — and asserts where the PLAN files it, never how # the planner decided. A fixture Jellyfin database stands in for the live one and -# the TMDb cache is an input file, so no test touches the network. +# the name caches are input files, so no test touches the network. +# +# The target contract is tvdb-grade naming: +# <Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.<ext> +# tvdb is the IDENTITY authority: a provider name a server derived from a folder +# is not a series name, so a polluted name is undone through the series' tvdb id. +# The review pile parks OUTSIDE the library — a parked folder inside it indexes +# as one junk show, and that show's rows are then mistaken for evidence. # # The two outcomes that cost a re-run of a 50k-file plan are a file filed as an # episode that is not one (it steals an episode number from its real owner) and a @@ -25,11 +32,11 @@ APPLY="$REPO_ROOT/scripts/media-apply.py" # at PATH SIZE — a file with deterministic content. at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; } -# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] — one row of the authority -# export scripts/media-census.py jellyfin writes. +# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] [TMDB] [TITLE] [TVDB] +# — one row of the authority export scripts/media-census.py jellyfin writes. title_row() { - printf '%s\t%s\t%s\t%s\t%s\t%s\t\n' "$1" "$2" "$3" "${4:-}" "${5:-}" "${6:-}" \ - >> "$TITLES" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "${4:-}" \ + "${5:-}" "${6:-}" "${7:-}" "${8:-}" "${9:-}" >> "$TITLES" } # dst SRC — the destination the plan gives SRC, or nothing when it does not move. @@ -40,6 +47,7 @@ dst() { setup() { LIB="$BATS_TEST_TMPDIR/TVShows" PARK="$LIB/_unsorted" + OUTSIDE="$BATS_TEST_TMPDIR/park" MOVIES="$BATS_TEST_TMPDIR/Movies" ENUM="$BATS_TEST_TMPDIR/enum.tsv" PROBED="$BATS_TEST_TMPDIR/probed.tsv" @@ -77,8 +85,14 @@ setup() { # A season folder whose rows are stale: the database numbers every file of this # pack season 16 episode 20, four seasons' worth of them. at "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" 6000 - # One episode-shaped file loose at the library root. at "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" 4000 + # A box-set folder whose Series item the server named AFTER THE FOLDER: the + # row's own name is junk, and only a provider id can undo it. + BOX="$LIB/Show Box (2016) Season 1 S01 (1080p WEB-DL)" + at "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" 6000 + # The park inside the library: the server indexed the whole pile as ONE show + # named after the folder, so its rows state a series that is not a show. + at "$PARK/Park Show S01E01.mkv" 5000 # The enum census lists every file; the planner reads that instead of walking # the tree, so the fixture writes the same three columns. @@ -92,17 +106,21 @@ with open(out, "w") as fh: fh.write(f"{os.stat(p).st_size}\t{os.path.splitext(f)[1]}\t{p}\n") PY - printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\n' > "$TITLES" + printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\ttitle\ttvdb\n' > "$TITLES" printf '{}\n' > "$CACHE" title_row "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" \ - "The Examples" 2010 1 1 + "The Examples" 2010 1 1 "" "" "Pilot" title_row "$ANIME/[grp] Anime Pack - 05.mkv" "Anime Pack" "" 1 5 - title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1 - title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1 - title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2 - title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3 + title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots" + title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots" + title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2 "" "" "Double Trouble" + title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3 "" "" "Loose Ends" title_row "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" \ "SNL Pack" "" 16 20 + title_row "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" \ + "Show Box (2016) Season 1" 2016 1 1 "" "2334" "Pilot" "78850" + # The row the server wrote for a parked file: the series is the PARK FOLDER. + title_row "$PARK/Park Show S01E01.mkv" "_unsorted" "" 1 1 } plan() { @@ -111,6 +129,12 @@ plan() { --out "$OUT" --exceptions "$EXC" } +plan_park() { # the review pile outside the library + run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --titles "$TITLES" \ + --probed "$PROBED" --tmdb-cache "$CACHE" --movies "$MOVIES" \ + --park "$OUTSIDE" --out "$OUT" --exceptions "$EXC" +} + @test "the census exports the provider's episode numbering as a TSV" { DB="$BATS_TEST_TMPDIR/jellyfin.db" python3 - "$DB" "$SEASON" <<'PY' @@ -125,7 +149,7 @@ sid = str(uuid.uuid4()) c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)", (sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples", pre, None, None, 2010, None, 0, - '{"ProviderIds":{"Tmdb":"1234"}}')) + '{"ProviderIds":{"Tmdb":"1234","Tvdb":"78850"}}')) c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)", (str(uuid.uuid4()), sid, "MediaBrowser.Controller.Entities.TV.Episode", "Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1, @@ -136,34 +160,87 @@ PY --out "$BATS_TEST_TMPDIR/exp.tsv" [ "$status" -eq 0 ] run head -1 "$BATS_TEST_TMPDIR/exp.tsv" - [ "${lines[0]}" = "file series year season episode extra tmdb" ] + [ "${lines[0]}" = "file series year season episode extra tmdb title tvdb" ] tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \ - | grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234$" + | grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234 Pilot 78850$" } -@test "an episode its own name states files as Show SxxEyy under Season NN" { +@test "the census reads provider ids from the table newer Jellyfin keeps" { + DB="$BATS_TEST_TMPDIR/jellyfin12.db" + python3 - "$DB" "$SEASON" <<'PY' +import os, sqlite3, sys, uuid +db, season = sys.argv[1], sys.argv[2] +c = sqlite3.connect(db) +pre = "/data/TVShows/" + os.path.basename(season) +c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber, + ParentIndexNumber, ProductionYear, ExtraType, + IsVirtualItem, Data)""") +# Jellyfin 12 moved the ids out of the Data blob and into their own table. +c.execute("create table BaseItemProviders (ItemId, ProviderId, ProviderValue)") +sid = str(uuid.uuid4()) +eid = str(uuid.uuid4()) +c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)", + (sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples", + pre, None, None, 2010, None, 0, '{}')) +c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)", + (eid, sid, "MediaBrowser.Controller.Entities.TV.Episode", + "Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1, + None, None, 0, '{}')) +c.execute("insert into BaseItemProviders values (?,?,?)", (sid, "Tvdb", "9999")) +c.commit() +PY + run python3 "$CENSUS" jellyfin --root "$LIB" --db "$DB" \ + --out "$BATS_TEST_TMPDIR/exp12.tsv" + [ "$status" -eq 0 ] + tail -1 "$BATS_TEST_TMPDIR/exp12.tsv" \ + | grep -q " Pilot 9999$" +} + +@test "an episode its own name states files as the titled standard name" { plan [ "$status" -eq 0 ] [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv")" \ - = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.mkv" ] + = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.mkv" ] +} + +@test "the tvdb identity names the series folder, not the name a folder left behind" { + printf '{"tvdb:series:78850": {"name": "Show Box", "year": "2016"}}\n' > "$CACHE" + plan + [ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" \ + = "$LIB/Show Box (2016)/Season 01/Show Box (2016) - S01E01 - Pilot.mkv" ] +} + +@test "a series no authority resolves keeps the provider's name and is reported" { + plan + [ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \ + = "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ] + grep -A9 -i "no tvdb" "$EXC" | grep -q "SNL Pack" +} + +@test "a row whose series is the park folder is not evidence of a show" { + plan_park + run awk -F '\t' -v p="$PARK/" '$1 == "move" && index($3, p) == 1 {print $3}' "$OUT" + [ -z "$output" ] + run awk -F '\t' '$1 == "move" && $2 ~ /Park Show S01E01/ {print $3}' "$OUT" + [[ "$output" == "$OUTSIDE/"* ]] } @test "a subtitle follows its episode and keeps its language flag" { plan [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt")" \ - = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.en.srt" ] + = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.en.srt" ] } @test "a subtitle two folders down inside Subs/<release-name>/ still follows its episode" { plan [ "$(dst "$SEASON/Subs/The.Examples.S01E01.Pilot.1080p.x265/The.Examples.S01E01.Pilot.1080p.x265.jpn.srt")" \ - = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.jpn.srt" ] + = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.jpn.srt" ] } @test "a subtitle that carries the container's word in its own name still follows the episode" { plan [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv.forced.srt")" \ - = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.forced.srt" ] + = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.forced.srt" ] } @test "a season image keeps its own name; a series nfo keeps the series folder" { @@ -178,10 +255,10 @@ PY [ -z "$(awk -F '\t' '$3 ~ /Season .*teaser/ {print $3}' "$OUT")" ] } -@test "a file only the provider numbers files at the episode it is matched to" { +@test "a file only the provider numbers files at the episode it is matched to, title or no title" { plan [ "$(dst "$ANIME/[grp] Anime Pack - 05.mkv")" \ - = "$LIB/Anime Pack/Season 01/Anime Pack S01E05.mkv" ] + = "$LIB/Anime Pack/Season 01/Anime Pack - S01E05.mkv" ] } @test "a bonus folder inside a release pack belongs to the pack's show, not to its own name" { @@ -193,19 +270,19 @@ PY @test "a Season folder outvotes the stale row its files carry" { plan [ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \ - = "$LIB/SNL Pack/Season 19/SNL Pack S19E01.avi" ] + = "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ] } @test "one file holding two episodes keeps both numbers in its name" { plan [ "$(dst "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv")" \ - = "$LIB/Show Two (2016)/Season 01/Show Two S01E02-E03.mkv" ] + = "$LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E02-E03 - Double Trouble.mkv" ] } @test "two files claiming one episode: one keeps the name, the other parks" { plan - n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two S01E01.mkv " "$OUT") + n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E01 - Two Pilots.mkv " "$OUT") [ "$n" -eq 1 ] run dst "$LIB/Dupes/Show Two S01E01.mkv" [[ "$output" == "$PARK/"* ]]