scripts/media-census.py

@@ -21,11 +21,12 @@ scripts/media-plan.py and scripts/media-shows.py read them instead of touching m
           the titles Plex already matched — the authority over any filename guess
 
   jellyfin file <TAB> series <TAB> year <TAB> season <TAB> episode <TAB> extra
-          <TAB> tmdb      (see JF_COLS)
+          <TAB> tmdb <TAB> title <TAB> tvdb      (see JF_COLS)
           one row per episode Jellyfin has matched and numbered, its path mapped
-          from the container's view back to the host; `year` and `tmdb` are the
+          from the container's view back to the host; `year`, `tmdb` and `tvdb`
-          SERIES' values, `extra` is Jellyfin's own extra type when the row is
+          are the SERIES' values, `title` is the episode title the row carries,
-          bonus material rather than an episode
+          `extra` is Jellyfin's own extra type when the row is bonus material
+          rather than an episode
 
   music file <TAB> artist <TAB> album <TAB> year <TAB> disc <TAB> track <TAB> title
          <TAB> mb <TAB> performer   (see MUSIC_COLS)
@@ -83,13 +84,14 @@ left join metadata_items par on par.id = mi.parent_id
 where p.file like ? and p.deleted_at is null and mi.deleted_at is null
 """
 JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db"
-JF_COLS = ["file", "series", "year", "season", "episode", "extra", "tmdb"]
+JF_COLS = ["file", "series", "year", "season", "episode", "extra",
+           "tmdb", "title", "tvdb"]
 # Jellyfin stores the entity's .NET class as its type; the TV tree is these three.
 JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode",
          "MediaBrowser.Controller.Entities.TV.Season",
          "MediaBrowser.Controller.Entities.TV.Series")
-JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, IsVirtualItem
+JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType,
-              from BaseItems where Type = ? and Path like ?""")
+                   IsVirtualItem, Name from BaseItems where Type = ? and Path like ?""")
 
 # The music tree is these two entity classes. The track row carries the tags
 # Jellyfin read out of the file; the album row carries what the provider matched,
@@ -254,9 +256,10 @@ def jellyfin(root, db, container, out):
 
     The numbering is what matters, not the name: a release group's file name says
     nothing about which episode a file is, and Jellyfin has already answered that
-    question against a provider. Season/episode come from the episode row; the
+    question against a provider. Season/episode and the episode's own title come
-    series name, its year and its TMDb id come from the Series row the episode
+    from the episode row; the series name, its year and its TMDb and Tvdb ids come
-    climbs to, because those are the only place Jellyfin keeps them.
+    from the Series row the episode climbs to, because those are the only place
+    Jellyfin keeps them.
     """
     try:
         con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
@@ -277,21 +280,38 @@ def jellyfin(root, db, container, out):
             " where Type in (?,?,?)", JF_TV):
         node[i] = (pid, typ, name, year, data)
 
+    # Jellyfin 12 keeps provider ids in their own table; older builds kept them in
+    # the row's Data blob. Both shapes are read, because the tvdb id the series
+    # naming depends on is wherever this build happened to write it.
+    prov = {}
+    if "BaseItemProviders" in {r[0] for r in con.execute(
+            "select name from sqlite_master where type='table'")}:
+        for iid, pid, val in con.execute(
+                "select ItemId, ProviderId, ProviderValue from BaseItemProviders"
+                " where ProviderId in ('Tmdb', 'Tvdb')"):
+            prov[(iid, pid)] = val
+
+    def provider_of(item_id, want):
+        got = prov.get((item_id, want))
+        if got:
+            return str(got)
+        try:
+            blob = json.loads(node[item_id][4] or "{}").get("ProviderIds") or {}
+        except (KeyError, ValueError, TypeError):
+            return ""
+        return str(blob.get(want, "") or "")
+
     def series_of(item_id):
         """Climb episode -> season -> series; empty when the chain is broken."""
         for _ in range(4):
             if item_id not in node:
-                return "", "", ""
+                return "", "", "", ""
-            pid, typ, name, year, data = node[item_id]
+            pid, typ = node[item_id][0], node[item_id][1]
             if typ.endswith(".Series"):
-                try:
+                return (node[item_id][2] or "", node[item_id][3] or "",
-                    tmdb = (json.loads(data or "{}")
+                        provider_of(item_id, "Tmdb"), provider_of(item_id, "Tvdb"))
-                            .get("ProviderIds") or {}).get("Tmdb", "")
-                except ValueError:
-                    tmdb = ""
-                return name or "", year or "", tmdb
             item_id = pid
-        return "", "", ""
+        return "", "", "", ""
 
     # The fleet's quadlets bind the share at /data, so the container's view of
     # this library is /data/<the library's own folder name> unless told otherwise.
@@ -302,17 +322,18 @@ def jellyfin(root, db, container, out):
     with open(out, "w", newline="") as fh:
         w = csv.writer(fh, delimiter="\t", lineterminator="\n")
         w.writerow(JF_COLS)
-        for iid, path, idx, pidx, extra, virt in con.execute(JF_SQL,
+        for iid, path, idx, pidx, extra, virt, epname in con.execute(
-                                                             (JF_TV[0], prefix + "%")):
+                JF_SQL, (JF_TV[0], prefix + "%")):
             if virt or not path or not path.startswith(prefix):
                 skipped += 1          # a virtual item is a variant, not a file
                 continue
-            sname, syear, tmdb = series_of(iid)
+            sname, syear, tmdb, tvdb = series_of(iid)
             w.writerow([root + "/" + path[len(prefix):],
                         sname.replace("\t", " "), syear,
                         "" if pidx is None else pidx,
                         "" if idx is None else idx,
-                        (extra or "").replace("\t", " "), tmdb])
+                        (extra or "").replace("\t", " "), tmdb,
+                        (epname or "").replace("\t", " "), tvdb])
             written += 1
     print(f"episodes={written} virtual-or-blank={skipped} -> {out}")
 

scripts/media-shows.py

@@ -6,29 +6,40 @@ r"""Plan a Jellyfin-format relayout of a TV library. DRY RUN ONLY: nothing moves
                 [--out TSV] [--exceptions TXT]
 
 Inputs are the TSVs scripts/media-census.py exports: `enumerate` lists what
-exists, `jellyfin` says which episode each file IS, and `probe` is optional
+exists, `jellyfin` says which episode each file IS (its series' TMDb and Tvdb
-duration evidence for the movie rule. The TMDb cache is a plain JSON file so a
+ids, and the episode title the row carries), and `probe` is optional duration
-plan stays reproducible: answers are read from it, and a miss is looked up only
+evidence for the movie rule. Every name and title answer lives in the cache JSON
-with --tmdb and TMDB_API_KEY in the environment (the key never enters argv, and
+so a plan stays reproducible: answers are read from it, and a miss is looked up
+only with --tmdb/--tvdb and their credentials (a key never enters argv, and
 never the cache).
 
 Target contract (jellyfin.org/docs/general/server/media/shows):
 
-    <root>/<Series> (<year>)/Season <NN>/<Series> SxxEyy.ext
+    <root>/<Series> (<year>)/Season <NN>/<Series> (<year>) - SxxEyy - <Title>.ext
     one file holding two episodes   SxxE01-E02      specials       Season 00
     a documented part of an episode SxxE03 Part 1   sidecars       <base>.<lang>.srt
     bonus material                  <series>/trailers|featurettes|… — never a season
 
+The series name and the episode title are the ones the IDENTITY authority — tvdb
+— states for the id the metadata server matched, not the words a release folder
+or a server's folder-derived Series name happens to carry.
+
 PLAN is the TSV scripts/media-apply.py replays: `action src dst kind`, every
 mkdir row before any move, and `keep` for a file already where the contract puts
 it. EXCEPTIONS is the review list: everything parked, and why.
 
 The authority chain, strongest first, for one video file:
 
-  1. a provider row — Jellyfin matched this exact path and numbered it.
+  1. a provider row — Jellyfin matched this exact path and numbered it. The
+     row's NAME is only evidence once an id backs it: the server names a Series
+     after the folder it scanned, so `24 (2001) Season 1` is a folder echo, and
+     the series' tvdb id is what undoes it.
   2. the file's own SxxEyy / 1x04 marker, the series taken from a sibling's
      provider row, else from the nearest ancestor folder that has one, else from
-     the release folder's own words.
+     a registered show its own words name, else — outside the review pile — from
+     the release folder's words. Inside a hidden top-level folder a file may
+     MATCH a registered show but never INVENT one: invention is what filled the
+     pile to begin with.
   3. nothing. An unnumbered file is PARKED under --park with its relative path
      kept: an invented `Season 01` steals an episode number from the file that
      really is S01E01, and a wrong number is worse than no number.
@@ -47,8 +58,11 @@ Guards, and why each is here:
   * Series art and the show nfo follow the SERIES, a season image follows its
     SEASON, a subtitle or episode nfo follows the episode's new base — an episode
     moved without its subtitles is a half-move.
-  * Hidden files, macOS forks, qBittorrent resume parts and the park pile itself
+  * Hidden files, macOS forks, qBittorrent resume parts are never moved. A file
-    are never moved. Skipping the park is what makes re-planning stable.
+    already inside --park is never moved either: that is what makes re-planning
+    stable. --park belongs OUTSIDE the library — parked files inside it get
+    indexed as one show named after the park folder, and a row like that is a
+    folder echo, never evidence (the guard in `row_names_the_park`).
 
 The plan crosses three trees (the library, its park, the movie library), so the
 executor runs with --root at the share that holds them, on its own journal —
@@ -56,7 +70,8 @@ one journal is one undo:
 
     python3 scripts/media-shows.py --root /mnt/entropy/TVShows \
         --enum /tmp/tv-enum.tsv --titles /tmp/jellyfin-tv.tsv \
-        --out /mnt/entropy/media-tv-plan.tsv
+        --tvdb --tmdb --out /mnt/entropy/media-tv-plan.tsv \
+        --park /mnt/entropy/media-park/TVShows
     python3 scripts/media-apply.py apply --plan /mnt/entropy/media-tv-plan.tsv \
         --root /mnt/entropy --journal /mnt/entropy/media-tv-relayout.jsonl
 """
@@ -80,14 +95,21 @@ ARGS.add_argument("--titles", default="/tmp/jellyfin-tv.tsv",
                   help="the provider's numbering (media-census.py jellyfin)")
 ARGS.add_argument("--probed", default="",
                   help="ffprobe census for the movie rule; absent = no durations")
-ARGS.add_argument("--tmdb-cache", default="/tmp/tmdb-cache.json",
+ARGS.add_argument("--tmdb-cache", default="/tmp/tv-names.json",
-                  help="TMDb answers as an input file, so a plan is reproducible")
+                  help="name answers (TMDb and Tvdb) as ONE input file, so a plan "
+                       "is reproducible; keys carry the provider they came from")
 ARGS.add_argument("--tmdb", action="store_true",
-                  help="look up cache misses (needs TMDB_API_KEY)")
+                  help="look up cache misses at TMDb (needs TMDB_API_KEY)")
+ARGS.add_argument("--tvdb", action="store_true",
+                  help="look up cache misses at Tvdb (needs the credentials file)")
+ARGS.add_argument("--tvdb-credentials",
+                  default=os.path.expanduser("~/.config/secrets/tvdb"),
+                  help="file with an api_key= line (and a pin= line for a "
+                       "user-issued key); mode 600, and never the cache")
 ARGS.add_argument("--movies", default="/mnt/entropy/Movies",
                   help="movie library a mis-filed film moves into")
 ARGS.add_argument("--park", default="",
-                  help="review pile (default: <root>/_unsorted)")
+                  help="review pile OUTSIDE the library (default: <root>/_unsorted)")
 ARGS.add_argument("--out", default="/tmp/tv-plan.tsv")
 ARGS.add_argument("--exceptions", default="/tmp/tv-exceptions.txt")
 ARGS = ARGS.parse_args()
@@ -251,6 +273,134 @@ def tmdb(kind, query):
     return CACHE[ck]
 
 
+# ---------------------------------------------------------------- tvdb, the identity
+# tvdb names the show and the episode: it is the only provider whose strings are
+# the ones the operator asked for, so its answer replaces a folder-derived Series
+# name. The login mints a JWT per run from a mode-600 file; neither the key nor
+# the token reaches argv, the cache, or the plan. A provider that will not answer
+# never invents a name: every lookup here returns None, and None keeps the
+# provider's own words and lands in the exceptions.
+TVDB = {"jwt": None, "dead": False}
+
+
+def http_json(url, body=None, headers=None):
+    req = urllib.request.Request(
+        url, data=json.dumps(body).encode() if body is not None else None,
+        headers={"Accept": "application/json", "Content-Type": "application/json",
+                 **(headers or {})})
+    with urllib.request.urlopen(req, timeout=30) as r:
+        return json.load(r)
+
+
+def tvdb_login():
+    """The run's JWT, or None: a project key needs only api_key=, a user-issued
+    key also needs the pin."""
+    if TVDB["jwt"] is not None:
+        return TVDB["jwt"] or None
+    TVDB["jwt"] = ""
+    try:
+        lines = open(ARGS.tvdb_credentials).read().splitlines()
+    except OSError:
+        print(f"tvdb: no credentials at {ARGS.tvdb_credentials}", file=sys.stderr)
+        return None
+    sec = {}
+    for line in lines:
+        line = line.strip()
+        if line and not line.startswith("#") and "=" in line:
+            k, v = line.split("=", 1)
+            sec[k.strip()] = v.strip()
+    if not sec.get("api_key"):
+        print(f"tvdb: no api_key in {ARGS.tvdb_credentials}", file=sys.stderr)
+        return None
+    body = {"apiKey": sec["api_key"]}
+    if sec.get("pin"):
+        body["token"] = sec["pin"]
+    try:
+        TVDB["jwt"] = (http_json("https://api4.thetvdb.com/v4/login", body)
+                       .get("data") or {}).get("token") or ""
+    except Exception as e:                                # noqa: BLE001
+        print(f"tvdb login failed: {e}", file=sys.stderr)
+        return None
+    return TVDB["jwt"] or None
+
+
+def tvdb(path):
+    """One Tvdb v4 GET, or None when it cannot be answered."""
+    if not ARGS.tvdb or TVDB["dead"]:
+        return None
+    jwt = tvdb_login()
+    if not jwt:
+        TVDB["dead"] = True
+        return None
+    try:
+        return http_json("https://api4.thetvdb.com/v4" + path,
+                         headers={"Authorization": "Bearer " + jwt}).get("data")
+    except Exception as e:                                # noqa: BLE001
+        print(f"tvdb {path} failed: {e}", file=sys.stderr)
+        return None
+
+
+def tvdb_series(sid):
+    """{name, year} for a tvdb series id."""
+    ck = f"tvdb:series:{sid}"
+    if ck in CACHE:
+        return CACHE[ck]
+    data = tvdb(f"/series/{sid}")
+    if not data:
+        return None
+    CACHE[ck] = {"name": (data.get("name") or "").strip(),
+                 "year": (data.get("firstAired") or "")[:4]}
+    return CACHE[ck]
+
+
+def tvdb_id_for_tmdb(mid):
+    """The tvdb id TMDb records for a series id — the bridge for the bulk of this
+    library, which the server matched by TMDb."""
+    ck = f"tmdb:external:{mid}"
+    if ck in CACHE:
+        return CACHE[ck] or ""
+    if not (ARGS.tmdb and API_KEY):
+        return ""
+    try:
+        got = http_json(f"https://api.themoviedb.org/3/tv/{mid}/external_ids",
+                        headers={"Authorization": "Bearer " + API_KEY})
+    except Exception as e:                                # noqa: BLE001
+        print(f"tmdb external ids failed for {mid}: {e}", file=sys.stderr)
+        return ""
+    CACHE[ck] = str(got.get("tvdb_id") or "")
+    return CACHE[ck]
+
+
+def tvdb_search(name):
+    """The tvdb series a provider name states, or None: only tvdb's own spelling
+    counts, so a near-miss never renames a show."""
+    ck = f"tvdb:search:{name}"
+    if ck in CACHE:
+        return CACHE[ck]
+    for hit in tvdb("/search?query=" + urllib.parse.quote(name) + "&type=series") or []:
+        if eng.norm(hit.get("name") or "") != eng.norm(name):
+            continue
+        CACHE[ck] = {"name": (hit.get("name") or "").strip(),
+                     "year": str(hit.get("year") or ""),
+                     "id": str(hit.get("tvdb_id") or "")}
+        return CACHE[ck]
+    CACHE[ck] = None
+    return None
+
+
+def tvdb_episodes(sid):
+    """`season/episode` -> title for one series, in the one call tvdb answers."""
+    ck = f"tvdb:episodes:{sid}"
+    if ck in CACHE:
+        return CACHE[ck]
+    data = tvdb(f"/series/{sid}/episodes/default")
+    if not data:
+        return None
+    CACHE[ck] = {f"{e.get('seasonNumber')}/{e.get('number')}":
+                 (e.get("name") or "").strip()
+                 for e in data.get("episodes") or []}
+    return CACHE[ck]
+
 # ------------------------------------------------------------------- the inputs
 rows = []
 for line in open(ARGS.enum):
@@ -264,6 +414,23 @@ for line in open(ARGS.enum):
     rows.append({"bytes": int(size), "ext": ext.lower(), "path": p, "base": base})
 held = {r["path"]: r for r in rows}
 
+def row_names_the_park(path, name):
+    """The library's own review pile is not a show. When the park sat INSIDE the
+    library the server indexed the whole folder as one series named after it, and
+    a row like that would rename a real show to `_unsorted` — so a row whose
+    series is the hidden top-level folder the file itself sits in is dropped."""
+    top = rel(path).split("/")[0]
+    return top.startswith("_") and eng.norm(name) == eng.norm(top)
+
+
+def in_review_pile(path):
+    """A file under the library's hidden top-level folder is review material by
+    that folder's own convention: its filename may MATCH a show a provider names
+    somewhere else, but it may never INVENT one — inventing one from a file's own
+    words is what filled the pile to begin with."""
+    return rel(path).split("/")[0].startswith("_")
+
+
 titles = {}
 with open(ARGS.titles) as fh:
     cols = fh.readline().rstrip("\n").split("\t")
@@ -272,7 +439,7 @@ with open(ARGS.titles) as fh:
         f += [""] * (len(cols) - len(f))
         row = dict(zip(cols, f))
         p = os.path.abspath(row.get("file", ""))
-        if p in held:
+        if p in held and not row_names_the_park(p, row.get("series", "")):
             titles[p] = row
 
 dur = {}
@@ -379,14 +546,17 @@ def series_name(name):
     return clean(re.sub(r"\s{2,}", " ", PACK_IDX_RE.sub("", name)).strip(" ."))
 
 
-def note(name, year="", tmdb_id=""):
+def note(name, year="", tmdb_id="", tvdb_id=""):
-    """Register a series; the first name wins, the first year and id stick."""
+    """Register a series; the first name wins, the first year and ids stick."""
     k = eng.norm(name)
-    s = series.setdefault(k, {"name": series_name(name), "year": "", "tmdb": ""})
+    s = series.setdefault(k, {"name": series_name(name), "year": "", "tmdb": "",
+                              "tvdb": ""})
     if year and not s["year"]:
         s["year"] = str(year)
     if tmdb_id and not s["tmdb"]:
         s["tmdb"] = str(tmdb_id)
+    if tvdb_id and not s["tvdb"]:
+        s["tvdb"] = str(tvdb_id)
 
 
 
@@ -413,7 +583,9 @@ for r in video:
         # every file under SNL 16-20/Season 16..19 carries season 16 episode 20.
         owner = (t["series"] if t and t["series"]
                  else series_of_dir(os.path.dirname(p))
-                 or title_of(r["base"][:marked[2]])[0])
+                 or known_in(r["base"])
+                 or ("" if in_review_pile(p)
+                     else title_of(r["base"][:marked[2]])[0]))
         if not eng.norm(owner).replace(" ", ""):
             unknown.append((r, "an episode marker with no series name to hang it on"))
             continue
@@ -427,7 +599,8 @@ for r in video:
                 continue
             eps = [ep]
         claimants.append((r, owner, season, eps))
-        note(owner, t["year"] if t else "", t["tmdb"] if t else "")
+        note(owner, t["year"] if t else "", t["tmdb"] if t else "",
+             t["tvdb"] if t else "")
         continue
     owner = series_of_dir(os.path.dirname(p))
     if fs is not None and owner and leading_index(r["base"]):
@@ -450,17 +623,80 @@ for k, s in series.items():
         if got and eng.norm(got["name"]) == k:
             s["year"] = got["year"]
 
-if ARGS.tmdb:
+# tvdb is the identity authority. The row's own tvdb id is the door; a TMDb id
+# crosses through TMDb's external ids; a bare name counts only when tvdb spells
+# it the same and the years do not disagree. A series nothing resolves keeps the
+# provider's words AND says so in the exceptions, which is where the operator
+# finds the folders the plan did not rename to authority.
+for s in series.values():
+    got = tvdb_series(s["tvdb"]) if s["tvdb"] else None
+    if not got and s["tmdb"]:
+        sid = tvdb_id_for_tmdb(s["tmdb"])
+        if sid:
+            s["tvdb"] = sid
+            got = tvdb_series(sid)
+    if not got:
+        hit = tvdb_search(s["name"])
+        if hit and hit["id"] and (not s["year"] or not hit["year"]
+                                  or hit["year"] == s["year"]):
+            s["tvdb"] = hit["id"]
+            got = hit
+    if got and got["name"]:
+        s["canon"], s["canon_year"] = got["name"], got["year"]
+
+# One identity, one folder: the provider's spellings collapse to the shows tvdb
+# says they are, so a box set split over five release folders is ONE series and
+# two files claiming one episode MEET inside the duplicate rule below.
+alias, canon = {}, {}
+for k, s in series.items():
+    key = f"id:{s['tvdb']}" if s["tvdb"] else k
+    alias[k] = key
+    keep = canon.setdefault(key, s)
+    if keep is not s:
+        for f in ("year", "tvdb", "canon", "canon_year"):
+            if not keep.get(f) and s.get(f):
+                keep[f] = s[f]
+
+if ARGS.tmdb or ARGS.tvdb:
     with open(ARGS.tmdb_cache, "w") as fh:
         json.dump(CACHE, fh, indent=1, sort_keys=True)
 
 
+def identity(s):
+    """(name, year) a folder and its files carry: tvdb's whenever it answered,
+    the provider's own words only for a series nothing resolved."""
+    if s.get("canon"):
+        return s["canon"], s["canon_year"] or s["year"]
+    return s["name"], s["year"]
+
+
+def series_key(name):
+    """The merged identity a provider spelling belongs to."""
+    return alias.get(eng.norm(name), eng.norm(name))
+
+
+def label_of(s):
+    """`<Series> (<year>)`, the form both matchers read."""
+    nm, yr = identity(s)
+    nm = clean(nm)
+    return f"{nm} ({yr})" if yr else nm
+
+
+def episode_title(r, s, season, ep):
+    """The title this episode carries: tvdb's for the id the series resolved to,
+    else the title the matched row states, else nothing at all — a dangling ` - `
+    is not a title, and an invented one is worse than none."""
+    if s["tvdb"]:
+        t = ((tvdb_episodes(s["tvdb"]) or {}).get(f"{season}/{ep}") or "").strip()
+        if t:
+            return clean(t)
+    t = ((titles.get(r["path"]) or {}).get("title") or "").strip()
+    return clean(t) if t else ""
+
+
 def series_dir(name):
-    s = series.get(eng.norm(name))
+    s = canon.get(series_key(name))
-    if not s:
+    return f"{ROOT}/{label_of(s)}" if s else f"{ROOT}/{clean(series_name(name))}"
-        s = {"name": clean(name), "year": ""}
-    label = f"{s['name']} ({s['year']})" if s["year"] else s["name"]
-    return f"{ROOT}/{clean(label)}"
 
 
 # ------------------------------------------------------------------- the plan
@@ -500,25 +736,28 @@ def park(r, reason):
 by_ep = collections.defaultdict(list)
 for r, name, season, eps in claimants:
     for e in eps:
-        by_ep[(eng.norm(name), season, e)].append(r)
+        by_ep[(series_key(name), season, e)].append(r)
 winners, losers = set(), {}
 for (key, season, e), rs in by_ep.items():
     rs.sort(key=lambda r: (-r["bytes"], r["path"]))
     winners.add(rs[0]["path"])
     for r in rs[1:]:
         losers.setdefault(r["path"],
-                          f"duplicate of {series[key]['name']} S{season:02d}E{e:02d}"
+                          f"duplicate of {canon[key]['name']} S{season:02d}E{e:02d}"
                           " (the bigger copy keeps the standard name)")
 for r, name, season, eps in claimants:
     if r["path"] in losers:
         park(r, losers[r["path"]])
         continue
-    s = series[eng.norm(name)]
+    s = canon[series_key(name)]
-    label = f" S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps)
+    stem = f"{label_of(s)} - S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps)
+    title = episode_title(r, s, season, eps[0])
+    if title:
+        stem += f" - {title}"
     part = PART_RE.search(os.path.splitext(r["base"])[0])
-    tail = f" Part {part.group(1).upper()}" if part and len(eps) == 1 else ""
+    if part and len(eps) == 1:
-    book(r, f"{series_dir(name)}/Season {season:02d}", s["name"] + label + tail + r["ext"],
+        stem += f" Part {part.group(1).upper()}"
-         "tv")
+    book(r, f"{series_dir(name)}/Season {season:02d}", stem + r["ext"], "tv")
 
 for r, name, kind in extras:
     book(r, f"{series_dir(name)}/{kind}",
@@ -680,7 +919,8 @@ def write_plan():
 
 
 def write_exceptions():
-    no_year = sorted(s["name"] for s in series.values() if not s["year"])
+    no_year = sorted({s["name"] for s in canon.values() if not s["year"]})
+    no_tvdb = sorted({identity(s)[0] for s in canon.values() if not s.get("canon")})
     with open(ARGS.exceptions, "w") as fh:
         fh.write(f"TV relayout exceptions for {ROOT}\n\n")
         fh.write(f"Everything below is parked under {PARK}, keeping the relative path it\n")
@@ -698,6 +938,12 @@ def write_exceptions():
             fh.write(f"{len(no_year):>6} series carry no year from any provider, so their\n")
             fh.write("       folders are named without one; Jellyfin still matches them.\n")
             fh.write("       " + ", ".join(no_year[:300]) + "\n")
+        if no_tvdb:
+            fh.write(f"\n{len(no_tvdb):>6} series carry no tvdb identity, so their folders\n")
+            fh.write("       keep the name the metadata provider gave them and no episode\n")
+            fh.write("       title was taken. Nothing was invented: give these a resolvable\n")
+            fh.write("       id (--tvdb) and re-plan; a rename is idempotent.\n")
+            fh.write("       " + ", ".join(no_tvdb[:300]) + "\n")
     print(f"exceptions -> {ARGS.exceptions}")
 
 

serverAppConfigs-docs/Dankserver/Media Relayout.md

@@ -187,12 +187,16 @@ show. A name whose marker is `Part 2` is a film and stays put.
 
 `/mnt/entropy/TVShows` is scene packs rather than a library, and Jellyfin has
 already matched and numbered most of it. `scripts/media-shows.py` re-files that
-library into the same shape, and it never guesses: every series name and every
+library into the shape Jellyfin reads, with **tvdb as the name authority**:
-episode number it writes traces to an authority, and what has none is parked for
+
-a human to read.
+    <Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.ext
+
+It never guesses: every series name, episode number and title it writes traces to
+an authority, and what has none is parked for a human to read.
 
 ```bash
-# 1. The authority: Jellyfin's own matched rows (the DB opens read-only).
+# 1. The authority: Jellyfin's own matched rows — each carries the series' TMDb
+#    and Tvdb ids and the title the row holds (the DB opens read-only).
 python3 scripts/media-census.py jellyfin --root /mnt/entropy/TVShows \
     --out /tmp/tv_titles.tsv
 # 2. What is on disk (stat only), and the runtimes the film arm needs.
@@ -201,10 +205,15 @@ python3 scripts/media-census.py enumerate --root /mnt/entropy/TVShows \
 python3 scripts/media-census.py probe --root /mnt/entropy/TVShows \
     --enum /tmp/tv_enum.tsv --jobs 6 --out /tmp/tv_probed.tsv
 #    The probe is the slow step: ffprobe reads both ends of every file.
-# 3. The plan. --tmdb needs TMDB_API_KEY; without it a series no provider dated
+# 3. The plan. --tmdb needs TMDB_API_KEY, --tvdb needs ~/.config/secrets/tvdb
-#    is simply named without a year, which Jellyfin accepts.
+#    (`api_key=`, never argv); both write into the SAME cache file, so the plan is
+#    reproducible offline afterwards. Run it once to fill the cache, then again:
+#    the second run is the one whose numbers are quoted.
 python3 scripts/media-shows.py --root /mnt/entropy/TVShows --enum /tmp/tv_enum.tsv \
-    --titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv --out /mnt/entropy/media-relayout-shows-plan.tsv \
+    --titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv \
+    --tmdb-cache /mnt/entropy/media-tv-names.json --tmdb --tvdb \
+    --park /mnt/entropy/media-park/TVShows \
+    --out /mnt/entropy/media-relayout-shows-plan.tsv \
     --exceptions /mnt/entropy/media-relayout-shows-exceptions.txt
 # 4. Read the exceptions file FIRST — it is the review, not a log. Same executor
 #    as above, but on this arm's OWN journal: a shared journal cannot be unwound.
@@ -217,8 +226,11 @@ Evidence, strongest first — the order the planner believes things:
 - the **episode marker the file's own name states** (`S01E02-E03`, `1x04`);
 - the **`Season NN` folder it sits in and the number its name leads with inside
   it**: `Season 19/01 Charles Barkley, Nirvana.avi` is S19E01;
-- the **provider row**: identity (the series name and its year), and the
+- the **provider row**: which episode the file IS, and the numbering of a file
-  numbering of a file whose own name says nothing;
+  whose own name says nothing;
+- the **row's tvdb id, else its TMDb id carried through TMDb's external ids, else
+  an exact tvdb name search**: that id names the SERIES FOLDER and supplies the
+  episode title, because a name the server read off a folder is not a show name;
 - a **bonus folder** anywhere above the file, or Jellyfin's `ExtraType` → the
   series' canonical extra folder;
 - a runtime over `SHORT_MAX` makes it film-shaped, and it leaves for `--movies`
@@ -237,11 +249,22 @@ Two refusals carry the design, and a dry run on the real library found both:
   the file's own leading number, win; a row that still insists on season 1989
   parks its file instead of creating `Season 1989`.
 
-The provider's name is copied verbatim, warts included: Jellyfin carries a Series
+**A provider name is evidence only when an id backs it.** Jellyfin names a Series
-named `Black Clover 001` (92 episodes, no year) because it matched a pack off a
+after the folder it scanned, so a library filed by release folders ends up with
-filename. Renaming a show is Jellyfin's work — a folder rename never touches its
+one Series item per box set — `24 (2001) Season 1`, `Modern Family S05` — and
-provider ids — so the mover keeps the name it is handed and strips only the pack
+copying those names verbatim launders folder junk into the contract. The planner
-index a release leads with (`4a. Seed DESTINY` → `Seed DESTINY`).
+resolves each series to its tvdb id and writes tvdb's name and year; the merged
+identity is also what turns a box set split across five release folders into ONE
+folder, and what makes two files claiming one episode MEET in the duplicate rule.
+A series no id resolves keeps the provider's words and is listed under *no tvdb
+identity* in the exceptions — that list, not a rename, is the review.
+
+**The review pile lives outside the library.** Files parked inside
+`/mnt/entropy/TVShows` were indexed as one show named `_unsorted`, and that
+show's rows then looked like evidence. So a row whose series is the hidden
+top-level folder the file itself sits in is dropped, and `--park` outside the root
+makes the next plan re-file the pile instead of skipping it. Inside a hidden
+top-level folder a file's own words may MATCH a registered show, never invent one.
 
 ## The music library's own relayout
 

tests/media_shows.bats

@@ -8,7 +8,14 @@
 # file name. So every test here states which authority a file had — name only,
 # provider only, both, neither — and asserts where the PLAN files it, never how
 # the planner decided. A fixture Jellyfin database stands in for the live one and
-# the TMDb cache is an input file, so no test touches the network.
+# the name caches are input files, so no test touches the network.
+#
+# The target contract is tvdb-grade naming:
+#   <Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.<ext>
+# tvdb is the IDENTITY authority: a provider name a server derived from a folder
+# is not a series name, so a polluted name is undone through the series' tvdb id.
+# The review pile parks OUTSIDE the library — a parked folder inside it indexes
+# as one junk show, and that show's rows are then mistaken for evidence.
 #
 # The two outcomes that cost a re-run of a 50k-file plan are a file filed as an
 # episode that is not one (it steals an episode number from its real owner) and a
@@ -25,11 +32,11 @@ APPLY="$REPO_ROOT/scripts/media-apply.py"
 # at PATH SIZE — a file with deterministic content.
 at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
 
-# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] — one row of the authority
+# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] [TMDB] [TITLE] [TVDB]
-# export scripts/media-census.py jellyfin writes.
+# — one row of the authority export scripts/media-census.py jellyfin writes.
 title_row() {
-    printf '%s\t%s\t%s\t%s\t%s\t%s\t\n' "$1" "$2" "$3" "${4:-}" "${5:-}" "${6:-}" \
+    printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "${4:-}" \
-        >> "$TITLES"
+        "${5:-}" "${6:-}" "${7:-}" "${8:-}" "${9:-}" >> "$TITLES"
 }
 
 # dst SRC — the destination the plan gives SRC, or nothing when it does not move.
@@ -40,6 +47,7 @@ dst() {
 setup() {
     LIB="$BATS_TEST_TMPDIR/TVShows"
     PARK="$LIB/_unsorted"
+    OUTSIDE="$BATS_TEST_TMPDIR/park"
     MOVIES="$BATS_TEST_TMPDIR/Movies"
     ENUM="$BATS_TEST_TMPDIR/enum.tsv"
     PROBED="$BATS_TEST_TMPDIR/probed.tsv"
@@ -77,8 +85,14 @@ setup() {
     # A season folder whose rows are stale: the database numbers every file of this
     # pack season 16 episode 20, four seasons' worth of them.
     at "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" 6000
-    # One episode-shaped file loose at the library root.
     at "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" 4000
+    # A box-set folder whose Series item the server named AFTER THE FOLDER: the
+    # row's own name is junk, and only a provider id can undo it.
+    BOX="$LIB/Show Box (2016) Season 1 S01 (1080p WEB-DL)"
+    at "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" 6000
+    # The park inside the library: the server indexed the whole pile as ONE show
+    # named after the folder, so its rows state a series that is not a show.
+    at "$PARK/Park Show S01E01.mkv" 5000
 
     # The enum census lists every file; the planner reads that instead of walking
     # the tree, so the fixture writes the same three columns.
@@ -92,17 +106,21 @@ with open(out, "w") as fh:
             fh.write(f"{os.stat(p).st_size}\t{os.path.splitext(f)[1]}\t{p}\n")
 PY
 
-    printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\n' > "$TITLES"
+    printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\ttitle\ttvdb\n' > "$TITLES"
     printf '{}\n' > "$CACHE"
     title_row "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" \
-        "The Examples" 2010 1 1
+        "The Examples" 2010 1 1 "" "" "Pilot"
     title_row "$ANIME/[grp] Anime Pack - 05.mkv" "Anime Pack" "" 1 5
-    title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1
+    title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots"
-    title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1
+    title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots"
-    title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2
+    title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2 "" "" "Double Trouble"
-    title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3
+    title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3 "" "" "Loose Ends"
     title_row "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" \
         "SNL Pack" "" 16 20
+    title_row "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" \
+        "Show Box (2016) Season 1" 2016 1 1 "" "2334" "Pilot" "78850"
+    # The row the server wrote for a parked file: the series is the PARK FOLDER.
+    title_row "$PARK/Park Show S01E01.mkv" "_unsorted" "" 1 1
 }
 
 plan() {
@@ -111,6 +129,12 @@ plan() {
         --out "$OUT" --exceptions "$EXC"
 }
 
+plan_park() {                                  # the review pile outside the library
+    run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --titles "$TITLES" \
+        --probed "$PROBED" --tmdb-cache "$CACHE" --movies "$MOVIES" \
+        --park "$OUTSIDE" --out "$OUT" --exceptions "$EXC"
+}
+
 @test "the census exports the provider's episode numbering as a TSV" {
     DB="$BATS_TEST_TMPDIR/jellyfin.db"
     python3 - "$DB" "$SEASON" <<'PY'
@@ -125,7 +149,7 @@ sid = str(uuid.uuid4())
 c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
           (sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples",
            pre, None, None, 2010, None, 0,
-           '{"ProviderIds":{"Tmdb":"1234"}}'))
+           '{"ProviderIds":{"Tmdb":"1234","Tvdb":"78850"}}'))
 c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
           (str(uuid.uuid4()), sid, "MediaBrowser.Controller.Entities.TV.Episode",
            "Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1,
@@ -136,34 +160,87 @@ PY
         --out "$BATS_TEST_TMPDIR/exp.tsv"
     [ "$status" -eq 0 ]
     run head -1 "$BATS_TEST_TMPDIR/exp.tsv"
-    [ "${lines[0]}" = "file	series	year	season	episode	extra	tmdb" ]
+    [ "${lines[0]}" = "file	series	year	season	episode	extra	tmdb	title	tvdb" ]
     tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
-        | grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv	The Examples	2010	1	1		1234$"
+        | grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv	The Examples	2010	1	1		1234	Pilot	78850$"
 }
 
-@test "an episode its own name states files as Show SxxEyy under Season NN" {
+@test "the census reads provider ids from the table newer Jellyfin keeps" {
+    DB="$BATS_TEST_TMPDIR/jellyfin12.db"
+    python3 - "$DB" "$SEASON" <<'PY'
+import os, sqlite3, sys, uuid
+db, season = sys.argv[1], sys.argv[2]
+c = sqlite3.connect(db)
+pre = "/data/TVShows/" + os.path.basename(season)
+c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber,
+                                      ParentIndexNumber, ProductionYear, ExtraType,
+                                      IsVirtualItem, Data)""")
+# Jellyfin 12 moved the ids out of the Data blob and into their own table.
+c.execute("create table BaseItemProviders (ItemId, ProviderId, ProviderValue)")
+sid = str(uuid.uuid4())
+eid = str(uuid.uuid4())
+c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
+          (sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples",
+           pre, None, None, 2010, None, 0, '{}'))
+c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
+          (eid, sid, "MediaBrowser.Controller.Entities.TV.Episode",
+           "Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1,
+           None, None, 0, '{}'))
+c.execute("insert into BaseItemProviders values (?,?,?)", (sid, "Tvdb", "9999"))
+c.commit()
+PY
+    run python3 "$CENSUS" jellyfin --root "$LIB" --db "$DB" \
+        --out "$BATS_TEST_TMPDIR/exp12.tsv"
+    [ "$status" -eq 0 ]
+    tail -1 "$BATS_TEST_TMPDIR/exp12.tsv" \
+        | grep -q "	Pilot	9999$"
+}
+
+@test "an episode its own name states files as the titled standard name" {
     plan
     [ "$status" -eq 0 ]
     [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv")" \
-        = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.mkv" ]
+        = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.mkv" ]
+}
+
+@test "the tvdb identity names the series folder, not the name a folder left behind" {
+    printf '{"tvdb:series:78850": {"name": "Show Box", "year": "2016"}}\n' > "$CACHE"
+    plan
+    [ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" \
+        = "$LIB/Show Box (2016)/Season 01/Show Box (2016) - S01E01 - Pilot.mkv" ]
+}
+
+@test "a series no authority resolves keeps the provider's name and is reported" {
+    plan
+    [ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \
+        = "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ]
+    grep -A9 -i "no tvdb" "$EXC" | grep -q "SNL Pack"
+}
+
+@test "a row whose series is the park folder is not evidence of a show" {
+    plan_park
+    run awk -F '\t' -v p="$PARK/" '$1 == "move" && index($3, p) == 1 {print $3}' "$OUT"
+    [ -z "$output" ]
+    run awk -F '\t' '$1 == "move" && $2 ~ /Park Show S01E01/ {print $3}' "$OUT"
+    [[ "$output" == "$OUTSIDE/"* ]]
 }
 
 @test "a subtitle follows its episode and keeps its language flag" {
     plan
     [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt")" \
-        = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.en.srt" ]
+        = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.en.srt" ]
 }
 
 @test "a subtitle two folders down inside Subs/<release-name>/ still follows its episode" {
     plan
     [ "$(dst "$SEASON/Subs/The.Examples.S01E01.Pilot.1080p.x265/The.Examples.S01E01.Pilot.1080p.x265.jpn.srt")" \
-        = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.jpn.srt" ]
+        = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.jpn.srt" ]
 }
 
 @test "a subtitle that carries the container's word in its own name still follows the episode" {
     plan
     [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv.forced.srt")" \
-        = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.forced.srt" ]
+        = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.forced.srt" ]
 }
 
 @test "a season image keeps its own name; a series nfo keeps the series folder" {
@@ -178,10 +255,10 @@ PY
     [ -z "$(awk -F '\t' '$3 ~ /Season .*teaser/ {print $3}' "$OUT")" ]
 }
 
-@test "a file only the provider numbers files at the episode it is matched to" {
+@test "a file only the provider numbers files at the episode it is matched to, title or no title" {
     plan
     [ "$(dst "$ANIME/[grp] Anime Pack - 05.mkv")" \
-        = "$LIB/Anime Pack/Season 01/Anime Pack S01E05.mkv" ]
+        = "$LIB/Anime Pack/Season 01/Anime Pack - S01E05.mkv" ]
 }
 
 @test "a bonus folder inside a release pack belongs to the pack's show, not to its own name" {
@@ -193,19 +270,19 @@ PY
 @test "a Season folder outvotes the stale row its files carry" {
     plan
     [ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \
-        = "$LIB/SNL Pack/Season 19/SNL Pack S19E01.avi" ]
+        = "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ]
 }
 
 
 @test "one file holding two episodes keeps both numbers in its name" {
     plan
     [ "$(dst "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv")" \
-        = "$LIB/Show Two (2016)/Season 01/Show Two S01E02-E03.mkv" ]
+        = "$LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E02-E03 - Double Trouble.mkv" ]
 }
 
 @test "two files claiming one episode: one keeps the name, the other parks" {
     plan
-    n=$(grep -c "	$LIB/Show Two (2016)/Season 01/Show Two S01E01.mkv	" "$OUT")
+    n=$(grep -c "	$LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E01 - Two Pilots.mkv	" "$OUT")
     [ "$n" -eq 1 ]
     run dst "$LIB/Dupes/Show Two S01E01.mkv"
     [[ "$output" == "$PARK/"* ]]