scripts/media-shows.py

@@ -404,19 +404,79 @@ def tvdb_search(name):
     return None
 
 
+# Tvdb's default episode query answers AT MOST this many rows and states no
+# total: One Piece (1999), Crayon Shin-chan (1992) and Detective Conan (1996)
+# each answer exactly 500 while holding over a thousand. A list that long bounds
+# nothing above itself, so it must never be read as "the series holds 500".
+EPISODE_PAGE_CAP = 500
+
+
 def tvdb_episodes(sid):
-    """`season/episode` -> title for one series, in the one call tvdb answers."""
+    """`season/episode` -> title for one series, in the one call tvdb answers.
+    Whether that call came back a cut-off page is recorded in the cache beside
+    it, so a warm cache answers the same question a cold one did."""
     ck = f"tvdb:episodes:{sid}"
-    if ck in CACHE:
-        return CACHE[ck]
-    data = tvdb(f"/series/{sid}/episodes/default")
-    if not data:
-        return None
-    CACHE[ck] = {f"{e.get('seasonNumber')}/{e.get('number')}":
-                 (e.get("name") or "").strip()
-                 for e in data.get("episodes") or []}
+    if ck not in CACHE:
+        data = tvdb(f"/series/{sid}/episodes/default")
+        if not data:
+            return None
+        rows = data.get("episodes") or []
+        CACHE[ck] = {f"{e.get('seasonNumber')}/{e.get('number')}":
+                     (e.get("name") or "").strip() for e in rows}
+        CACHE[f"tvdb:capped:{sid}"] = len(rows) >= EPISODE_PAGE_CAP
     return CACHE[ck]
 
+
+def tvdb_list_capped(sid):
+    """True when tvdb's episode answer for this series is a page, not a list."""
+    rows = tvdb_episodes(sid)
+    if not rows:
+        return False
+    return CACHE.get(f"tvdb:capped:{sid}", len(rows) >= EPISODE_PAGE_CAP)
+
+
+
+# tvdb's titles read backwards, for the rows that carry a title and no numbering.
+_TITLE_IDX = {}
+
+
+def tvdb_title_index(sid):
+    """tvdb's episode list as `title -> (season, episode)`. A title two episodes
+    share names no episode: both halves are dropped, because a file that parks is
+    honest and a file filed at the wrong number is a lie the server believes."""
+    if sid in _TITLE_IDX:
+        return _TITLE_IDX[sid]
+    index, tie = {}, set()
+    for key, name in (tvdb_episodes(sid) or {}).items():
+        season, _, ep = key.partition("/")
+        t = eng.norm(name)
+        if not t or t in tie:
+            continue
+        if t in index:
+            del index[t]
+            tie.add(t)
+            continue
+        try:
+            n_season, n_ep = int(season), int(ep)
+        except ValueError:
+            continue                          # half-numbers ("12.5") state no slot
+        index[t] = (n_season, n_ep)
+    _TITLE_IDX[sid] = index
+    return index
+
+
+def tvdb_number(t):
+    """The (season, episode) tvdb states for a provider row whose own numbering is
+    not a season at all. Jellyfin's date matcher files specials by AIRDATE, so the
+    row says season 2002 — a YEAR — and only the title the provider recorded says
+    which episode it is. No id, no title, or a title tvdb gives twice: no number,
+    and the file parks with the reason it always carried."""
+    sid = (t.get("tvdb") or "").strip()
+    title = (t.get("title") or "").strip()
+    if not sid or not title:
+        return None
+    return tvdb_title_index(sid).get(eng.norm(title))
+
 # ------------------------------------------------------------------- the inputs
 rows = []
 for line in open(ARGS.enum):
@@ -613,9 +673,16 @@ for r in video:
         else:
             season = fs if fs is not None else num(t["season"])
             ep = leading_index(r["base"]) or num(t["episode"])
-            if not 0 < season <= 200 or ep <= 0:
-                unknown.append((r, f"a provider row of season {season} episode {ep}"))
-                continue
+            if not 0 <= season <= 200 or ep <= 0:
+                # Season 0 IS a season: specials live in Season 00 by Jellyfin's own
+                # contract. A season that is a YEAR is the date matcher's doing, and
+                # the only authority on it is tvdb's episode list for the series.
+                recovered = tvdb_number(t)
+                if not recovered:
+                    unknown.append(
+                        (r, f"a provider row of season {season} episode {ep}"))
+                    continue
+                season, ep = recovered
             eps = [ep]
         claimants.append((r, owner, season, eps))
         note(owner, t["year"] if t else "", t["tmdb"] if t else "",
@@ -705,11 +772,41 @@ def label_of(s):
     return f"{nm} ({yr})"
 
 
+def contradicted_seasons(s):
+    """Seasons whose numbering tvdb's own episode list contradicts — the one place
+        a tvdb title can be a confident lie. An anime pack numbered by its release group
+        books S05E01 in a season tvdb does not hold, or books more episodes there than
+        tvdb lists, and the slot its number points at is a DIFFERENT episode. Books <=
+        tvdb's count stays plausible: a partial pack is the normal shape. A list the
+        authority cut off at its page limit (EPISODE_PAGE_CAP) contradicts nothing —
+        it says nothing about the episodes it never sent."""
+    if "contradicted" in s:
+        return s["contradicted"]
+    bad = set()
+    eps = None if not s["tvdb"] or tvdb_list_capped(s["tvdb"]) \
+            else tvdb_episodes(s["tvdb"])
+    if eps:
+        have = collections.Counter()
+        for k in eps:
+            season, _, _ep = k.partition("/")
+            if season.isdigit():
+                have[int(season)] += 1
+        booked = collections.defaultdict(set)
+        for _r, name, season, claimed in claimants:
+            if canon.get(series_key(name)) is s:
+                booked[season].update(claimed)
+        bad = {n for n, seen in booked.items() if len(seen) > have.get(n, 0)}
+    s["contradicted"] = bad
+    return bad
+
+
 def episode_title(r, s, season, ep):
     """The title this episode carries: tvdb's for the id the series resolved to,
-    else the title the matched row states, else nothing at all — a dangling ` - `
-    is not a title, and an invented one is worse than none."""
-    if s["tvdb"]:
+    else the title the matched row states, else nothing at all — a dangling ` - ` is
+    not a title, and an invented one is worse than none. A season tvdb's own list
+    contradicts gets no tvdb title: its number points somewhere else, and a wrong
+    title is worse than a bare one."""
+    if s["tvdb"] and season not in contradicted_seasons(s):
         t = ((tvdb_episodes(s["tvdb"]) or {}).get(f"{season}/{ep}") or "").strip()
         if t:
             return clean(t)
@@ -718,8 +815,17 @@ def episode_title(r, s, season, ep):
 
 
 def series_dir(name):
+    """One series, one folder — and the folder CARRIES the tvdb id. Both matchers
+    read `[tvdbid-NNNN]` (Jellyfin's Tvdb agent, Plex's scanner), so a show whose
+    title is ambiguous resolves by number instead of by a human guessing which
+    folder is which. The id rides the FOLDER only: Jellyfin gives a file name 180
+    characters and the episode title needs the room. A series nothing resolved
+    keeps the provider's words with no tag at all."""
     s = canon.get(series_key(name))
-    return f"{ROOT}/{label_of(s)}" if s else f"{ROOT}/{clean(series_name(name))}"
+    if not s:
+        return f"{ROOT}/{clean(series_name(name))}"
+    tag = f" [tvdbid-{s['tvdb']}]" if s["tvdb"] else ""
+    return f"{ROOT}/{label_of(s)}{tag}"
 
 
 # ------------------------------------------------------------------- the plan
@@ -973,6 +1079,8 @@ def write_plan():
 def write_exceptions():
     no_year = sorted({s["name"] for s in canon.values() if not s["year"]})
     no_tvdb = sorted({identity(s)[0] for s in canon.values() if not s.get("canon")})
+    clash = sorted({(identity(s)[0], tuple(sorted(contradicted_seasons(s))))
+                    for s in canon.values() if contradicted_seasons(s)})
     with open(ARGS.exceptions, "w") as fh:
         fh.write(f"TV relayout exceptions for {ROOT}\n\n")
         fh.write(f"Everything below is parked under {PARK}, keeping the relative path it\n")
@@ -1002,6 +1110,19 @@ def write_exceptions():
             fh.write("       title was taken. Nothing was invented: give these a resolvable\n")
             fh.write("       id (--tvdb) and re-plan; a rename is idempotent.\n")
             fh.write("       " + ", ".join(no_tvdb[:300]) + "\n")
+        if clash:
+            fh.write(f"\n{len(clash):>6} series carry seasons whose numbering tvdb"
+                     " contradicts:\n")
+            fh.write("       a pack books more episodes in a season than tvdb holds"
+                     " there — an anime\n       pack numbered by its release group is"
+                     " the usual shape. Those files still\n       rename to the"
+                     " standard shape, but they do NOT ride with tvdb's titles:"
+                     "\n       the slot tvdb answers for that number is a different"
+                     " episode. Correct the\n       numbering and re-plan; a rename"
+                     " is idempotent.\n")
+            fh.write("       " + "; ".join(
+                f"{n} S{'/'.join(f'{x:02d}' for x in ss)}"
+                for n, ss in clash[:300]) + "\n")
     print(f"exceptions -> {ARGS.exceptions}")
 
 

serverAppConfigs-docs/Dankserver/Media Relayout.md

@@ -181,7 +181,8 @@ show. A name whose marker is `Part 2` is a film and stays put.
 already matched and numbered most of it. `scripts/media-shows.py` re-files that
 library into the shape Jellyfin reads, with **tvdb as the name authority**:
 
-    <Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.ext
+    <Series> (<year>) [tvdbid-NNNN]/Season NN/
+        <Series> (<year>) - SxxEyy - <Episode Title>.ext
 
 It never guesses: every series name, episode number and title it writes traces to
 an authority, and what has none is parked for a human to read.
@@ -234,7 +235,11 @@ Evidence, strongest first — the order the planner believes things:
   whose own name says nothing;
 - the **row's tvdb id, else its TMDb id carried through TMDb's external ids, else
   an exact tvdb name search**: that id names the SERIES FOLDER and supplies the
-  episode title, because a name the server read off a folder is not a show name;
+  episode title, because a name the server read off a folder is not a show name.
+  The folder CARRIES the id: `[tvdbid-NNNN]` is what Jellyfin's Tvdb agent and
+  Plex's scanner both read, so a show whose title is ambiguous resolves by number
+  instead of by a human guessing which folder is which. The id rides the folder
+  only — a file name gets 180 characters and the episode title needs the room.
 - a **bonus folder** anywhere above the file, or Jellyfin's `ExtraType` → the
   series' canonical extra folder;
 - a runtime over `SHORT_MAX` makes it film-shaped, and it leaves for `--movies` only
@@ -244,7 +249,7 @@ Evidence, strongest first — the order the planner believes things:
   for the word "russia", *Special Edition I* for a file saying *Edition III*. Else it
   parks.
 
-Two refusals carry the design, and a dry run on the real library found both:
+Four refusals carry the design: what the planner will NOT do is the point of it.
 
 - **Folders resolve a series; they never invent one.** A folder's words may
   resolve a file to a series a provider has already named — `(CBB) Tengen Toppa
@@ -254,8 +259,27 @@ Two refusals carry the design, and a dry run on the real library found both:
 - **A scraped row does not outrank the filesystem.** Jellyfin numbers
   multi-season packs badly: every file under `SNL 16-20/Season 16..19` carries
   season 16 episode 20, and one Gundam row says season 2004. The folder, then
-  the file's own leading number, win; a row that still insists on season 1989
-  parks its file instead of creating `Season 1989`.
+  the file's own leading number, win.
+- **A season that is a YEAR is the date matcher, not a season.** Jellyfin files
+  specials by airdate, so a row can arrive saying `season 2002`; that is not a
+  season and it never becomes a folder. The one authority on such a row is tvdb's
+  episode list read backwards: the title the row carries names the (season,
+  episode) tvdb gives it. A title tvdb gives to two episodes names nothing, and
+  the file parks with the reason it always carried. `Season 00` IS a legal
+  season — specials live there — so a row that really says 0 files there instead
+  of parking.
+- **A season tvdb's own list contradicts gets no tvdb title.** An anime pack
+  numbered by its release group books `S05E01` in a season tvdb does not hold, or
+  books more episodes there than tvdb lists: the slot its number points at is a
+  different episode, and dressing it in that episode's title is a confident lie.
+  The file still renames to the standard shape; the title does not ride, and the
+  series is listed under *seasons whose numbering tvdb contradicts* in the
+  exceptions. A season holding FEWER files than tvdb lists stays plausible — a
+  partial pack is the normal shape. Neither reading holds for a list tvdb
+  cut off at its own page limit: the default episode query answers at most
+  500 rows and states no total, and One Piece, Crayon Shin-chan and
+  Detective Conan each answer exactly 500 while holding over a thousand —
+  a page bounds nothing above itself, so it contradicts nothing.
 
 **A provider name is evidence only when an id backs it.** Jellyfin names a Series
 after the folder it scanned, so a library filed by release folders ends up with

tests/media_shows.bats

@@ -11,7 +11,11 @@
 # the name caches are input files, so no test touches the network.
 #
 # The target contract is tvdb-grade naming:
-#   <Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.<ext>
+#   <Series> (<year>) [tvdbid-NNNN]/Season NN/<Series> (<year>) - SxxEyy - <Title>.<ext>
+# The id rides the FOLDER only: it is what both matchers read the series by, and a
+# 180-character ceiling on a file name is not the place to spend it. Season 00
+# holds specials; a provider row whose "season" is a YEAR is renumbered by the
+# episode title tvdb states for the series' id, never by a guess.
 # tvdb is the IDENTITY authority: a provider name a server derived from a folder
 # is not a series name, so a polluted name is undone through the series' tvdb id.
 # The review pile parks OUTSIDE the library — a parked folder inside it indexes
@@ -39,6 +43,15 @@ title_row() {
         "${5:-}" "${6:-}" "${7:-}" "${8:-}" "${9:-}" >> "$TITLES"
 }
 
+# file PATH SIZE — `at` plus the row the enum census would have carried had the
+# file existed when setup walked the tree. A test that needs a shape the fixture
+# does not ship adds it here, or the planner never sees it.
+file() {
+    at "$1" "$2"
+    local b="${1##*/}"
+    printf '%s\t.%s\t%s\n' "$2" "${b##*.}" "$1" >> "$ENUM"
+}
+
 # dst SRC — the destination the plan gives SRC, or nothing when it does not move.
 dst() {
     awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $3}' "$OUT"
@@ -220,11 +233,12 @@ PY
         = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.mkv" ]
 }
 
-@test "the tvdb identity names the series folder, not the name a folder left behind" {
+@test "the tvdb identity names the series folder, id and all, not the folder's echo" {
     printf '{"tvdb:series:78850": {"name": "Show Box", "year": "2016"}}\n' > "$CACHE"
     plan
-    [ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" \
-        = "$LIB/Show Box (2016)/Season 01/Show Box (2016) - S01E01 - Pilot.mkv" ]
+    want="$LIB/Show Box (2016) [tvdbid-78850]/Season 01"
+    want+="/Show Box (2016) - S01E01 - Pilot.mkv"
+    [ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" = "$want" ]
 }
 
 @test "a provider name that already carries the year is not given it twice" {
@@ -232,8 +246,9 @@ PY
     # and `Acapulco (2021) (2021)` is a folder neither matcher nor human reuses.
     printf '{"tvdb:series:78850": {"name": "Show Box (2016)", "year": "2016"}}\n' > "$CACHE"
     plan
-    [ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" \
-        = "$LIB/Show Box (2016)/Season 01/Show Box (2016) - S01E01 - Pilot.mkv" ]
+    want="$LIB/Show Box (2016) [tvdbid-78850]/Season 01"
+    want+="/Show Box (2016) - S01E01 - Pilot.mkv"
+    [ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" = "$want" ]
     run grep -F '(2016) (2016)' "$OUT"
     [ -z "$output" ]
 }
@@ -457,3 +472,97 @@ PY
     plan
     [ -z "$(awk -F '\t' '$1 == "move" {print $2}' "$OUT")" ]
 }
+
+# Specials and the year-shaped season. Jellyfin's contract puts specials in
+# Season 00, and its date matcher hands back rows whose "season" is the AIRDATE
+# YEAR. Both used to park — 1,412 and 1,033 files of one 52k plan — although one
+# is a legal season and the other is renumberable from the title tvdb states.
+@test "a provider's season 0 is a special, so it files under Season 00" {
+    file "$LIB/Show Bumps/Best Of The Examples.mkv" 5000
+    title_row "$LIB/Show Bumps/Best Of The Examples.mkv" "The Examples" 2010 0 2
+    plan
+    [ "$(dst "$LIB/Show Bumps/Best Of The Examples.mkv")" \
+        = "$LIB/The Examples (2010)/Season 00/The Examples (2010) - S00E02.mkv" ]
+}
+
+# These three tests carry a tvdb id of their own. The fixture's box set is 78850 and
+# ONE id means ONE show: reusing it here would merge these files into that series and
+# the duplicate rule would park them before any numbering question is asked.
+@test "a row whose season is a year is renumbered by the title tvdb states" {
+    printf '{"tvdb:series:4242": %s, "tvdb:episodes:4242": %s}\n' \
+        '{"name": "The Examples", "year": "2010"}' \
+        '{"1/7": "Best Of"}' > "$CACHE"
+    file "$LIB/Show Bumps/Best Of The Examples.mkv" 5000
+    title_row "$LIB/Show Bumps/Best Of The Examples.mkv" \
+        "The Examples" 2010 2010 11 "" "" "Best Of" "4242"
+    plan
+    want="$LIB/The Examples (2010) [tvdbid-4242]/Season 01"
+    want+="/The Examples (2010) - S01E07 - Best Of.mkv"
+    [ "$(dst "$LIB/Show Bumps/Best Of The Examples.mkv")" = "$want" ]
+}
+
+@test "a title tvdb gives twice never renumbers an episode" {
+    printf '{"tvdb:series:4242": %s, "tvdb:episodes:4242": %s}\n' \
+        '{"name": "The Examples", "year": "2010"}' \
+        '{"1/7": "Best Of", "2/3": "Best Of"}' > "$CACHE"
+    file "$LIB/Show Bumps/Best Of The Examples.mkv" 5000
+    title_row "$LIB/Show Bumps/Best Of The Examples.mkv" \
+        "The Examples" 2010 2010 11 "" "" "Best Of" "4242"
+    plan_park
+    run awk -F '\t' -v p="$OUTSIDE/" '$1 == "move" && index($3, p) == 1 {print $3}' "$OUT"
+    [[ "$output" == *"Best Of The Examples.mkv"* ]]
+}
+
+# Anime numbering is the one place a tvdb title can be confidently wrong: a pack
+# numbered by its release group still names S05E01, and tvdb answers that slot with
+# a different episode. A season the authority does not hold, or holds fewer of than
+# the pack books, is CONTRADICTED: the rename still happens, the title does not.
+@test "a season tvdb contradicts keeps the rename but never its titles" {
+    printf '{"tvdb:series:4242": %s, "tvdb:episodes:4242": %s}\n' \
+        '{"name": "Anime Gate", "year": "2010"}' \
+        '{"1/1": "Pilot"}' > "$CACHE"
+    file "$LIB/Gate Pack/Season 01/one S01E01.mkv" 5000
+    file "$LIB/Gate Pack/Season 05/two S05E01.mkv" 5000
+    file "$LIB/Gate Pack/Season 05/three S05E02.mkv" 5000
+    title_row "$LIB/Gate Pack/Season 01/one S01E01.mkv" \
+        "Anime Gate" 2010 1 1 "" "" "" "4242"
+    title_row "$LIB/Gate Pack/Season 05/two S05E01.mkv" \
+        "Anime Gate" 2010 5 1 "" "" "" "4242"
+    title_row "$LIB/Gate Pack/Season 05/three S05E02.mkv" \
+        "Anime Gate" 2010 5 2 "" "" "" "4242"
+    plan
+    want="$LIB/Anime Gate (2010) [tvdbid-4242]/Season 01"
+    want+="/Anime Gate (2010) - S01E01 - Pilot.mkv"
+    [ "$(dst "$LIB/Gate Pack/Season 01/one S01E01.mkv")" = "$want" ]
+    want="$LIB/Anime Gate (2010) [tvdbid-4242]/Season 05"
+    want+="/Anime Gate (2010) - S05E01.mkv"
+    [ "$(dst "$LIB/Gate Pack/Season 05/two S05E01.mkv")" = "$want" ]
+    grep -A6 -i "numbering tvdb contradicts" "$EXC" | grep -q "Anime Gate"
+}
+
+# Tvdb's default episode query answers at most 500 rows and states no total, so a
+# list that long is a PAGE, not a list: One Piece (1999), Crayon Shin-chan (1992)
+# and Detective Conan (1996) each answer exactly 500 while holding over a thousand.
+# Read as the whole truth, that page contradicts every long-running show's pack
+# and strips the tvdb title from all of it.
+@test "an episode list the authority paginates never contradicts the pack" {
+    python3 - "$CACHE" <<'PY'
+import json, sys
+json.dump({"tvdb:series:4242": {"name": "Long Run", "year": "1999"},
+           "tvdb:episodes:4242": {f"1/{n}": f"Episode {n}"
+                                  for n in range(1, 501)}},
+          open(sys.argv[1], "w"))
+PY
+    for n in $(seq 1 501); do
+        ep=$(printf '%02d' "$n")
+        file "$LIB/Long Pack/Season 01/long $ep.mkv" 1000
+        title_row "$LIB/Long Pack/Season 01/long $ep.mkv" \
+            "Long Run" 1999 1 "$n" "" "" "" "4242"
+    done
+    plan
+    want="$LIB/Long Run (1999) [tvdbid-4242]/Season 01"
+    want+="/Long Run (1999) - S01E01 - Episode 1.mkv"
+    [ "$(dst "$LIB/Long Pack/Season 01/long 01.mkv")" = "$want" ]
+    run grep -i "numbering tvdb contradicts" "$EXC"
+    [ "$status" -ne 0 ]
+}