diff --git a/scripts/media-census.py b/scripts/media-census.py index fd9c5b3..6a9db9e 100755 --- a/scripts/media-census.py +++ b/scripts/media-census.py @@ -21,11 +21,12 @@ scripts/media-plan.py and scripts/media-shows.py read them instead of touching m the titles Plex already matched — the authority over any filename guess jellyfin file series year season episode extra - tmdb (see JF_COLS) + tmdb title tvdb (see JF_COLS) one row per episode Jellyfin has matched and numbered, its path mapped - from the container's view back to the host; `year` and `tmdb` are the - SERIES' values, `extra` is Jellyfin's own extra type when the row is - bonus material rather than an episode + from the container's view back to the host; `year`, `tmdb` and `tvdb` + are the SERIES' values, `title` is the episode title the row carries, + `extra` is Jellyfin's own extra type when the row is bonus material + rather than an episode music file artist album year disc track title mb performer (see MUSIC_COLS) @@ -83,13 +84,14 @@ left join metadata_items par on par.id = mi.parent_id where p.file like ? and p.deleted_at is null and mi.deleted_at is null """ JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db" -JF_COLS = ["file", "series", "year", "season", "episode", "extra", "tmdb"] +JF_COLS = ["file", "series", "year", "season", "episode", "extra", + "tmdb", "title", "tvdb"] # Jellyfin stores the entity's .NET class as its type; the TV tree is these three. JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode", "MediaBrowser.Controller.Entities.TV.Season", "MediaBrowser.Controller.Entities.TV.Series") -JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, IsVirtualItem - from BaseItems where Type = ? and Path like ?""") +JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, + IsVirtualItem, Name from BaseItems where Type = ? and Path like ?""") # The music tree is these two entity classes. The track row carries the tags # Jellyfin read out of the file; the album row carries what the provider matched, @@ -254,9 +256,10 @@ def jellyfin(root, db, container, out): The numbering is what matters, not the name: a release group's file name says nothing about which episode a file is, and Jellyfin has already answered that - question against a provider. Season/episode come from the episode row; the - series name, its year and its TMDb id come from the Series row the episode - climbs to, because those are the only place Jellyfin keeps them. + question against a provider. Season/episode and the episode's own title come + from the episode row; the series name, its year and its TMDb and Tvdb ids come + from the Series row the episode climbs to, because those are the only place + Jellyfin keeps them. """ try: con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) @@ -277,21 +280,38 @@ def jellyfin(root, db, container, out): " where Type in (?,?,?)", JF_TV): node[i] = (pid, typ, name, year, data) + # Jellyfin 12 keeps provider ids in their own table; older builds kept them in + # the row's Data blob. Both shapes are read, because the tvdb id the series + # naming depends on is wherever this build happened to write it. + prov = {} + if "BaseItemProviders" in {r[0] for r in con.execute( + "select name from sqlite_master where type='table'")}: + for iid, pid, val in con.execute( + "select ItemId, ProviderId, ProviderValue from BaseItemProviders" + " where ProviderId in ('Tmdb', 'Tvdb')"): + prov[(iid, pid)] = val + + def provider_of(item_id, want): + got = prov.get((item_id, want)) + if got: + return str(got) + try: + blob = json.loads(node[item_id][4] or "{}").get("ProviderIds") or {} + except (KeyError, ValueError, TypeError): + return "" + return str(blob.get(want, "") or "") + def series_of(item_id): """Climb episode -> season -> series; empty when the chain is broken.""" for _ in range(4): if item_id not in node: - return "", "", "" - pid, typ, name, year, data = node[item_id] + return "", "", "", "" + pid, typ = node[item_id][0], node[item_id][1] if typ.endswith(".Series"): - try: - tmdb = (json.loads(data or "{}") - .get("ProviderIds") or {}).get("Tmdb", "") - except ValueError: - tmdb = "" - return name or "", year or "", tmdb + return (node[item_id][2] or "", node[item_id][3] or "", + provider_of(item_id, "Tmdb"), provider_of(item_id, "Tvdb")) item_id = pid - return "", "", "" + return "", "", "", "" # The fleet's quadlets bind the share at /data, so the container's view of # this library is /data/ unless told otherwise. @@ -302,17 +322,18 @@ def jellyfin(root, db, container, out): with open(out, "w", newline="") as fh: w = csv.writer(fh, delimiter="\t", lineterminator="\n") w.writerow(JF_COLS) - for iid, path, idx, pidx, extra, virt in con.execute(JF_SQL, - (JF_TV[0], prefix + "%")): + for iid, path, idx, pidx, extra, virt, epname in con.execute( + JF_SQL, (JF_TV[0], prefix + "%")): if virt or not path or not path.startswith(prefix): skipped += 1 # a virtual item is a variant, not a file continue - sname, syear, tmdb = series_of(iid) + sname, syear, tmdb, tvdb = series_of(iid) w.writerow([root + "/" + path[len(prefix):], sname.replace("\t", " "), syear, "" if pidx is None else pidx, "" if idx is None else idx, - (extra or "").replace("\t", " "), tmdb]) + (extra or "").replace("\t", " "), tmdb, + (epname or "").replace("\t", " "), tvdb]) written += 1 print(f"episodes={written} virtual-or-blank={skipped} -> {out}") diff --git a/scripts/media-shows.py b/scripts/media-shows.py index 9d55613..6137661 100755 --- a/scripts/media-shows.py +++ b/scripts/media-shows.py @@ -6,29 +6,40 @@ r"""Plan a Jellyfin-format relayout of a TV library. DRY RUN ONLY: nothing moves [--out TSV] [--exceptions TXT] Inputs are the TSVs scripts/media-census.py exports: `enumerate` lists what -exists, `jellyfin` says which episode each file IS, and `probe` is optional -duration evidence for the movie rule. The TMDb cache is a plain JSON file so a -plan stays reproducible: answers are read from it, and a miss is looked up only -with --tmdb and TMDB_API_KEY in the environment (the key never enters argv, and +exists, `jellyfin` says which episode each file IS (its series' TMDb and Tvdb +ids, and the episode title the row carries), and `probe` is optional duration +evidence for the movie rule. Every name and title answer lives in the cache JSON +so a plan stays reproducible: answers are read from it, and a miss is looked up +only with --tmdb/--tvdb and their credentials (a key never enters argv, and never the cache). Target contract (jellyfin.org/docs/general/server/media/shows): - / ()/Season / SxxEyy.ext + / ()/Season / () - SxxEyy - .ext one file holding two episodes SxxE01-E02 specials Season 00 a documented part of an episode SxxE03 Part 1 sidecars <base>.<lang>.srt bonus material <series>/trailers|featurettes|… — never a season +The series name and the episode title are the ones the IDENTITY authority — tvdb +— states for the id the metadata server matched, not the words a release folder +or a server's folder-derived Series name happens to carry. + PLAN is the TSV scripts/media-apply.py replays: `action src dst kind`, every mkdir row before any move, and `keep` for a file already where the contract puts it. EXCEPTIONS is the review list: everything parked, and why. The authority chain, strongest first, for one video file: - 1. a provider row — Jellyfin matched this exact path and numbered it. + 1. a provider row — Jellyfin matched this exact path and numbered it. The + row's NAME is only evidence once an id backs it: the server names a Series + after the folder it scanned, so `24 (2001) Season 1` is a folder echo, and + the series' tvdb id is what undoes it. 2. the file's own SxxEyy / 1x04 marker, the series taken from a sibling's provider row, else from the nearest ancestor folder that has one, else from - the release folder's own words. + a registered show its own words name, else — outside the review pile — from + the release folder's words. Inside a hidden top-level folder a file may + MATCH a registered show but never INVENT one: invention is what filled the + pile to begin with. 3. nothing. An unnumbered file is PARKED under --park with its relative path kept: an invented `Season 01` steals an episode number from the file that really is S01E01, and a wrong number is worse than no number. @@ -47,8 +58,11 @@ Guards, and why each is here: * Series art and the show nfo follow the SERIES, a season image follows its SEASON, a subtitle or episode nfo follows the episode's new base — an episode moved without its subtitles is a half-move. - * Hidden files, macOS forks, qBittorrent resume parts and the park pile itself - are never moved. Skipping the park is what makes re-planning stable. + * Hidden files, macOS forks, qBittorrent resume parts are never moved. A file + already inside --park is never moved either: that is what makes re-planning + stable. --park belongs OUTSIDE the library — parked files inside it get + indexed as one show named after the park folder, and a row like that is a + folder echo, never evidence (the guard in `row_names_the_park`). The plan crosses three trees (the library, its park, the movie library), so the executor runs with --root at the share that holds them, on its own journal — @@ -56,7 +70,8 @@ one journal is one undo: python3 scripts/media-shows.py --root /mnt/entropy/TVShows \ --enum /tmp/tv-enum.tsv --titles /tmp/jellyfin-tv.tsv \ - --out /mnt/entropy/media-tv-plan.tsv + --tvdb --tmdb --out /mnt/entropy/media-tv-plan.tsv \ + --park /mnt/entropy/media-park/TVShows python3 scripts/media-apply.py apply --plan /mnt/entropy/media-tv-plan.tsv \ --root /mnt/entropy --journal /mnt/entropy/media-tv-relayout.jsonl """ @@ -80,14 +95,21 @@ ARGS.add_argument("--titles", default="/tmp/jellyfin-tv.tsv", help="the provider's numbering (media-census.py jellyfin)") ARGS.add_argument("--probed", default="", help="ffprobe census for the movie rule; absent = no durations") -ARGS.add_argument("--tmdb-cache", default="/tmp/tmdb-cache.json", - help="TMDb answers as an input file, so a plan is reproducible") +ARGS.add_argument("--tmdb-cache", default="/tmp/tv-names.json", + help="name answers (TMDb and Tvdb) as ONE input file, so a plan " + "is reproducible; keys carry the provider they came from") ARGS.add_argument("--tmdb", action="store_true", - help="look up cache misses (needs TMDB_API_KEY)") + help="look up cache misses at TMDb (needs TMDB_API_KEY)") +ARGS.add_argument("--tvdb", action="store_true", + help="look up cache misses at Tvdb (needs the credentials file)") +ARGS.add_argument("--tvdb-credentials", + default=os.path.expanduser("~/.config/secrets/tvdb"), + help="file with an api_key= line (and a pin= line for a " + "user-issued key); mode 600, and never the cache") ARGS.add_argument("--movies", default="/mnt/entropy/Movies", help="movie library a mis-filed film moves into") ARGS.add_argument("--park", default="", - help="review pile (default: <root>/_unsorted)") + help="review pile OUTSIDE the library (default: <root>/_unsorted)") ARGS.add_argument("--out", default="/tmp/tv-plan.tsv") ARGS.add_argument("--exceptions", default="/tmp/tv-exceptions.txt") ARGS = ARGS.parse_args() @@ -105,6 +127,7 @@ try: except (OSError, ValueError): CACHE = {} API_KEY = os.environ.get("TMDB_API_KEY", "") +TMDB = {"tries": 0, "failed": 0} VIDEO_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg", ".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".flv", ".vob", ".rm", @@ -218,6 +241,20 @@ def leading_index(name): return int(m.group(1)) if m else None +def tmdb_request(url): + """One TMDb call, authenticated. The two credential shapes are NOT + interchangeable and the difference is a 401 on every single call: a v3 key + travels as the `api_key` query parameter, and only a v4 request token is + accepted as a Bearer header. A token in argv is world-readable in + /proc/<pid>/cmdline — this URL is built inside the process and never reaches + argv, so the query form is safe here.""" + TMDB["tries"] += 1 + if API_KEY.count(".") == 2: # three segments: a v4 JWT + return http_json(url, headers={"Authorization": "Bearer " + API_KEY}) + return http_json(url + ("&" if "?" in url else "?") + "api_key=" + + urllib.parse.quote(API_KEY)) + + def tmdb(kind, query): """A provider answer for a query: the cache first, TMDb once, then never again.""" ck = f"{kind}:{query}" @@ -231,26 +268,155 @@ def tmdb(kind, query): path = "movie" if kind == "movie" else "tv" url = ("https://api.themoviedb.org/3/search/" + path + "?query=" + urllib.parse.quote(query) + "&include_adult=false") - req = urllib.request.Request(url, headers={ - # The key rides a header, never the URL: a token in argv is world-readable. - "Authorization": "Bearer " + API_KEY, - "Accept": "application/json"}) try: - with urllib.request.urlopen(req, timeout=20) as r: - found = json.load(r).get("results") or [] + found = tmdb_request(url).get("results") or [] except Exception as e: # noqa: BLE001 + TMDB["failed"] += 1 print(f"tmdb {kind} lookup failed for {query!r}: {e}", file=sys.stderr) return None if not found: CACHE[ck] = None return None top = found[0] + # The two search endpoints date their rows differently: movies carry + # `release_date`, tv carries `first_air_date`. Reading only one of them leaves + # every film answer year-less, and a year-less answer can never be refused by + # the year guard below. + when = str(top.get("year") or "") or \ + str(top.get("release_date") or top.get("first_air_date") or "")[:4] CACHE[ck] = {"name": (top.get("title") or top.get("name") or "").strip(), - "year": str(top.get("year") or "") or (top.get("first_air_date") or "")[:4], - "id": top.get("id")} + "year": when, "id": top.get("id")} return CACHE[ck] +# ---------------------------------------------------------------- tvdb, the identity +# tvdb names the show and the episode: it is the only provider whose strings are +# the ones the operator asked for, so its answer replaces a folder-derived Series +# name. The login mints a JWT per run from a mode-600 file; neither the key nor +# the token reaches argv, the cache, or the plan. A provider that will not answer +# never invents a name: every lookup here returns None, and None keeps the +# provider's own words and lands in the exceptions. +TVDB = {"jwt": None, "dead": False} + + +def http_json(url, body=None, headers=None): + req = urllib.request.Request( + url, data=json.dumps(body).encode() if body is not None else None, + headers={"Accept": "application/json", "Content-Type": "application/json", + **(headers or {})}) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + + +def tvdb_login(): + """The run's JWT, or None: a project key needs only api_key=, a user-issued + key also needs the pin.""" + if TVDB["jwt"] is not None: + return TVDB["jwt"] or None + TVDB["jwt"] = "" + try: + lines = open(ARGS.tvdb_credentials).read().splitlines() + except OSError: + print(f"tvdb: no credentials at {ARGS.tvdb_credentials}", file=sys.stderr) + return None + sec = {} + for line in lines: + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + sec[k.strip()] = v.strip() + if not sec.get("api_key"): + print(f"tvdb: no api_key in {ARGS.tvdb_credentials}", file=sys.stderr) + return None + body = {"apiKey": sec["api_key"]} + if sec.get("pin"): + body["token"] = sec["pin"] + try: + TVDB["jwt"] = (http_json("https://api4.thetvdb.com/v4/login", body) + .get("data") or {}).get("token") or "" + except Exception as e: # noqa: BLE001 + print(f"tvdb login failed: {e}", file=sys.stderr) + return None + return TVDB["jwt"] or None + + +def tvdb(path): + """One Tvdb v4 GET, or None when it cannot be answered.""" + if not ARGS.tvdb or TVDB["dead"]: + return None + jwt = tvdb_login() + if not jwt: + TVDB["dead"] = True + return None + try: + return http_json("https://api4.thetvdb.com/v4" + path, + headers={"Authorization": "Bearer " + jwt}).get("data") + except Exception as e: # noqa: BLE001 + print(f"tvdb {path} failed: {e}", file=sys.stderr) + return None + + +def tvdb_series(sid): + """{name, year} for a tvdb series id.""" + ck = f"tvdb:series:{sid}" + if ck in CACHE: + return CACHE[ck] + data = tvdb(f"/series/{sid}") + if not data: + return None + CACHE[ck] = {"name": (data.get("name") or "").strip(), + "year": (data.get("firstAired") or "")[:4]} + return CACHE[ck] + + +def tvdb_id_for_tmdb(mid): + """The tvdb id TMDb records for a series id — the bridge for the bulk of this + library, which the server matched by TMDb.""" + ck = f"tmdb:external:{mid}" + if ck in CACHE: + return CACHE[ck] or "" + if not (ARGS.tmdb and API_KEY): + return "" + try: + got = tmdb_request("https://api.themoviedb.org/3/tv/" + mid + "/external_ids") + except Exception as e: # noqa: BLE001 + TMDB["failed"] += 1 + print(f"tmdb external ids failed for {mid}: {e}", file=sys.stderr) + return "" + CACHE[ck] = str(got.get("tvdb_id") or "") + return CACHE[ck] + + +def tvdb_search(name): + """The tvdb series a provider name states, or None: only tvdb's own spelling + counts, so a near-miss never renames a show.""" + ck = f"tvdb:search:{name}" + if ck in CACHE: + return CACHE[ck] + for hit in tvdb("/search?query=" + urllib.parse.quote(name) + "&type=series") or []: + if eng.norm(hit.get("name") or "") != eng.norm(name): + continue + CACHE[ck] = {"name": (hit.get("name") or "").strip(), + "year": str(hit.get("year") or ""), + "id": str(hit.get("tvdb_id") or "")} + return CACHE[ck] + CACHE[ck] = None + return None + + +def tvdb_episodes(sid): + """`season/episode` -> title for one series, in the one call tvdb answers.""" + ck = f"tvdb:episodes:{sid}" + if ck in CACHE: + return CACHE[ck] + data = tvdb(f"/series/{sid}/episodes/default") + if not data: + return None + CACHE[ck] = {f"{e.get('seasonNumber')}/{e.get('number')}": + (e.get("name") or "").strip() + for e in data.get("episodes") or []} + return CACHE[ck] + # ------------------------------------------------------------------- the inputs rows = [] for line in open(ARGS.enum): @@ -264,6 +430,26 @@ for line in open(ARGS.enum): rows.append({"bytes": int(size), "ext": ext.lower(), "path": p, "base": base}) held = {r["path"]: r for r in rows} +def row_names_the_park(path, name): + """The library's own review pile is not a show. When the park sat INSIDE the + library the server indexed the whole folder as one series named after it, and + a row like that would rename a real show to the pile — so a row whose series is + the hidden top-level folder the file itself sits in is dropped. The server also + sanitizes the display name, dropping the leading underscore, so `unsorted` and + `_unsorted` are the same pile: compare them without their leading punctuation, + or the pile survives as a visible series folder.""" + top = rel(path).split("/")[0] + return top.startswith("_") and eng.norm(name.lstrip("_")) == eng.norm(top.lstrip("_")) + + +def in_review_pile(path): + """A file under the library's hidden top-level folder is review material by + that folder's own convention: its filename may MATCH a show a provider names + somewhere else, but it may never INVENT one — inventing one from a file's own + words is what filled the pile to begin with.""" + return rel(path).split("/")[0].startswith("_") + + titles = {} with open(ARGS.titles) as fh: cols = fh.readline().rstrip("\n").split("\t") @@ -272,7 +458,7 @@ with open(ARGS.titles) as fh: f += [""] * (len(cols) - len(f)) row = dict(zip(cols, f)) p = os.path.abspath(row.get("file", "")) - if p in held: + if p in held and not row_names_the_park(p, row.get("series", "")): titles[p] = row dur = {} @@ -379,14 +565,17 @@ def series_name(name): return clean(re.sub(r"\s{2,}", " ", PACK_IDX_RE.sub("", name)).strip(" .")) -def note(name, year="", tmdb_id=""): - """Register a series; the first name wins, the first year and id stick.""" +def note(name, year="", tmdb_id="", tvdb_id=""): + """Register a series; the first name wins, the first year and ids stick.""" k = eng.norm(name) - s = series.setdefault(k, {"name": series_name(name), "year": "", "tmdb": ""}) + s = series.setdefault(k, {"name": series_name(name), "year": "", "tmdb": "", + "tvdb": ""}) if year and not s["year"]: s["year"] = str(year) if tmdb_id and not s["tmdb"]: s["tmdb"] = str(tmdb_id) + if tvdb_id and not s["tvdb"]: + s["tvdb"] = str(tvdb_id) @@ -413,7 +602,9 @@ for r in video: # every file under SNL 16-20/Season 16..19 carries season 16 episode 20. owner = (t["series"] if t and t["series"] else series_of_dir(os.path.dirname(p)) - or title_of(r["base"][:marked[2]])[0]) + or known_in(r["base"]) + or ("" if in_review_pile(p) + else title_of(r["base"][:marked[2]])[0])) if not eng.norm(owner).replace(" ", ""): unknown.append((r, "an episode marker with no series name to hang it on")) continue @@ -427,7 +618,8 @@ for r in video: continue eps = [ep] claimants.append((r, owner, season, eps)) - note(owner, t["year"] if t else "", t["tmdb"] if t else "") + note(owner, t["year"] if t else "", t["tmdb"] if t else "", + t["tvdb"] if t else "") continue owner = series_of_dir(os.path.dirname(p)) if fs is not None and owner and leading_index(r["base"]): @@ -450,17 +642,84 @@ for k, s in series.items(): if got and eng.norm(got["name"]) == k: s["year"] = got["year"] -if ARGS.tmdb: +# tvdb is the identity authority. The row's own tvdb id is the door; a TMDb id +# crosses through TMDb's external ids; a bare name counts only when tvdb spells +# it the same and the years do not disagree. A series nothing resolves keeps the +# provider's words AND says so in the exceptions, which is where the operator +# finds the folders the plan did not rename to authority. +for s in series.values(): + got = tvdb_series(s["tvdb"]) if s["tvdb"] else None + if not got and s["tmdb"]: + sid = tvdb_id_for_tmdb(s["tmdb"]) + if sid: + s["tvdb"] = sid + got = tvdb_series(sid) + if not got: + hit = tvdb_search(s["name"]) + if hit and hit["id"] and (not s["year"] or not hit["year"] + or hit["year"] == s["year"]): + s["tvdb"] = hit["id"] + got = hit + if got and got["name"]: + s["canon"], s["canon_year"] = got["name"], got["year"] + +# One identity, one folder: the provider's spellings collapse to the shows tvdb +# says they are, so a box set split over five release folders is ONE series and +# two files claiming one episode MEET inside the duplicate rule below. +alias, canon = {}, {} +for k, s in series.items(): + key = f"id:{s['tvdb']}" if s["tvdb"] else k + alias[k] = key + keep = canon.setdefault(key, s) + if keep is not s: + for f in ("year", "tvdb", "canon", "canon_year"): + if not keep.get(f) and s.get(f): + keep[f] = s[f] + +if ARGS.tmdb or ARGS.tvdb: with open(ARGS.tmdb_cache, "w") as fh: json.dump(CACHE, fh, indent=1, sort_keys=True) +def identity(s): + """(name, year) a folder and its files carry: tvdb's whenever it answered, + the provider's own words only for a series nothing resolved.""" + if s.get("canon"): + return s["canon"], s["canon_year"] or s["year"] + return s["name"], s["year"] + + +def series_key(name): + """The merged identity a provider spelling belongs to.""" + return alias.get(eng.norm(name), eng.norm(name)) + + +def label_of(s): + """`<Series> (<year>)`, the form both matchers read. A provider's name can + arrive WITH the year already on it — TVDb hands back `Acapulco (2021)` for the + reboot — and appending a second one names a folder nothing will match again.""" + nm, yr = identity(s) + nm = clean(nm) + if not yr or re.search(r"\(\d{4}\)\s*$", nm): + return nm + return f"{nm} ({yr})" + + +def episode_title(r, s, season, ep): + """The title this episode carries: tvdb's for the id the series resolved to, + else the title the matched row states, else nothing at all — a dangling ` - ` + is not a title, and an invented one is worse than none.""" + if s["tvdb"]: + t = ((tvdb_episodes(s["tvdb"]) or {}).get(f"{season}/{ep}") or "").strip() + if t: + return clean(t) + t = ((titles.get(r["path"]) or {}).get("title") or "").strip() + return clean(t) if t else "" + + def series_dir(name): - s = series.get(eng.norm(name)) - if not s: - s = {"name": clean(name), "year": ""} - label = f"{s['name']} ({s['year']})" if s["year"] else s["name"] - return f"{ROOT}/{clean(label)}" + s = canon.get(series_key(name)) + return f"{ROOT}/{label_of(s)}" if s else f"{ROOT}/{clean(series_name(name))}" # ------------------------------------------------------------------- the plan @@ -500,25 +759,28 @@ def park(r, reason): by_ep = collections.defaultdict(list) for r, name, season, eps in claimants: for e in eps: - by_ep[(eng.norm(name), season, e)].append(r) + by_ep[(series_key(name), season, e)].append(r) winners, losers = set(), {} for (key, season, e), rs in by_ep.items(): rs.sort(key=lambda r: (-r["bytes"], r["path"])) winners.add(rs[0]["path"]) for r in rs[1:]: losers.setdefault(r["path"], - f"duplicate of {series[key]['name']} S{season:02d}E{e:02d}" + f"duplicate of {canon[key]['name']} S{season:02d}E{e:02d}" " (the bigger copy keeps the standard name)") for r, name, season, eps in claimants: if r["path"] in losers: park(r, losers[r["path"]]) continue - s = series[eng.norm(name)] - label = f" S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps) + s = canon[series_key(name)] + stem = f"{label_of(s)} - S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps) + title = episode_title(r, s, season, eps[0]) + if title: + stem += f" - {title}" part = PART_RE.search(os.path.splitext(r["base"])[0]) - tail = f" Part {part.group(1).upper()}" if part and len(eps) == 1 else "" - book(r, f"{series_dir(name)}/Season {season:02d}", s["name"] + label + tail + r["ext"], - "tv") + if part and len(eps) == 1: + stem += f" Part {part.group(1).upper()}" + book(r, f"{series_dir(name)}/Season {season:02d}", stem + r["ext"], "tv") for r, name, kind in extras: book(r, f"{series_dir(name)}/{kind}", @@ -530,8 +792,13 @@ for r, name, kind in extras: for r in films: parsed, year = title_of(r["base"]) got = tmdb("movie", parsed) - words = eng.norm(got["name"]).split() if got else [] - if not got or not eng.mentions(parsed.split(), words) \ + # The provider's title must be a run of words the FILE itself states. The query + # is only what title_of could salvage from the name, and a short one ("russia") + # lets the top hit be a different work entirely: a 1963 Bond film is not a 1991 + # documentary episode, and Special Edition I is not Special Edition III. + said = eng.norm(os.path.splitext(r["base"])[0]).split() + named = eng.norm(got["name"]).split() if got else [] + if not got or not eng.mentions(named, said) \ or (year != "?" and got["year"] and got["year"] != year): park(r, "long file with no episode evidence, and no movie provider names it") continue @@ -655,6 +922,30 @@ for r, reason in unknown: # --------------------------------------------------------------------- output +def provider_broken(): + """TMDb was asked for names and did not answer: the plan is not the plan the + operator asked for, whatever the move count says.""" + return bool(ARGS.tmdb and (not API_KEY or TMDB["failed"])) + + +def provider_note(): + """Provider health on the summary line. The 401s are already on stderr, and a + terminal that scrolled past them shows a plan that looks finished: the count + belongs where it cannot be missed, because a half-resolved plan renames + folders with the folder's own junk words.""" + if not (ARGS.tmdb or ARGS.tvdb): + return "" + if not ARGS.tmdb: + return ", tmdb off (names from the cache and tvdb only)" + if not API_KEY: + return ", tmdb NOT ASKED (TMDB_API_KEY unset): NOT provider-complete" + if not TMDB["tries"]: + return ", tmdb cache-only (every name was already answered)" + done = TMDB["tries"] - TMDB["failed"] + return (f", tmdb {done}/{TMDB['tries']} answered" + + (" — NOT provider-complete" if TMDB["failed"] else "")) + + def write_plan(): """mkdir rows for every destination folder and its whole chain, then the moves. Parents first: the executor creates directories in the order they are listed.""" @@ -676,16 +967,23 @@ def write_plan(): sized = sum(held[s]["bytes"] for s, _d, _k in moves if s in held) print(f"plan: {len(moves) - len(parked)} moves, {len(parked)} parked" f" ({sized / 2**40:.2f} TiB touched), {len(keeps)} already placed," - f" {len(dirs)} folders -> {ARGS.out}") + f" {len(dirs)} folders{provider_note()} -> {ARGS.out}") def write_exceptions(): - no_year = sorted(s["name"] for s in series.values() if not s["year"]) + no_year = sorted({s["name"] for s in canon.values() if not s["year"]}) + no_tvdb = sorted({identity(s)[0] for s in canon.values() if not s.get("canon")}) with open(ARGS.exceptions, "w") as fh: fh.write(f"TV relayout exceptions for {ROOT}\n\n") fh.write(f"Everything below is parked under {PARK}, keeping the relative path it\n") fh.write("has today. Nothing is deleted and nothing is overwritten: a parked file\n") fh.write("is one rename from wherever the plan is pointed next.\n\n") + if provider_broken(): + fh.write("WARNING: " + provider_note().strip(", ").replace(": NOT", " — NOT") + + "\n Names and episode titles that should have come from TMDb are\n" + " missing, so folders keep the metadata provider's words. Fix the\n" + " credential and re-plan: the cache keeps every answer that did\n" + " arrive, so the re-run costs only the lookups that failed.\n\n") for reason in sorted(reasons, key=lambda r: -len(reasons[r])): rs = reasons[reason] fh.write(f"{len(rs):>6} files {reason}\n") @@ -698,6 +996,12 @@ def write_exceptions(): fh.write(f"{len(no_year):>6} series carry no year from any provider, so their\n") fh.write(" folders are named without one; Jellyfin still matches them.\n") fh.write(" " + ", ".join(no_year[:300]) + "\n") + if no_tvdb: + fh.write(f"\n{len(no_tvdb):>6} series carry no tvdb identity, so their folders\n") + fh.write(" keep the name the metadata provider gave them and no episode\n") + fh.write(" title was taken. Nothing was invented: give these a resolvable\n") + fh.write(" id (--tvdb) and re-plan; a rename is idempotent.\n") + fh.write(" " + ", ".join(no_tvdb[:300]) + "\n") print(f"exceptions -> {ARGS.exceptions}") diff --git a/serverAppConfigs-docs/Agent/Issue Tracker.md b/serverAppConfigs-docs/Agent/Issue Tracker.md index cd53cce..093847e 100644 --- a/serverAppConfigs-docs/Agent/Issue Tracker.md +++ b/serverAppConfigs-docs/Agent/Issue Tracker.md @@ -48,6 +48,14 @@ Use the `Authorization: token` header for everything. - **Read an issue**: `curl -s -K "$cfg" $API/issues/<number>` (fields: `number`, `title`, `body`, `labels[].name`, `state`) - **List issues**: `curl -s -K "$cfg" "$API/issues?state=open&limit=50"` — add `&labels=<id>` to filter - **Comment**: `curl -s -K "$cfg" -X POST -d "$(jq -n --rawfile b "$BODY" '{body:$b}')" $API/issues/<number>/comments` +- **Comment on a PR**: the conversation thread is `POST $API/issues/<PR-number>/comments` + (201, and the answer's `html_url` points at `pulls/<n>`; Gitea's issue index is + unified, so the PR's number is the same number). `$API/pulls/<n>/comments` is the + **review**-comment store and answers `405` to a plain body. +- **A body built by `jq --rawfile b /dev/stdin` inside `$(…)` is empty**: the command + substitution does not inherit the heredoc, so Gitea receives `{}` and answers 422 — + and `jq -r '.id'` on the error prints nothing, which reads as success. Write the + body to a file and pass `--rawfile` that file. - **Apply labels**: `POST $API/issues/<number>/labels` with `{"labels":[<label-ID numbers>]}` — Gitea wants numeric label **IDs**, not names; resolve them from `GET $API/labels` (see [[Triage Labels]] for which labels exist) - **Remove a label**: `DELETE $API/issues/<number>/labels/<label-id>` - **Close**: `PATCH $API/issues/<number>` with `{"state":"closed"}` diff --git a/serverAppConfigs-docs/Dankserver/Jellyfin.md b/serverAppConfigs-docs/Dankserver/Jellyfin.md index b2b6af3..d511ffd 100644 --- a/serverAppConfigs-docs/Dankserver/Jellyfin.md +++ b/serverAppConfigs-docs/Dankserver/Jellyfin.md @@ -182,6 +182,48 @@ scripts/jellyfin-providers.sh verify # plugin + key LENGTHS, never key bytes 2. Set metadata/language preferences (`LANG=en_US.UTF-8` pinned in the unit). 3. Configure clients to use `http://192.168.0.235:8096` or the guarded vhost. +## Triggering a scan + +The scan key lives in `~/.config/secrets/jellyfin` (mode 600; the same value is +`JELLYFIN_API_KEY` in the ignored `dankserver/.env`). It rides a mode-600 curl +config, never a header argument, in the `MediaBrowser` form: 12.4 authenticates +only `Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token` +for every key, valid or not. Before believing a 401, prove the key against the +server's own table — `select count(*) from ApiKeys where AccessToken = ?` on +`jellyfin.db` opened `mode=ro&immutable=1`. The backslashes below matter: + +```bash +umask 077 +{ printf 'header = "Authorization: MediaBrowser Token=\\"%s\\""\n' \ + "$(cat ~/.config/secrets/jellyfin)" + echo 'url = "http://127.0.0.1:8096/Library/Refresh"' + echo 'request = "POST"'; } >/tmp/jf.hdr +curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n" +shred -u /tmp/jf.hdr +``` + +`204` is acceptance; anything else queued no scan, so re-issue the identical +request instead of diagnosing. A `401` from a key that answers `200` on a +hand-typed request is this block's quoting, not the server: bash `printf` +collapses `\"` to `"`, curl ends the quoted value at that quote, and Jellyfin +receives `Authorization: MediaBrowser Token=` with nothing after it (`\\"` answers +204, `\"` answers 401 — both run on this host). + +One accepted refresh is the whole procedure. A second one while a scan runs moves +`Scan Media Library` to `Cancelling`, and once the runner holds `Cancelling` every +later `204` is accepted and then discarded — the status code stops meaning +anything. The fix is `systemctl --user restart jellyfin.service` (never +`deploy.sh`: that runs the never-tested metadata-view cutover), one POST after +health answers, and then believe `ScheduledTasks` State at 130 % CPU, not the +response code. + +**A scan that must see a new folder has to be recursive.** `POST /Library/Refresh` +is the incremental pass, and inotify does not cross a rootless podman bind, so +folders created on the host stay invisible to it. The move is +`POST /Items/{id}/Refresh?Recursive=true` (204) per library, the id from +`GET /Library/VirtualFolders` — its field is `ItemId`, not `Id` — and that same +record's `RefreshStatus` is the progress read. + ## See Also - [[Plex]] — co-resident media server on the same volume diff --git a/serverAppConfigs-docs/Dankserver/Media Relayout.md b/serverAppConfigs-docs/Dankserver/Media Relayout.md index 64d809a..d3ecab2 100644 --- a/serverAppConfigs-docs/Dankserver/Media Relayout.md +++ b/serverAppConfigs-docs/Dankserver/Media Relayout.md @@ -81,41 +81,12 @@ An apply also leaves the release folders it emptied standing where they were — hundreds of them at the library root — which is what `sweep` removes, dry-run first, from the same journal. -After it lands, both servers need a scan or the change is invisible. - -```bash -# Run this from the host shell through `podman exec`, never from a neighbour -# container: Plex's HTTPD closes without a single byte on any request that -# arrives from another container's netns (10.89.x.x reads as "empty reply", not -# as a 401). The token rides stdin, never argv, where any process on the host -# could read it out of /proc/<pid>/cmdline. -umask 077 -podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \ - '/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok -podman exec -i plex sh -c 'read -r T - for k in $(curl -s -H "X-Plex-Token: $T" -H "Accept: application/json" \ - http://127.0.0.1:32400/library/sections | grep -o "\"key\":\"[0-9]*\"" | cut -d\" -f4); do - curl -s -o /dev/null -w "plex section $k: %{http_code}\n" -X POST \ - -H "X-Plex-Token: $T" "http://127.0.0.1:32400/library/sections/$k/refresh" - done' </tmp/px.tok -``` - -[[Jellyfin]]'s scan key lives in `~/.config/secrets/jellyfin` (mode 600; the -same value is `JELLYFIN_API_KEY` in the ignored `dankserver/.env`). It rides a -mode-600 curl config, never a header argument, in the `MediaBrowser` form — -Jellyfin 12.4 refuses a bare `X-Emby-Token`, and the backslashes below matter: - -```bash -umask 077 -{ printf 'header = "Authorization: MediaBrowser Token=\\"%s\\""\n' \ - "$(cat ~/.config/secrets/jellyfin)" - echo 'url = "http://127.0.0.1:8096/Library/Refresh"' - echo 'request = "POST"'; } >/tmp/jf.hdr -curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n" -shred -u /tmp/jf.hdr -``` - -`204` is acceptance; anything else queued no scan. A `401` from a key that answers `200` on a hand-typed request is this block's quoting, not the server — bash `printf` collapses `\"` to `"`, curl ends the quoted value at that quote, and Jellyfin receives `Authorization: MediaBrowser Token=` with nothing after it (`\\"` answers 204, `\"` answers 401, both run on this host). One accepted refresh is the whole procedure: a second one while a scan runs moves `Scan Media Library` to `Cancelling`, and once the runner holds `Cancelling` every later `204` is accepted and then dropped — the fix is `systemctl --user restart jellyfin.service` (never `deploy.sh`: that runs the metadata-view cutover), one POST after health answers, and then believe `ScheduledTasks` State `Running`, not the status code. +After it lands, both servers need a scan or the change is invisible. [[Plex]] and +[[Jellyfin]] own the exact requests; two laws hold at this step: `204` is +acceptance and anything else queued nothing — re-issue the identical request +rather than diagnosing — and ONE accepted refresh is the whole procedure, because +a second while a scan runs moves `Scan Media Library` to `Cancelling` and every +later `204` is accepted and then discarded. ### Proving the move landed @@ -208,12 +179,16 @@ show. A name whose marker is `Part 2` is a film and stays put. `/mnt/entropy/TVShows` is scene packs rather than a library, and Jellyfin has already matched and numbered most of it. `scripts/media-shows.py` re-files that -library into the same shape, and it never guesses: every series name and every -episode number it writes traces to an authority, and what has none is parked for -a human to read. +library into the shape Jellyfin reads, with **tvdb as the name authority**: + + <Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.ext + +It never guesses: every series name, episode number and title it writes traces to +an authority, and what has none is parked for a human to read. ```bash -# 1. The authority: Jellyfin's own matched rows (the DB opens read-only). +# 1. The authority: Jellyfin's own matched rows — each carries the series' TMDb +# and Tvdb ids and the title the row holds (the DB opens read-only). python3 scripts/media-census.py jellyfin --root /mnt/entropy/TVShows \ --out /tmp/tv_titles.tsv # 2. What is on disk (stat only), and the runtimes the film arm needs. @@ -221,16 +196,33 @@ python3 scripts/media-census.py enumerate --root /mnt/entropy/TVShows \ --out /tmp/tv_enum.tsv python3 scripts/media-census.py probe --root /mnt/entropy/TVShows \ --enum /tmp/tv_enum.tsv --jobs 6 --out /tmp/tv_probed.tsv -# The probe is the slow step: ffprobe reads both ends of every file. -# 3. The plan. --tmdb needs TMDB_API_KEY; without it a series no provider dated -# is simply named without a year, which Jellyfin accepts. +# The probe is the slow step: ffprobe reads both ends of every file. So re-use +# the last one — a duration belongs to the file, not to its name: re-key the +# previous census through that pass's own journal (`src` -> `dst`) instead of +# seeking the array again. It is also the only column the film arm reads, and +# only for a file with no marker and no provider row (4,794 of 58,551 today, +# all of them covered by a re-key that costs seconds, not an hour). +# 3. The plan. --tmdb needs TMDB_API_KEY — the v3 key in ~/.config/secrets/tmdb, +# which travels as the `api_key` query parameter. Only a v4 request token is +# accepted as a Bearer header, and the wrong shape answers 401 on EVERY call, +# which the summary line flags as NOT provider-complete. --tvdb reads its +# `api_key=` line from ~/.config/secrets/tvdb, never argv. Both write the SAME +# cache file, so the plan is reproducible offline afterwards: run it once to +# fill the cache, and quote the second run. +# Paths are only strings to a dry run, so this step can run on the workstation +# against scp'd TSVs — which is where the two provider secrets live. python3 scripts/media-shows.py --root /mnt/entropy/TVShows --enum /tmp/tv_enum.tsv \ - --titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv --out /mnt/entropy/media-relayout-shows-plan.tsv \ - --exceptions /mnt/entropy/media-relayout-shows-exceptions.txt + --titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv \ + --tmdb-cache /mnt/entropy/media-tv-names.json --tmdb --tvdb \ + --park /mnt/entropy/media-park/TVShows \ + --out /mnt/entropy/media-relayout-shows2-plan.tsv \ + --exceptions /mnt/entropy/media-relayout-shows2-exceptions.txt # 4. Read the exceptions file FIRST — it is the review, not a log. Same executor -# as above, but on this arm's OWN journal: a shared journal cannot be unwound. -scripts/media-apply.py apply --plan /mnt/entropy/media-relayout-shows-plan.tsv \ - --root /mnt/entropy --journal /mnt/entropy/media-relayout-shows.jsonl --apply +# as above, but on this arm's OWN journal, NAMED PER PASS: the journal of an +# applied pass is its unwind record, so a second pass that reuses the name +# destroys the only map back. +scripts/media-apply.py apply --plan /mnt/entropy/media-relayout-shows2-plan.tsv \ + --root /mnt/entropy --journal /mnt/entropy/media-relayout-shows2.jsonl --apply ``` Evidence, strongest first — the order the planner believes things: @@ -238,12 +230,19 @@ Evidence, strongest first — the order the planner believes things: - the **episode marker the file's own name states** (`S01E02-E03`, `1x04`); - the **`Season NN` folder it sits in and the number its name leads with inside it**: `Season 19/01 Charles Barkley, Nirvana.avi` is S19E01; -- the **provider row**: identity (the series name and its year), and the - numbering of a file whose own name says nothing; +- the **provider row**: which episode the file IS, and the numbering of a file + whose own name says nothing; +- the **row's tvdb id, else its TMDb id carried through TMDb's external ids, else + an exact tvdb name search**: that id names the SERIES FOLDER and supplies the + episode title, because a name the server read off a folder is not a show name; - a **bonus folder** anywhere above the file, or Jellyfin's `ExtraType` → the series' canonical extra folder; -- a runtime over `SHORT_MAX` makes it film-shaped, and it leaves for `--movies` - only when a movie provider names it — otherwise it parks. +- a runtime over `SHORT_MAX` makes it film-shaped, and it leaves for `--movies` only + when the movie provider's title is a run of words the FILE itself states and its + year agrees with the file's: the query is only what `title_of` salvaged from a + release name, and the first hit for a short one is a near miss — a 1963 Bond film + for the word "russia", *Special Edition I* for a file saying *Edition III*. Else it + parks. Two refusals carry the design, and a dry run on the real library found both: @@ -258,11 +257,25 @@ Two refusals carry the design, and a dry run on the real library found both: the file's own leading number, win; a row that still insists on season 1989 parks its file instead of creating `Season 1989`. -The provider's name is copied verbatim, warts included: Jellyfin carries a Series -named `Black Clover 001` (92 episodes, no year) because it matched a pack off a -filename. Renaming a show is Jellyfin's work — a folder rename never touches its -provider ids — so the mover keeps the name it is handed and strips only the pack -index a release leads with (`4a. Seed DESTINY` → `Seed DESTINY`). +**A provider name is evidence only when an id backs it.** Jellyfin names a Series +after the folder it scanned, so a library filed by release folders ends up with +one Series item per box set — `24 (2001) Season 1`, `Modern Family S05` — and +copying those names verbatim launders folder junk into the contract. The planner +resolves each series to its tvdb id and writes tvdb's name and year; the merged +identity is also what turns a box set split across five release folders into ONE +folder, and what makes two files claiming one episode MEET in the duplicate rule. +A series no id resolves keeps the provider's words and is listed under *no tvdb +identity* in the exceptions — that list, not a rename, is the review. + +**The review pile lives outside the library.** Files parked inside +`/mnt/entropy/TVShows` were indexed as one show named `_unsorted`, and that +show's rows then looked like evidence. So a row whose series is the hidden +top-level folder the file itself sits in is dropped — and the folder is matched +with its leading underscore stripped, because the server *displays* `_unsorted` as +`unsorted`: compare the spelled names and the pile survives the relayout as a +visible series folder that real files get renamed into. `--park` outside the root +makes the next plan re-file the pile instead of skipping it. Inside a hidden +top-level folder a file's own words may MATCH a registered show, never invent one. Release-group folder names are the one thing no provider can read, so the pending TVShows naming plan files 179 top-level directories (9,931 files) under @@ -322,19 +335,6 @@ executor, the one-journal-per-arm law, the sweep and the rollback. free space, and `rename(2)` refuses it anyway. Trash goes beside the item — `/mnt/entropy/.AGENTTRASH/<date>-<topic>/` — so it costs one inode change and rolls back exactly like the move that made it. -- **A Jellyfin 401 is the header, not the key.** 12.4 authenticates only - `Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token` for - every key, valid or not. This page carried "no credential on this box can - trigger a scan, the two Seerr keys answer 401" for weeks because of that - header — the keys were innocent. Before believing a 401, prove the key against - the server's own table: `select count(*) from ApiKeys where AccessToken = ?` - on `jellyfin.db` opened `mode=ro&immutable=1`. -- **A scan that must see a new folder has to be recursive.** `POST - /Library/Refresh` is the incremental pass, and inotify does not cross a - rootless podman bind, so folders created on the host stay invisible to it. The - move is `POST /Items/{id}/Refresh?Recursive=true` (204) per library, the id - from `GET /Library/VirtualFolders` — its field is `ItemId`, not `Id` — and - that same record's `RefreshStatus` is the progress read. - **A file whose name carries no words is bonus material, not a folder.** `.Extra 1 ( 1H 27min multisub ).mkv` parses to `1`; a movie folder named `1` is not a movie. It becomes a review line telling a human to name the file. @@ -363,7 +363,7 @@ executor, the one-journal-per-arm law, the sweep and the rollback. recreates the folder, so the sweep is undone by the same command as the moves. Proof carries downward: inside a folder the journal proves emptied, every empty directory goes with it. A release's `Screens/` or `Sample/` is never a - move source itself, so it can hold an emptied parent on disk after a sweep. + move source itself, so it can hold an emptied parent until the next sweep. A failed `rmdir` is not a removal: only a folder that is really gone leaves scope, so the next sweep finishes what a permission problem stopped. - **An episode-shaped name marks a SERIES, not a film — and the series is the diff --git a/serverAppConfigs-docs/Dankserver/Plex.md b/serverAppConfigs-docs/Dankserver/Plex.md index 0e7b831..9cc843e 100644 --- a/serverAppConfigs-docs/Dankserver/Plex.md +++ b/serverAppConfigs-docs/Dankserver/Plex.md @@ -87,3 +87,45 @@ root runs as the `dank` user, and the deployer adopts the data root with while the HTTP layer finishes loading; the first boot of a years-old library took ~10 minutes (2026-09 cutover). The deployer gate now allows 600 s before failing; later boots are much faster (warm page cache). + +## Triggering a scan + +A rename on disk is invisible until the section is rescanned, and a moved file is +a `Couldn't find the file to stream` 404 until that scan passes it. Run this from +the host shell through `podman exec`, never from a neighbour container: PMS's +HTTPD closes without a single byte on any request that arrives from another +container's netns (10.89.x.x reads as "empty reply", not as a 401). The token +rides stdin, never argv, where any process on the host could read it out of +`/proc/<pid>/cmdline`; it lives in `Preferences.xml` inside the RW data bind. + +```bash +umask 077 +podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \ + '/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok +cat >/tmp/pxscan.sh <<'EOS' +read -r T +cfg=$(mktemp) +mk() { { printf 'header = "X-Plex-Token: %s"\nheader = "Accept: application/json"\n' "$T" + printf 'url = "%s"\n' "$1"; } > "$cfg"; } +mk "http://127.0.0.1:32400/library/sections" +for k in $(curl -s -K "$cfg" | grep -o '"key":"[0-9]*"' | cut -d\" -f4); do + mk "http://127.0.0.1:32400/library/sections/$k/refresh" + printf 'request = "POST"\n' >> "$cfg" + echo "plex section $k: $(curl -s -o /dev/null -w '%{http_code}' -K "$cfg")" +done +shred -u "$cfg" +EOS +podman cp /tmp/pxscan.sh plex:/tmp/pxscan.sh +podman exec -i plex sh /tmp/pxscan.sh </tmp/px.tok +shred -u /tmp/px.tok +``` + +Every section answers `204` — or `200` with a Directory body when +`Accept: application/json` is sent. Inside the quoted `printf` format a +backslash-newline is LITERAL, not a continuation, so `mk` writes two `printf`s: +one glued line and one stray backslash is a config file curl rejects, and the +failure prints as `option -K: is unknown`, never as a bad config line. + +Proof that a rename landed is not a scan response: `scripts/media-verify.py` +replays the run's journal against the database — the only read that +distinguishes "not scanned yet" from "gone"; see [[Media Relayout]]. diff --git a/tests/media_shows.bats b/tests/media_shows.bats index fd48a5f..043dc8a 100755 --- a/tests/media_shows.bats +++ b/tests/media_shows.bats @@ -8,7 +8,14 @@ # file name. So every test here states which authority a file had — name only, # provider only, both, neither — and asserts where the PLAN files it, never how # the planner decided. A fixture Jellyfin database stands in for the live one and -# the TMDb cache is an input file, so no test touches the network. +# the name caches are input files, so no test touches the network. +# +# The target contract is tvdb-grade naming: +# <Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.<ext> +# tvdb is the IDENTITY authority: a provider name a server derived from a folder +# is not a series name, so a polluted name is undone through the series' tvdb id. +# The review pile parks OUTSIDE the library — a parked folder inside it indexes +# as one junk show, and that show's rows are then mistaken for evidence. # # The two outcomes that cost a re-run of a 50k-file plan are a file filed as an # episode that is not one (it steals an episode number from its real owner) and a @@ -25,11 +32,11 @@ APPLY="$REPO_ROOT/scripts/media-apply.py" # at PATH SIZE — a file with deterministic content. at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; } -# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] — one row of the authority -# export scripts/media-census.py jellyfin writes. +# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] [TMDB] [TITLE] [TVDB] +# — one row of the authority export scripts/media-census.py jellyfin writes. title_row() { - printf '%s\t%s\t%s\t%s\t%s\t%s\t\n' "$1" "$2" "$3" "${4:-}" "${5:-}" "${6:-}" \ - >> "$TITLES" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "${4:-}" \ + "${5:-}" "${6:-}" "${7:-}" "${8:-}" "${9:-}" >> "$TITLES" } # dst SRC — the destination the plan gives SRC, or nothing when it does not move. @@ -40,6 +47,7 @@ dst() { setup() { LIB="$BATS_TEST_TMPDIR/TVShows" PARK="$LIB/_unsorted" + OUTSIDE="$BATS_TEST_TMPDIR/park" MOVIES="$BATS_TEST_TMPDIR/Movies" ENUM="$BATS_TEST_TMPDIR/enum.tsv" PROBED="$BATS_TEST_TMPDIR/probed.tsv" @@ -77,8 +85,15 @@ setup() { # A season folder whose rows are stale: the database numbers every file of this # pack season 16 episode 20, four seasons' worth of them. at "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" 6000 - # One episode-shaped file loose at the library root. at "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" 4000 + # A box-set folder whose Series item the server named AFTER THE FOLDER: the + # row's own name is junk, and only a provider id can undo it. + BOX="$LIB/Show Box (2016) Season 1 S01 (1080p WEB-DL)" + at "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" 6000 + # The park inside the library: the server indexed the whole pile as ONE show + # named after the folder, so its rows state a series that is not a show. + at "$PARK/Park Show S01E01.mkv" 5000 + at "$PARK/Twisted.Metal.S01E01.1080p.WEB-DL.mkv" 5200 # The enum census lists every file; the planner reads that instead of walking # the tree, so the fixture writes the same three columns. @@ -92,17 +107,21 @@ with open(out, "w") as fh: fh.write(f"{os.stat(p).st_size}\t{os.path.splitext(f)[1]}\t{p}\n") PY - printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\n' > "$TITLES" + printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\ttitle\ttvdb\n' > "$TITLES" printf '{}\n' > "$CACHE" title_row "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" \ - "The Examples" 2010 1 1 + "The Examples" 2010 1 1 "" "" "Pilot" title_row "$ANIME/[grp] Anime Pack - 05.mkv" "Anime Pack" "" 1 5 - title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1 - title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1 - title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2 - title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3 + title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots" + title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots" + title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2 "" "" "Double Trouble" + title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3 "" "" "Loose Ends" title_row "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" \ "SNL Pack" "" 16 20 + title_row "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" \ + "Show Box (2016) Season 1" 2016 1 1 "" "2334" "Pilot" "78850" + # The row the server wrote for a parked file: the series is the PARK FOLDER. + title_row "$PARK/Park Show S01E01.mkv" "_unsorted" "" 1 1 } plan() { @@ -111,6 +130,28 @@ plan() { --out "$OUT" --exceptions "$EXC" } +plan_park() { # the review pile outside the library + run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --titles "$TITLES" \ + --probed "$PROBED" --tmdb-cache "$CACHE" --movies "$MOVIES" \ + --park "$OUTSIDE" --out "$OUT" --exceptions "$EXC" +} + +@test "--tmdb with no key is reported as an unfinished plan, not a finished one" { + # A dead provider is invisible where it happens: every 401 is one line on + # stderr, and a terminal that scrolled past them shows a plan that reads as + # complete while every folder still carries the metadata provider's junk. The + # plan is still written — the answers that DID arrive are worth their rename — + # so the state has to ride in the summary line and in the review file. + # env -u keeps this offline no matter what the developer's shell exports. + run env -u TMDB_API_KEY python3 "$PLAN" --root "$LIB" --enum "$ENUM" \ + --titles "$TITLES" --probed "$PROBED" --tmdb-cache "$CACHE" \ + --movies "$MOVIES" --tmdb --out "$OUT" --exceptions "$EXC" + [ "$status" -eq 0 ] + grep -q 'tmdb NOT ASKED (TMDB_API_KEY unset): NOT provider-complete' <<<"$output" + grep -q 'WARNING: tmdb NOT ASKED (TMDB_API_KEY unset) — NOT provider-complete' "$EXC" + grep -qE '^plan: [1-9][0-9]* moves' <<<"$output" +} + @test "the census exports the provider's episode numbering as a TSV" { DB="$BATS_TEST_TMPDIR/jellyfin.db" python3 - "$DB" "$SEASON" <<'PY' @@ -125,7 +166,7 @@ sid = str(uuid.uuid4()) c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)", (sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples", pre, None, None, 2010, None, 0, - '{"ProviderIds":{"Tmdb":"1234"}}')) + '{"ProviderIds":{"Tmdb":"1234","Tvdb":"78850"}}')) c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)", (str(uuid.uuid4()), sid, "MediaBrowser.Controller.Entities.TV.Episode", "Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1, @@ -136,34 +177,113 @@ PY --out "$BATS_TEST_TMPDIR/exp.tsv" [ "$status" -eq 0 ] run head -1 "$BATS_TEST_TMPDIR/exp.tsv" - [ "${lines[0]}" = "file series year season episode extra tmdb" ] + [ "${lines[0]}" = "file series year season episode extra tmdb title tvdb" ] tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \ - | grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234$" + | grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234 Pilot 78850$" } -@test "an episode its own name states files as Show SxxEyy under Season NN" { +@test "the census reads provider ids from the table newer Jellyfin keeps" { + DB="$BATS_TEST_TMPDIR/jellyfin12.db" + python3 - "$DB" "$SEASON" <<'PY' +import os, sqlite3, sys, uuid +db, season = sys.argv[1], sys.argv[2] +c = sqlite3.connect(db) +pre = "/data/TVShows/" + os.path.basename(season) +c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber, + ParentIndexNumber, ProductionYear, ExtraType, + IsVirtualItem, Data)""") +# Jellyfin 12 moved the ids out of the Data blob and into their own table. +c.execute("create table BaseItemProviders (ItemId, ProviderId, ProviderValue)") +sid = str(uuid.uuid4()) +eid = str(uuid.uuid4()) +c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)", + (sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples", + pre, None, None, 2010, None, 0, '{}')) +c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)", + (eid, sid, "MediaBrowser.Controller.Entities.TV.Episode", + "Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1, + None, None, 0, '{}')) +c.execute("insert into BaseItemProviders values (?,?,?)", (sid, "Tvdb", "9999")) +c.commit() +PY + run python3 "$CENSUS" jellyfin --root "$LIB" --db "$DB" \ + --out "$BATS_TEST_TMPDIR/exp12.tsv" + [ "$status" -eq 0 ] + tail -1 "$BATS_TEST_TMPDIR/exp12.tsv" \ + | grep -q " Pilot 9999$" +} + +@test "an episode its own name states files as the titled standard name" { plan [ "$status" -eq 0 ] [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv")" \ - = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.mkv" ] + = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.mkv" ] +} + +@test "the tvdb identity names the series folder, not the name a folder left behind" { + printf '{"tvdb:series:78850": {"name": "Show Box", "year": "2016"}}\n' > "$CACHE" + plan + [ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" \ + = "$LIB/Show Box (2016)/Season 01/Show Box (2016) - S01E01 - Pilot.mkv" ] +} + +@test "a provider name that already carries the year is not given it twice" { + # TVDb answers the reboot as `Acapulco (2021)`. The folder contract is ONE year, + # and `Acapulco (2021) (2021)` is a folder neither matcher nor human reuses. + printf '{"tvdb:series:78850": {"name": "Show Box (2016)", "year": "2016"}}\n' > "$CACHE" + plan + [ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" \ + = "$LIB/Show Box (2016)/Season 01/Show Box (2016) - S01E01 - Pilot.mkv" ] + run grep -F '(2016) (2016)' "$OUT" + [ -z "$output" ] +} + +@test "a series no authority resolves keeps the provider's name and is reported" { + plan + [ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \ + = "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ] + grep -A9 -i "no tvdb" "$EXC" | grep -q "SNL Pack" +} + +@test "a row whose series is the park folder is not evidence of a show" { + plan_park + run awk -F '\t' -v p="$PARK/" '$1 == "move" && index($3, p) == 1 {print $3}' "$OUT" + [ -z "$output" ] + run awk -F '\t' '$1 == "move" && $2 ~ /Park Show S01E01/ {print $3}' "$OUT" + [[ "$output" == "$OUTSIDE/"* ]] +} + +@test "the pile's sanitized display name is still the pile, not a series" { + # Jellyfin drops the leading underscore when it shows the hidden folder, so the + # row says the series is "unsorted" while the file sits in `_unsorted`. Match + # only the underscored spelling and the plan renames real files into a visible + # folder named after the review pile — which is how 617 of them got there. + title_row "$PARK/Twisted.Metal.S01E01.1080p.WEB-DL.mkv" "unsorted" "" 1 1 \ + "" "" "Twisted Metal" + plan_park + run awk -F '\t' -v l="$LIB" \ + '$1 == "move" && index($3, l "/unsorted/") == 1 {print $3}' "$OUT" + [ -z "$output" ] + run awk -F '\t' '$1 == "move" && $2 ~ /Twisted\.Metal\.S01E01/ {print $3}' "$OUT" + [[ "$output" == "$OUTSIDE/"* ]] } @test "a subtitle follows its episode and keeps its language flag" { plan [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt")" \ - = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.en.srt" ] + = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.en.srt" ] } @test "a subtitle two folders down inside Subs/<release-name>/ still follows its episode" { plan [ "$(dst "$SEASON/Subs/The.Examples.S01E01.Pilot.1080p.x265/The.Examples.S01E01.Pilot.1080p.x265.jpn.srt")" \ - = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.jpn.srt" ] + = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.jpn.srt" ] } @test "a subtitle that carries the container's word in its own name still follows the episode" { plan [ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv.forced.srt")" \ - = "$LIB/The Examples (2010)/Season 01/The Examples S01E01.forced.srt" ] + = "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.forced.srt" ] } @test "a season image keeps its own name; a series nfo keeps the series folder" { @@ -178,10 +298,10 @@ PY [ -z "$(awk -F '\t' '$3 ~ /Season .*teaser/ {print $3}' "$OUT")" ] } -@test "a file only the provider numbers files at the episode it is matched to" { +@test "a file only the provider numbers files at the episode it is matched to, title or no title" { plan [ "$(dst "$ANIME/[grp] Anime Pack - 05.mkv")" \ - = "$LIB/Anime Pack/Season 01/Anime Pack S01E05.mkv" ] + = "$LIB/Anime Pack/Season 01/Anime Pack - S01E05.mkv" ] } @test "a bonus folder inside a release pack belongs to the pack's show, not to its own name" { @@ -193,19 +313,19 @@ PY @test "a Season folder outvotes the stale row its files carry" { plan [ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \ - = "$LIB/SNL Pack/Season 19/SNL Pack S19E01.avi" ] + = "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ] } @test "one file holding two episodes keeps both numbers in its name" { plan [ "$(dst "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv")" \ - = "$LIB/Show Two (2016)/Season 01/Show Two S01E02-E03.mkv" ] + = "$LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E02-E03 - Double Trouble.mkv" ] } @test "two files claiming one episode: one keeps the name, the other parks" { plan - n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two S01E01.mkv " "$OUT") + n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E01 - Two Pilots.mkv " "$OUT") [ "$n" -eq 1 ] run dst "$LIB/Dupes/Show Two S01E01.mkv" [[ "$output" == "$PARK/"* ]] @@ -240,6 +360,38 @@ json.dump({"movie:some documentary": [[ "$output" == "$PARK/"* ]] } +@test "a movie answer the file does not itself name stays in the review pile" { + # TMDb's top hit for a truncated query is a NEAR miss, not an answer: "mobile + # suit gundam 00" comes back as Special Edition I while the file is Special + # Edition III, and a one-word query "russia" comes back as a 1963 Bond film. A + # file leaves the TV library only when it states the title the provider names. + printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \ + "$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED" + python3 -c 'import json, sys +json.dump({"movie:some documentary": + {"name": "Some Documentary Special Part Two", "year": "2015", "id": 55}}, + open(sys.argv[1], "w"))' "$CACHE" + plan + run dst "$LIB/Longform/Some Documentary Special (2015).mkv" + [[ "$output" == "$PARK/"* ]] + run awk -F '\t' -v m="$MOVIES/" '$1 == "move" && index($3, m) == 1 {print $3}' "$OUT" + [ -z "$output" ] +} + +@test "a movie answer from another year is not that film" { + # The year the file states outranks a provider's eagerness: the 2015 special is + # not the 1999 one, however alike their names are. + printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \ + "$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED" + python3 -c 'import json, sys +json.dump({"movie:some documentary": + {"name": "Some Documentary Special", "year": "1999", "id": 55}}, + open(sys.argv[1], "w"))' "$CACHE" + plan + run dst "$LIB/Longform/Some Documentary Special (2015).mkv" + [[ "$output" == "$PARK/"* ]] +} + @test "the plan creates every folder before the move that needs it" { plan python3 - "$OUT" "$LIB" "$MOVIES" "$PARK" <<'PY'