From 32b58dd787e43ac76960d9a24080532f24743996 Mon Sep 17 00:00:00 2001 From: Frank Date: Sat, 31 Oct 2026 07:16:46 -0700 Subject: [PATCH] fix(media): TMDb's v3 key travels as a query parameter; Bearer is a JWT only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first tvdb-grade TV plan built 49,771 moves with not ONE TMDb answer: every search and every external_ids call came back 401. The credential on this box is the v3 key (~/.config/secrets/tmdb, `api_key=`), and TMDb takes a v3 key only as the `api_key` query parameter — a v4 request token is the only thing the Bearer header accepts. Measured on the same credential in one pass: the query form resolves `New Girl (2011)`, the Bearer form raises HTTP 401. tmdb_request() now picks the transport from the shape it was handed (three dot-separated segments = a JWT → Bearer, anything else → query parameter), and the old header comment's reasoning is corrected: a token in argv is world-readable in /proc//cmdline, but this URL is built inside the process and never reaches argv, so the query form leaks nothing. The failure mode that let it look like a finished plan is fixed with it. The 401s were stderr-only, so the summary printed a confident move count while every series folder still carried the metadata provider's folder-echo name — the exact junk this contract exists to undo. Provider health is now part of the summary line and of the exceptions file, and a test covers it offline with `env -u TMDB_API_KEY`: a plan that asked and could not ask says NOT provider-complete in both places, and still writes the moves it can justify, because the answers that did arrive are worth their rename. --- scripts/media-shows.py | 59 ++++++++++++++++--- .../Dankserver/Media Relayout.md | 11 ++-- tests/media_shows.bats | 16 +++++ 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/scripts/media-shows.py b/scripts/media-shows.py index b722726..481674b 100755 --- a/scripts/media-shows.py +++ b/scripts/media-shows.py @@ -127,6 +127,7 @@ try: except (OSError, ValueError): CACHE = {} API_KEY = os.environ.get("TMDB_API_KEY", "") +TMDB = {"tries": 0, "failed": 0} VIDEO_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg", ".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".flv", ".vob", ".rm", @@ -240,6 +241,20 @@ def leading_index(name): return int(m.group(1)) if m else None +def tmdb_request(url): + """One TMDb call, authenticated. The two credential shapes are NOT + interchangeable and the difference is a 401 on every single call: a v3 key + travels as the `api_key` query parameter, and only a v4 request token is + accepted as a Bearer header. A token in argv is world-readable in + /proc//cmdline — this URL is built inside the process and never reaches + argv, so the query form is safe here.""" + TMDB["tries"] += 1 + if API_KEY.count(".") == 2: # three segments: a v4 JWT + return http_json(url, headers={"Authorization": "Bearer " + API_KEY}) + return http_json(url + ("&" if "?" in url else "?") + "api_key=" + + urllib.parse.quote(API_KEY)) + + def tmdb(kind, query): """A provider answer for a query: the cache first, TMDb once, then never again.""" ck = f"{kind}:{query}" @@ -253,14 +268,10 @@ def tmdb(kind, query): path = "movie" if kind == "movie" else "tv" url = ("https://api.themoviedb.org/3/search/" + path + "?query=" + urllib.parse.quote(query) + "&include_adult=false") - req = urllib.request.Request(url, headers={ - # The key rides a header, never the URL: a token in argv is world-readable. - "Authorization": "Bearer " + API_KEY, - "Accept": "application/json"}) try: - with urllib.request.urlopen(req, timeout=20) as r: - found = json.load(r).get("results") or [] + found = tmdb_request(url).get("results") or [] except Exception as e: # noqa: BLE001 + TMDB["failed"] += 1 print(f"tmdb {kind} lookup failed for {query!r}: {e}", file=sys.stderr) return None if not found: @@ -362,9 +373,9 @@ def tvdb_id_for_tmdb(mid): if not (ARGS.tmdb and API_KEY): return "" try: - got = http_json(f"https://api.themoviedb.org/3/tv/{mid}/external_ids", - headers={"Authorization": "Bearer " + API_KEY}) + got = tmdb_request("https://api.themoviedb.org/3/tv/" + mid + "/external_ids") except Exception as e: # noqa: BLE001 + TMDB["failed"] += 1 print(f"tmdb external ids failed for {mid}: {e}", file=sys.stderr) return "" CACHE[ck] = str(got.get("tvdb_id") or "") @@ -894,6 +905,30 @@ for r, reason in unknown: # --------------------------------------------------------------------- output +def provider_broken(): + """TMDb was asked for names and did not answer: the plan is not the plan the + operator asked for, whatever the move count says.""" + return bool(ARGS.tmdb and (not API_KEY or TMDB["failed"])) + + +def provider_note(): + """Provider health on the summary line. The 401s are already on stderr, and a + terminal that scrolled past them shows a plan that looks finished: the count + belongs where it cannot be missed, because a half-resolved plan renames + folders with the folder's own junk words.""" + if not (ARGS.tmdb or ARGS.tvdb): + return "" + if not ARGS.tmdb: + return ", tmdb off (names from the cache and tvdb only)" + if not API_KEY: + return ", tmdb NOT ASKED (TMDB_API_KEY unset): NOT provider-complete" + if not TMDB["tries"]: + return ", tmdb cache-only (every name was already answered)" + done = TMDB["tries"] - TMDB["failed"] + return (f", tmdb {done}/{TMDB['tries']} answered" + + (" — NOT provider-complete" if TMDB["failed"] else "")) + + def write_plan(): """mkdir rows for every destination folder and its whole chain, then the moves. Parents first: the executor creates directories in the order they are listed.""" @@ -915,7 +950,7 @@ def write_plan(): sized = sum(held[s]["bytes"] for s, _d, _k in moves if s in held) print(f"plan: {len(moves) - len(parked)} moves, {len(parked)} parked" f" ({sized / 2**40:.2f} TiB touched), {len(keeps)} already placed," - f" {len(dirs)} folders -> {ARGS.out}") + f" {len(dirs)} folders{provider_note()} -> {ARGS.out}") def write_exceptions(): @@ -926,6 +961,12 @@ def write_exceptions(): fh.write(f"Everything below is parked under {PARK}, keeping the relative path it\n") fh.write("has today. Nothing is deleted and nothing is overwritten: a parked file\n") fh.write("is one rename from wherever the plan is pointed next.\n\n") + if provider_broken(): + fh.write("WARNING: " + provider_note().strip(", ").replace(": NOT", " — NOT") + + "\n Names and episode titles that should have come from TMDb are\n" + " missing, so folders keep the metadata provider's words. Fix the\n" + " credential and re-plan: the cache keeps every answer that did\n" + " arrive, so the re-run costs only the lookups that failed.\n\n") for reason in sorted(reasons, key=lambda r: -len(reasons[r])): rs = reasons[reason] fh.write(f"{len(rs):>6} files {reason}\n") diff --git a/serverAppConfigs-docs/Dankserver/Media Relayout.md b/serverAppConfigs-docs/Dankserver/Media Relayout.md index 7a056e1..86fce13 100644 --- a/serverAppConfigs-docs/Dankserver/Media Relayout.md +++ b/serverAppConfigs-docs/Dankserver/Media Relayout.md @@ -181,10 +181,13 @@ python3 scripts/media-census.py probe --root /mnt/entropy/TVShows \ # seeking the array again. It is also the only column the film arm reads, and # only for a file with no marker and no provider row (4,794 of 58,551 today, # all of them covered by a re-key that costs seconds, not an hour). -# 3. The plan. --tmdb needs TMDB_API_KEY, --tvdb needs ~/.config/secrets/tvdb -# (`api_key=`, never argv); both write into the SAME cache file, so the plan is -# reproducible offline afterwards. Run it once to fill the cache, then again: -# the second run is the one whose numbers are quoted. +# 3. The plan. --tmdb needs TMDB_API_KEY — the v3 key in ~/.config/secrets/tmdb, +# which travels as the `api_key` query parameter. Only a v4 request token is +# accepted as a Bearer header, and the wrong shape answers 401 on EVERY call, +# which the summary line flags as NOT provider-complete. --tvdb reads its +# `api_key=` line from ~/.config/secrets/tvdb, never argv. Both write the SAME +# cache file, so the plan is reproducible offline afterwards: run it once to +# fill the cache, and quote the second run. # Paths are only strings to a dry run, so this step can run on the workstation # against scp'd TSVs — which is where the two provider secrets live. python3 scripts/media-shows.py --root /mnt/entropy/TVShows --enum /tmp/tv_enum.tsv \ diff --git a/tests/media_shows.bats b/tests/media_shows.bats index 6502013..71fe428 100755 --- a/tests/media_shows.bats +++ b/tests/media_shows.bats @@ -135,6 +135,22 @@ plan_park() { # the review pile outside the lib --park "$OUTSIDE" --out "$OUT" --exceptions "$EXC" } +@test "--tmdb with no key is reported as an unfinished plan, not a finished one" { + # A dead provider is invisible where it happens: every 401 is one line on + # stderr, and a terminal that scrolled past them shows a plan that reads as + # complete while every folder still carries the metadata provider's junk. The + # plan is still written — the answers that DID arrive are worth their rename — + # so the state has to ride in the summary line and in the review file. + # env -u keeps this offline no matter what the developer's shell exports. + run env -u TMDB_API_KEY python3 "$PLAN" --root "$LIB" --enum "$ENUM" \ + --titles "$TITLES" --probed "$PROBED" --tmdb-cache "$CACHE" \ + --movies "$MOVIES" --tmdb --out "$OUT" --exceptions "$EXC" + [ "$status" -eq 0 ] + grep -q 'tmdb NOT ASKED (TMDB_API_KEY unset): NOT provider-complete' <<<"$output" + grep -q 'WARNING: tmdb NOT ASKED (TMDB_API_KEY unset) — NOT provider-complete' "$EXC" + grep -qE '^plan: [1-9][0-9]* moves' <<<"$output" +} + @test "the census exports the provider's episode numbering as a TSV" { DB="$BATS_TEST_TMPDIR/jellyfin.db" python3 - "$DB" "$SEASON" <<'PY'