Frank
441b908937
The Movies relayout planned from Plex; the TV library has no Plex rows and
58k files whose names lie about which episode they hold. Jellyfin has already
matched and numbered 47,676 of them, so the planner treats that numbering as
the authority and the file name as the fallback: an episode number the name
states, or a provider row, or nothing - and "nothing" parks the file under
_unsorted with a reason instead of inventing a number.
- media-census.py jellyfin: one TSV row per matched episode, its container
path mapped back to the host, series name/year/TMDb id from the Series row
(the episode row carries none of them). Both TSV writers now end lines with
\n so a shell consumer sees the last field clean.
- media-shows.py: the plan. Series folders take the provider's own spelling;
one file claiming two episodes keeps both numbers; two files claiming one
episode - the bigger keeps the standard name, the other parks; bonus
material files under the series' extras folder, never as an episode; a long
file inside the TV library leaves for Movies only when TMDb answers with a
matching title, otherwise it parks. Re-running the plan on a library it
already filed moves nothing.
- tests/media_shows.bats: 15 behaviors, including the executor preflight and
the re-plan stability law.
272 lines
11 KiB
Bash
Executable File
272 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the TV-library relayout — scripts/media-census.py jellyfin (the metadata
|
|
# authority export) and scripts/media-shows.py (the plan).
|
|
#
|
|
# The TV library is 52k files written in five different release layouts, and the
|
|
# one thing that makes a rename of that size survivable is that the plan is
|
|
# derived from what a metadata provider already matched, not from a guess at the
|
|
# file name. So every test here states which authority a file had — name only,
|
|
# provider only, both, neither — and asserts where the PLAN files it, never how
|
|
# the planner decided. A fixture Jellyfin database stands in for the live one and
|
|
# the TMDb cache is an input file, so no test touches the network.
|
|
#
|
|
# The two outcomes that cost a re-run of a 50k-file plan are a file filed as an
|
|
# episode that is not one (it steals an episode number from its real owner) and a
|
|
# move that lands on an existing file. Everything below is written to catch
|
|
# exactly those, plus the gate that matters most: the plan must be replayable by
|
|
# scripts/media-apply.py, whose preflight is the fleet's only overwrite guard.
|
|
|
|
load helpers
|
|
|
|
PLAN="$REPO_ROOT/scripts/media-shows.py"
|
|
CENSUS="$REPO_ROOT/scripts/media-census.py"
|
|
APPLY="$REPO_ROOT/scripts/media-apply.py"
|
|
|
|
# at PATH SIZE — a file with deterministic content.
|
|
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
|
|
|
|
# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] — one row of the authority
|
|
# export scripts/media-census.py jellyfin writes.
|
|
title_row() {
|
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t\n' "$1" "$2" "$3" "${4:-}" "${5:-}" "${6:-}" \
|
|
>> "$TITLES"
|
|
}
|
|
|
|
# dst SRC — the destination the plan gives SRC, or nothing when it does not move.
|
|
dst() {
|
|
awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $3}' "$OUT"
|
|
}
|
|
|
|
setup() {
|
|
LIB="$BATS_TEST_TMPDIR/TVShows"
|
|
PARK="$LIB/_unsorted"
|
|
MOVIES="$BATS_TEST_TMPDIR/Movies"
|
|
ENUM="$BATS_TEST_TMPDIR/enum.tsv"
|
|
PROBED="$BATS_TEST_TMPDIR/probed.tsv"
|
|
TITLES="$BATS_TEST_TMPDIR/titles.tsv"
|
|
CACHE="$BATS_TEST_TMPDIR/tmdb.json"
|
|
OUT="$BATS_TEST_TMPDIR/plan.tsv"
|
|
EXC="$BATS_TEST_TMPDIR/exceptions.txt"
|
|
: > "$PROBED"
|
|
|
|
SEASON="$LIB/The Examples (2010) Season 1 S01 (1080p BluRay x265 Silence)"
|
|
# An episode its own name states, with a subtitle, a season image and a
|
|
# trailer: four things that have to end up in four different places.
|
|
at "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" 9000
|
|
at "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt" 200
|
|
at "$SEASON/folder.jpg" 300
|
|
at "$SEASON/Trailers/teaser.mkv" 1000
|
|
at "$SEASON/tvshow.nfo" 400
|
|
# An anime pack: no episode marker anywhere in its name, numbered by the
|
|
# provider only. Its year is the one thing no authority states.
|
|
ANIME="$LIB/Anime Pack [grp] (1080p HEVC)"
|
|
at "$ANIME/[grp] Anime Pack - 05.mkv" 5000
|
|
# Two releases claiming one episode, and a file holding two episodes.
|
|
at "$LIB/Dupes/Show Two S01E01.mkv" 4000
|
|
at "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" 7000
|
|
at "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" 7000
|
|
# A file no authority numbers, and a long file the movie provider knows.
|
|
at "$LIB/Orphan Tapes/clip001.mp4" 3000
|
|
at "$LIB/Longform/Some Documentary Special (2015).mkv" 8000
|
|
# One episode-shaped file loose at the library root.
|
|
at "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" 4000
|
|
|
|
# The enum census lists every file; the planner reads that instead of walking
|
|
# the tree, so the fixture writes the same three columns.
|
|
python3 - "$LIB" "$ENUM" <<'PY'
|
|
import os, sys
|
|
root, out = sys.argv[1], sys.argv[2]
|
|
with open(out, "w") as fh:
|
|
for d, _, fs in os.walk(root):
|
|
for f in fs:
|
|
p = os.path.join(d, f)
|
|
fh.write(f"{os.stat(p).st_size}\t{os.path.splitext(f)[1]}\t{p}\n")
|
|
PY
|
|
|
|
printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\n' > "$TITLES"
|
|
printf '{}\n' > "$CACHE"
|
|
title_row "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" \
|
|
"The Examples" 2010 1 1
|
|
title_row "$ANIME/[grp] Anime Pack - 05.mkv" "Anime Pack" "" 1 5
|
|
title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1
|
|
title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1
|
|
title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2
|
|
title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3
|
|
}
|
|
|
|
plan() {
|
|
run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --titles "$TITLES" \
|
|
--probed "$PROBED" --tmdb-cache "$CACHE" --movies "$MOVIES" \
|
|
--out "$OUT" --exceptions "$EXC"
|
|
}
|
|
|
|
@test "the census exports the provider's episode numbering as a TSV" {
|
|
DB="$BATS_TEST_TMPDIR/jellyfin.db"
|
|
python3 - "$DB" "$SEASON" <<'PY'
|
|
import os, sqlite3, sys, uuid
|
|
db, season = sys.argv[1], sys.argv[2]
|
|
c = sqlite3.connect(db)
|
|
pre = "/data/TVShows/" + os.path.basename(season)
|
|
c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber,
|
|
ParentIndexNumber, ProductionYear, ExtraType,
|
|
IsVirtualItem, Data)""")
|
|
sid = str(uuid.uuid4())
|
|
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
|
(sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples",
|
|
pre, None, None, 2010, None, 0,
|
|
'{"ProviderIds":{"Tmdb":"1234"}}'))
|
|
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
|
(str(uuid.uuid4()), sid, "MediaBrowser.Controller.Entities.TV.Episode",
|
|
"Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1,
|
|
None, None, 0, '{}'))
|
|
c.commit()
|
|
PY
|
|
run python3 "$CENSUS" jellyfin --root "$LIB" --db "$DB" \
|
|
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
|
[ "$status" -eq 0 ]
|
|
run head -1 "$BATS_TEST_TMPDIR/exp.tsv"
|
|
[ "${lines[0]}" = "file series year season episode extra tmdb" ]
|
|
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
|
| grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234$"
|
|
}
|
|
|
|
@test "an episode its own name states files as Show SxxEyy under Season NN" {
|
|
plan
|
|
[ "$status" -eq 0 ]
|
|
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv")" \
|
|
= "$LIB/The Examples (2010)/Season 01/The Examples S01E01.mkv" ]
|
|
}
|
|
|
|
@test "a subtitle follows its episode and keeps its language flag" {
|
|
plan
|
|
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt")" \
|
|
= "$LIB/The Examples (2010)/Season 01/The Examples S01E01.en.srt" ]
|
|
}
|
|
|
|
@test "a season image keeps its own name; a series nfo keeps the series folder" {
|
|
plan
|
|
[ "$(dst "$SEASON/folder.jpg")" = "$LIB/The Examples (2010)/Season 01/folder.jpg" ]
|
|
[ "$(dst "$SEASON/tvshow.nfo")" = "$LIB/The Examples (2010)/tvshow.nfo" ]
|
|
}
|
|
|
|
@test "bonus material files as an extra, never as an episode" {
|
|
plan
|
|
[ "$(dst "$SEASON/Trailers/teaser.mkv")" = "$LIB/The Examples (2010)/trailers/teaser.mkv" ]
|
|
[ -z "$(awk -F '\t' '$3 ~ /Season .*teaser/ {print $3}' "$OUT")" ]
|
|
}
|
|
|
|
@test "a file only the provider numbers files at the episode it is matched to" {
|
|
plan
|
|
[ "$(dst "$ANIME/[grp] Anime Pack - 05.mkv")" \
|
|
= "$LIB/Anime Pack/Season 01/Anime Pack S01E05.mkv" ]
|
|
}
|
|
|
|
@test "one file holding two episodes keeps both numbers in its name" {
|
|
plan
|
|
[ "$(dst "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv")" \
|
|
= "$LIB/Show Two (2016)/Season 01/Show Two S01E02-E03.mkv" ]
|
|
}
|
|
|
|
@test "two files claiming one episode: one keeps the name, the other parks" {
|
|
plan
|
|
n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two S01E01.mkv " "$OUT")
|
|
[ "$n" -eq 1 ]
|
|
run dst "$LIB/Dupes/Show Two S01E01.mkv"
|
|
[[ "$output" == "$PARK/"* ]]
|
|
run dst "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv"
|
|
[[ "$output" == "$LIB/Show Two (2016)/Season 01/"* ]]
|
|
}
|
|
|
|
@test "a file no authority numbers is parked, never renamed into the library" {
|
|
plan
|
|
run dst "$LIB/Orphan Tapes/clip001.mp4"
|
|
[[ "$output" == "$PARK/Orphan Tapes/clip001.mp4" ]]
|
|
grep -q "clip001.mp4" "$EXC"
|
|
}
|
|
|
|
@test "a long file the movie provider knows leaves the TV library for the movie one" {
|
|
printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \
|
|
"$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED"
|
|
python3 -c 'import json, sys
|
|
json.dump({"movie:some documentary":
|
|
{"name": "Some Documentary Special", "year": 2015, "id": 55}},
|
|
open(sys.argv[1], "w"))' "$CACHE"
|
|
plan
|
|
[ "$(dst "$LIB/Longform/Some Documentary Special (2015).mkv")" \
|
|
= "$MOVIES/Some Documentary Special (2015)/Some Documentary Special (2015).mkv" ]
|
|
}
|
|
|
|
@test "a long file no movie provider knows is parked, not moved into Movies" {
|
|
printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \
|
|
"$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED"
|
|
plan
|
|
run dst "$LIB/Longform/Some Documentary Special (2015).mkv"
|
|
[[ "$output" == "$PARK/"* ]]
|
|
}
|
|
|
|
@test "the plan creates every folder before the move that needs it" {
|
|
plan
|
|
python3 - "$OUT" "$LIB" "$MOVIES" "$PARK" <<'PY'
|
|
import os, sys
|
|
plan, known = sys.argv[1], set(sys.argv[2:])
|
|
made, bad = set(), []
|
|
for line in open(plan).read().splitlines()[1:]:
|
|
action, _src, dest, _kind = line.split("\t")
|
|
if action == "mkdir":
|
|
made.add(dest)
|
|
elif action == "move":
|
|
d = os.path.dirname(dest)
|
|
while d not in known and d != os.path.dirname(d):
|
|
if d not in made:
|
|
bad.append(f"{dest}: {d} was never made")
|
|
d = os.path.dirname(d)
|
|
print("\n".join(bad[:5]))
|
|
sys.exit(1 if bad else 0)
|
|
PY
|
|
}
|
|
|
|
@test "no move lands outside the library or the movie library" {
|
|
plan
|
|
run awk -F '\t' -v l="$LIB" -v m="$MOVIES" \
|
|
'$1 == "move" && $3 !~ "^" l "/" && $3 !~ "^" m "/" {print $3}' "$OUT"
|
|
[ -z "$output" ]
|
|
}
|
|
|
|
@test "the executor preflights the plan and no target is claimed twice" {
|
|
plan
|
|
run python3 "$APPLY" apply --plan "$OUT" --root "$BATS_TEST_TMPDIR" \
|
|
--journal "$BATS_TEST_TMPDIR/journal.jsonl"
|
|
[ "$status" -eq 0 ]
|
|
dup=$(awk -F '\t' '$1 == "move" {print $3}' "$OUT" | sort | uniq -d)
|
|
[ -z "$dup" ]
|
|
}
|
|
|
|
@test "re-planning a library the plan already satisfies moves nothing" {
|
|
plan
|
|
# Applying the plan and re-censusing is exactly what the operator does next:
|
|
# every authority file is keyed by path, so all three are replayed through the
|
|
# plan the way a fresh census would see them. (The enum has no header; the
|
|
# titles export does, and its header row is not a path so it survives.)
|
|
python3 - "$OUT" "$ENUM" "$TITLES" "$PROBED" <<'PY'
|
|
import os, sys
|
|
plan = sys.argv[1]
|
|
moves = {}
|
|
for line in open(plan).read().splitlines()[1:]:
|
|
action, src, dest, _kind = line.split("\t")
|
|
if action == "move":
|
|
moves[src] = dest
|
|
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
os.rename(src, dest)
|
|
for path, col in ((sys.argv[2], 2), (sys.argv[3], 0), (sys.argv[4], 1)):
|
|
if not os.path.getsize(path):
|
|
continue
|
|
rows = [line.rstrip("\n").split("\t") for line in open(path)]
|
|
with open(path, "w") as fh:
|
|
for f in rows:
|
|
f[col] = moves.get(f[col], f[col])
|
|
fh.write("\t".join(f) + "\n")
|
|
PY
|
|
plan
|
|
[ -z "$(awk -F '\t' '$1 == "move" {print $2}' "$OUT")" ]
|
|
}
|