Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a5a9c04da2 | ||
|
|
4b67c3c4d8 | ||
|
|
afad21dda5 |
No files matched your search
+45
-58
@@ -21,12 +21,11 @@ scripts/media-plan.py and scripts/media-shows.py read them instead of touching m
|
|||||||
the titles Plex already matched — the authority over any filename guess
|
the titles Plex already matched — the authority over any filename guess
|
||||||
|
|
||||||
jellyfin file <TAB> series <TAB> year <TAB> season <TAB> episode <TAB> extra
|
jellyfin file <TAB> series <TAB> year <TAB> season <TAB> episode <TAB> extra
|
||||||
<TAB> tmdb <TAB> title <TAB> tvdb (see JF_COLS)
|
<TAB> tmdb (see JF_COLS)
|
||||||
one row per episode Jellyfin has matched and numbered, its path mapped
|
one row per episode Jellyfin has matched and numbered, its path mapped
|
||||||
from the container's view back to the host; `year`, `tmdb` and `tvdb`
|
from the container's view back to the host; `year` and `tmdb` are the
|
||||||
are the SERIES' values, `title` is the episode title the row carries,
|
SERIES' values, `extra` is Jellyfin's own extra type when the row is
|
||||||
`extra` is Jellyfin's own extra type when the row is bonus material
|
bonus material rather than an episode
|
||||||
rather than an episode
|
|
||||||
|
|
||||||
music file <TAB> artist <TAB> album <TAB> year <TAB> disc <TAB> track <TAB> title
|
music file <TAB> artist <TAB> album <TAB> year <TAB> disc <TAB> track <TAB> title
|
||||||
<TAB> mb <TAB> performer (see MUSIC_COLS)
|
<TAB> mb <TAB> performer (see MUSIC_COLS)
|
||||||
@@ -84,14 +83,13 @@ left join metadata_items par on par.id = mi.parent_id
|
|||||||
where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
||||||
"""
|
"""
|
||||||
JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db"
|
JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db"
|
||||||
JF_COLS = ["file", "series", "year", "season", "episode", "extra",
|
JF_COLS = ["file", "series", "year", "season", "episode", "extra", "tmdb"]
|
||||||
"tmdb", "title", "tvdb"]
|
|
||||||
# Jellyfin stores the entity's .NET class as its type; the TV tree is these three.
|
# Jellyfin stores the entity's .NET class as its type; the TV tree is these three.
|
||||||
JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode",
|
JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode",
|
||||||
"MediaBrowser.Controller.Entities.TV.Season",
|
"MediaBrowser.Controller.Entities.TV.Season",
|
||||||
"MediaBrowser.Controller.Entities.TV.Series")
|
"MediaBrowser.Controller.Entities.TV.Series")
|
||||||
JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType,
|
JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, IsVirtualItem
|
||||||
IsVirtualItem, Name from BaseItems where Type = ? and Path like ?""")
|
from BaseItems where Type = ? and Path like ?""")
|
||||||
|
|
||||||
# The music tree is these two entity classes. The track row carries the tags
|
# The music tree is these two entity classes. The track row carries the tags
|
||||||
# Jellyfin read out of the file; the album row carries what the provider matched,
|
# Jellyfin read out of the file; the album row carries what the provider matched,
|
||||||
@@ -130,6 +128,26 @@ where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
|||||||
and al.deleted_at is null
|
and al.deleted_at is null
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
def open_plex_ro(db):
|
||||||
|
"""Plex's library, read the way Plex writes it.
|
||||||
|
|
||||||
|
`immutable=1` tells sqlite the file never changes, so it skips the
|
||||||
|
write-ahead log: every row a scan has committed but not yet checkpointed
|
||||||
|
is invisible, and reading a live database that way can fail outright as
|
||||||
|
"database disk image is malformed". Plain ro reads the WAL through its
|
||||||
|
-shm; immutable is only the fallback for a database whose -shm cannot be
|
||||||
|
read. The same law scripts/media-verify.py states.
|
||||||
|
"""
|
||||||
|
last = None
|
||||||
|
for opt in ("mode=ro", "mode=ro&immutable=1"):
|
||||||
|
try:
|
||||||
|
return sqlite3.connect(f"file:{db}?{opt}", uri=True)
|
||||||
|
except sqlite3.Error as e:
|
||||||
|
last = e
|
||||||
|
sys.exit(f"cannot open the Plex database read-only ({db}): {last}\n"
|
||||||
|
"Plex runs rootless here: run this as the container's owner, never "
|
||||||
|
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||||
|
|
||||||
|
|
||||||
def enumerate_lib(root, out):
|
def enumerate_lib(root, out):
|
||||||
"""One row per real file under root: size, extension, path."""
|
"""One row per real file under root: size, extension, path."""
|
||||||
@@ -229,13 +247,7 @@ def probe(enum, out, jobs):
|
|||||||
|
|
||||||
def plex(root, db, out):
|
def plex(root, db, out):
|
||||||
"""Plex's matched titles for files under root, from its DB, read-only."""
|
"""Plex's matched titles for files under root, from its DB, read-only."""
|
||||||
uri = f"file:{db}?mode=ro&immutable=1"
|
con = open_plex_ro(db)
|
||||||
try:
|
|
||||||
con = sqlite3.connect(uri, uri=True)
|
|
||||||
except sqlite3.Error as e:
|
|
||||||
sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n"
|
|
||||||
"Plex runs rootless here: run this as the container's owner, never "
|
|
||||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
|
||||||
with open(out, "w", newline="") as fh:
|
with open(out, "w", newline="") as fh:
|
||||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||||
w.writerow(PLEX_COLS)
|
w.writerow(PLEX_COLS)
|
||||||
@@ -256,10 +268,9 @@ def jellyfin(root, db, container, out):
|
|||||||
|
|
||||||
The numbering is what matters, not the name: a release group's file name says
|
The numbering is what matters, not the name: a release group's file name says
|
||||||
nothing about which episode a file is, and Jellyfin has already answered that
|
nothing about which episode a file is, and Jellyfin has already answered that
|
||||||
question against a provider. Season/episode and the episode's own title come
|
question against a provider. Season/episode come from the episode row; the
|
||||||
from the episode row; the series name, its year and its TMDb and Tvdb ids come
|
series name, its year and its TMDb id come from the Series row the episode
|
||||||
from the Series row the episode climbs to, because those are the only place
|
climbs to, because those are the only place Jellyfin keeps them.
|
||||||
Jellyfin keeps them.
|
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||||
@@ -280,38 +291,21 @@ def jellyfin(root, db, container, out):
|
|||||||
" where Type in (?,?,?)", JF_TV):
|
" where Type in (?,?,?)", JF_TV):
|
||||||
node[i] = (pid, typ, name, year, data)
|
node[i] = (pid, typ, name, year, data)
|
||||||
|
|
||||||
# Jellyfin 12 keeps provider ids in their own table; older builds kept them in
|
|
||||||
# the row's Data blob. Both shapes are read, because the tvdb id the series
|
|
||||||
# naming depends on is wherever this build happened to write it.
|
|
||||||
prov = {}
|
|
||||||
if "BaseItemProviders" in {r[0] for r in con.execute(
|
|
||||||
"select name from sqlite_master where type='table'")}:
|
|
||||||
for iid, pid, val in con.execute(
|
|
||||||
"select ItemId, ProviderId, ProviderValue from BaseItemProviders"
|
|
||||||
" where ProviderId in ('Tmdb', 'Tvdb')"):
|
|
||||||
prov[(iid, pid)] = val
|
|
||||||
|
|
||||||
def provider_of(item_id, want):
|
|
||||||
got = prov.get((item_id, want))
|
|
||||||
if got:
|
|
||||||
return str(got)
|
|
||||||
try:
|
|
||||||
blob = json.loads(node[item_id][4] or "{}").get("ProviderIds") or {}
|
|
||||||
except (KeyError, ValueError, TypeError):
|
|
||||||
return ""
|
|
||||||
return str(blob.get(want, "") or "")
|
|
||||||
|
|
||||||
def series_of(item_id):
|
def series_of(item_id):
|
||||||
"""Climb episode -> season -> series; empty when the chain is broken."""
|
"""Climb episode -> season -> series; empty when the chain is broken."""
|
||||||
for _ in range(4):
|
for _ in range(4):
|
||||||
if item_id not in node:
|
if item_id not in node:
|
||||||
return "", "", "", ""
|
return "", "", ""
|
||||||
pid, typ = node[item_id][0], node[item_id][1]
|
pid, typ, name, year, data = node[item_id]
|
||||||
if typ.endswith(".Series"):
|
if typ.endswith(".Series"):
|
||||||
return (node[item_id][2] or "", node[item_id][3] or "",
|
try:
|
||||||
provider_of(item_id, "Tmdb"), provider_of(item_id, "Tvdb"))
|
tmdb = (json.loads(data or "{}")
|
||||||
|
.get("ProviderIds") or {}).get("Tmdb", "")
|
||||||
|
except ValueError:
|
||||||
|
tmdb = ""
|
||||||
|
return name or "", year or "", tmdb
|
||||||
item_id = pid
|
item_id = pid
|
||||||
return "", "", "", ""
|
return "", "", ""
|
||||||
|
|
||||||
# The fleet's quadlets bind the share at /data, so the container's view of
|
# The fleet's quadlets bind the share at /data, so the container's view of
|
||||||
# this library is /data/<the library's own folder name> unless told otherwise.
|
# this library is /data/<the library's own folder name> unless told otherwise.
|
||||||
@@ -322,18 +316,17 @@ def jellyfin(root, db, container, out):
|
|||||||
with open(out, "w", newline="") as fh:
|
with open(out, "w", newline="") as fh:
|
||||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||||
w.writerow(JF_COLS)
|
w.writerow(JF_COLS)
|
||||||
for iid, path, idx, pidx, extra, virt, epname in con.execute(
|
for iid, path, idx, pidx, extra, virt in con.execute(JF_SQL,
|
||||||
JF_SQL, (JF_TV[0], prefix + "%")):
|
(JF_TV[0], prefix + "%")):
|
||||||
if virt or not path or not path.startswith(prefix):
|
if virt or not path or not path.startswith(prefix):
|
||||||
skipped += 1 # a virtual item is a variant, not a file
|
skipped += 1 # a virtual item is a variant, not a file
|
||||||
continue
|
continue
|
||||||
sname, syear, tmdb, tvdb = series_of(iid)
|
sname, syear, tmdb = series_of(iid)
|
||||||
w.writerow([root + "/" + path[len(prefix):],
|
w.writerow([root + "/" + path[len(prefix):],
|
||||||
sname.replace("\t", " "), syear,
|
sname.replace("\t", " "), syear,
|
||||||
"" if pidx is None else pidx,
|
"" if pidx is None else pidx,
|
||||||
"" if idx is None else idx,
|
"" if idx is None else idx,
|
||||||
(extra or "").replace("\t", " "), tmdb,
|
(extra or "").replace("\t", " "), tmdb])
|
||||||
(epname or "").replace("\t", " "), tvdb])
|
|
||||||
written += 1
|
written += 1
|
||||||
print(f"episodes={written} virtual-or-blank={skipped} -> {out}")
|
print(f"episodes={written} virtual-or-blank={skipped} -> {out}")
|
||||||
|
|
||||||
@@ -426,13 +419,7 @@ def audiobooks(root, db, out):
|
|||||||
at least one un-indexed chapter, so the relayout that reads this file must
|
at least one un-indexed chapter, so the relayout that reads this file must
|
||||||
not renumber anything from it.
|
not renumber anything from it.
|
||||||
"""
|
"""
|
||||||
uri = f"file:{db}?mode=ro&immutable=1"
|
con = open_plex_ro(db)
|
||||||
try:
|
|
||||||
con = sqlite3.connect(uri, uri=True)
|
|
||||||
except sqlite3.Error as e:
|
|
||||||
sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n"
|
|
||||||
"Plex runs rootless here: run this as the container's owner, never "
|
|
||||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
|
||||||
tables = {r[0] for r in con.execute(
|
tables = {r[0] for r in con.execute(
|
||||||
"select name from sqlite_master where type='table'")}
|
"select name from sqlite_master where type='table'")}
|
||||||
missing = {"media_parts", "media_items", "metadata_items"} - tables
|
missing = {"media_parts", "media_items", "metadata_items"} - tables
|
||||||
|
|||||||
+45
-349
@@ -6,40 +6,29 @@ r"""Plan a Jellyfin-format relayout of a TV library. DRY RUN ONLY: nothing moves
|
|||||||
[--out TSV] [--exceptions TXT]
|
[--out TSV] [--exceptions TXT]
|
||||||
|
|
||||||
Inputs are the TSVs scripts/media-census.py exports: `enumerate` lists what
|
Inputs are the TSVs scripts/media-census.py exports: `enumerate` lists what
|
||||||
exists, `jellyfin` says which episode each file IS (its series' TMDb and Tvdb
|
exists, `jellyfin` says which episode each file IS, and `probe` is optional
|
||||||
ids, and the episode title the row carries), and `probe` is optional duration
|
duration evidence for the movie rule. The TMDb cache is a plain JSON file so a
|
||||||
evidence for the movie rule. Every name and title answer lives in the cache JSON
|
plan stays reproducible: answers are read from it, and a miss is looked up only
|
||||||
so a plan stays reproducible: answers are read from it, and a miss is looked up
|
with --tmdb and TMDB_API_KEY in the environment (the key never enters argv, and
|
||||||
only with --tmdb/--tvdb and their credentials (a key never enters argv, and
|
|
||||||
never the cache).
|
never the cache).
|
||||||
|
|
||||||
Target contract (jellyfin.org/docs/general/server/media/shows):
|
Target contract (jellyfin.org/docs/general/server/media/shows):
|
||||||
|
|
||||||
<root>/<Series> (<year>)/Season <NN>/<Series> (<year>) - SxxEyy - <Title>.ext
|
<root>/<Series> (<year>)/Season <NN>/<Series> SxxEyy.ext
|
||||||
one file holding two episodes SxxE01-E02 specials Season 00
|
one file holding two episodes SxxE01-E02 specials Season 00
|
||||||
a documented part of an episode SxxE03 Part 1 sidecars <base>.<lang>.srt
|
a documented part of an episode SxxE03 Part 1 sidecars <base>.<lang>.srt
|
||||||
bonus material <series>/trailers|featurettes|… — never a season
|
bonus material <series>/trailers|featurettes|… — never a season
|
||||||
|
|
||||||
The series name and the episode title are the ones the IDENTITY authority — tvdb
|
|
||||||
— states for the id the metadata server matched, not the words a release folder
|
|
||||||
or a server's folder-derived Series name happens to carry.
|
|
||||||
|
|
||||||
PLAN is the TSV scripts/media-apply.py replays: `action src dst kind`, every
|
PLAN is the TSV scripts/media-apply.py replays: `action src dst kind`, every
|
||||||
mkdir row before any move, and `keep` for a file already where the contract puts
|
mkdir row before any move, and `keep` for a file already where the contract puts
|
||||||
it. EXCEPTIONS is the review list: everything parked, and why.
|
it. EXCEPTIONS is the review list: everything parked, and why.
|
||||||
|
|
||||||
The authority chain, strongest first, for one video file:
|
The authority chain, strongest first, for one video file:
|
||||||
|
|
||||||
1. a provider row — Jellyfin matched this exact path and numbered it. The
|
1. a provider row — Jellyfin matched this exact path and numbered it.
|
||||||
row's NAME is only evidence once an id backs it: the server names a Series
|
|
||||||
after the folder it scanned, so `24 (2001) Season 1` is a folder echo, and
|
|
||||||
the series' tvdb id is what undoes it.
|
|
||||||
2. the file's own SxxEyy / 1x04 marker, the series taken from a sibling's
|
2. the file's own SxxEyy / 1x04 marker, the series taken from a sibling's
|
||||||
provider row, else from the nearest ancestor folder that has one, else from
|
provider row, else from the nearest ancestor folder that has one, else from
|
||||||
a registered show its own words name, else — outside the review pile — from
|
the release folder's own words.
|
||||||
the release folder's words. Inside a hidden top-level folder a file may
|
|
||||||
MATCH a registered show but never INVENT one: invention is what filled the
|
|
||||||
pile to begin with.
|
|
||||||
3. nothing. An unnumbered file is PARKED under --park with its relative path
|
3. nothing. An unnumbered file is PARKED under --park with its relative path
|
||||||
kept: an invented `Season 01` steals an episode number from the file that
|
kept: an invented `Season 01` steals an episode number from the file that
|
||||||
really is S01E01, and a wrong number is worse than no number.
|
really is S01E01, and a wrong number is worse than no number.
|
||||||
@@ -58,11 +47,8 @@ Guards, and why each is here:
|
|||||||
* Series art and the show nfo follow the SERIES, a season image follows its
|
* Series art and the show nfo follow the SERIES, a season image follows its
|
||||||
SEASON, a subtitle or episode nfo follows the episode's new base — an episode
|
SEASON, a subtitle or episode nfo follows the episode's new base — an episode
|
||||||
moved without its subtitles is a half-move.
|
moved without its subtitles is a half-move.
|
||||||
* Hidden files, macOS forks, qBittorrent resume parts are never moved. A file
|
* Hidden files, macOS forks, qBittorrent resume parts and the park pile itself
|
||||||
already inside --park is never moved either: that is what makes re-planning
|
are never moved. Skipping the park is what makes re-planning stable.
|
||||||
stable. --park belongs OUTSIDE the library — parked files inside it get
|
|
||||||
indexed as one show named after the park folder, and a row like that is a
|
|
||||||
folder echo, never evidence (the guard in `row_names_the_park`).
|
|
||||||
|
|
||||||
The plan crosses three trees (the library, its park, the movie library), so the
|
The plan crosses three trees (the library, its park, the movie library), so the
|
||||||
executor runs with --root at the share that holds them, on its own journal —
|
executor runs with --root at the share that holds them, on its own journal —
|
||||||
@@ -70,8 +56,7 @@ one journal is one undo:
|
|||||||
|
|
||||||
python3 scripts/media-shows.py --root /mnt/entropy/TVShows \
|
python3 scripts/media-shows.py --root /mnt/entropy/TVShows \
|
||||||
--enum /tmp/tv-enum.tsv --titles /tmp/jellyfin-tv.tsv \
|
--enum /tmp/tv-enum.tsv --titles /tmp/jellyfin-tv.tsv \
|
||||||
--tvdb --tmdb --out /mnt/entropy/media-tv-plan.tsv \
|
--out /mnt/entropy/media-tv-plan.tsv
|
||||||
--park /mnt/entropy/media-park/TVShows
|
|
||||||
python3 scripts/media-apply.py apply --plan /mnt/entropy/media-tv-plan.tsv \
|
python3 scripts/media-apply.py apply --plan /mnt/entropy/media-tv-plan.tsv \
|
||||||
--root /mnt/entropy --journal /mnt/entropy/media-tv-relayout.jsonl
|
--root /mnt/entropy --journal /mnt/entropy/media-tv-relayout.jsonl
|
||||||
"""
|
"""
|
||||||
@@ -95,21 +80,14 @@ ARGS.add_argument("--titles", default="/tmp/jellyfin-tv.tsv",
|
|||||||
help="the provider's numbering (media-census.py jellyfin)")
|
help="the provider's numbering (media-census.py jellyfin)")
|
||||||
ARGS.add_argument("--probed", default="",
|
ARGS.add_argument("--probed", default="",
|
||||||
help="ffprobe census for the movie rule; absent = no durations")
|
help="ffprobe census for the movie rule; absent = no durations")
|
||||||
ARGS.add_argument("--tmdb-cache", default="/tmp/tv-names.json",
|
ARGS.add_argument("--tmdb-cache", default="/tmp/tmdb-cache.json",
|
||||||
help="name answers (TMDb and Tvdb) as ONE input file, so a plan "
|
help="TMDb answers as an input file, so a plan is reproducible")
|
||||||
"is reproducible; keys carry the provider they came from")
|
|
||||||
ARGS.add_argument("--tmdb", action="store_true",
|
ARGS.add_argument("--tmdb", action="store_true",
|
||||||
help="look up cache misses at TMDb (needs TMDB_API_KEY)")
|
help="look up cache misses (needs TMDB_API_KEY)")
|
||||||
ARGS.add_argument("--tvdb", action="store_true",
|
|
||||||
help="look up cache misses at Tvdb (needs the credentials file)")
|
|
||||||
ARGS.add_argument("--tvdb-credentials",
|
|
||||||
default=os.path.expanduser("~/.config/secrets/tvdb"),
|
|
||||||
help="file with an api_key= line (and a pin= line for a "
|
|
||||||
"user-issued key); mode 600, and never the cache")
|
|
||||||
ARGS.add_argument("--movies", default="/mnt/entropy/Movies",
|
ARGS.add_argument("--movies", default="/mnt/entropy/Movies",
|
||||||
help="movie library a mis-filed film moves into")
|
help="movie library a mis-filed film moves into")
|
||||||
ARGS.add_argument("--park", default="",
|
ARGS.add_argument("--park", default="",
|
||||||
help="review pile OUTSIDE the library (default: <root>/_unsorted)")
|
help="review pile (default: <root>/_unsorted)")
|
||||||
ARGS.add_argument("--out", default="/tmp/tv-plan.tsv")
|
ARGS.add_argument("--out", default="/tmp/tv-plan.tsv")
|
||||||
ARGS.add_argument("--exceptions", default="/tmp/tv-exceptions.txt")
|
ARGS.add_argument("--exceptions", default="/tmp/tv-exceptions.txt")
|
||||||
ARGS = ARGS.parse_args()
|
ARGS = ARGS.parse_args()
|
||||||
@@ -127,7 +105,6 @@ try:
|
|||||||
except (OSError, ValueError):
|
except (OSError, ValueError):
|
||||||
CACHE = {}
|
CACHE = {}
|
||||||
API_KEY = os.environ.get("TMDB_API_KEY", "")
|
API_KEY = os.environ.get("TMDB_API_KEY", "")
|
||||||
TMDB = {"tries": 0, "failed": 0}
|
|
||||||
|
|
||||||
VIDEO_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg",
|
VIDEO_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg",
|
||||||
".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".flv", ".vob", ".rm",
|
".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".flv", ".vob", ".rm",
|
||||||
@@ -241,20 +218,6 @@ def leading_index(name):
|
|||||||
return int(m.group(1)) if m else None
|
return int(m.group(1)) if m else None
|
||||||
|
|
||||||
|
|
||||||
def tmdb_request(url):
|
|
||||||
"""One TMDb call, authenticated. The two credential shapes are NOT
|
|
||||||
interchangeable and the difference is a 401 on every single call: a v3 key
|
|
||||||
travels as the `api_key` query parameter, and only a v4 request token is
|
|
||||||
accepted as a Bearer header. A token in argv is world-readable in
|
|
||||||
/proc/<pid>/cmdline — this URL is built inside the process and never reaches
|
|
||||||
argv, so the query form is safe here."""
|
|
||||||
TMDB["tries"] += 1
|
|
||||||
if API_KEY.count(".") == 2: # three segments: a v4 JWT
|
|
||||||
return http_json(url, headers={"Authorization": "Bearer " + API_KEY})
|
|
||||||
return http_json(url + ("&" if "?" in url else "?") + "api_key="
|
|
||||||
+ urllib.parse.quote(API_KEY))
|
|
||||||
|
|
||||||
|
|
||||||
def tmdb(kind, query):
|
def tmdb(kind, query):
|
||||||
"""A provider answer for a query: the cache first, TMDb once, then never again."""
|
"""A provider answer for a query: the cache first, TMDb once, then never again."""
|
||||||
ck = f"{kind}:{query}"
|
ck = f"{kind}:{query}"
|
||||||
@@ -268,155 +231,26 @@ def tmdb(kind, query):
|
|||||||
path = "movie" if kind == "movie" else "tv"
|
path = "movie" if kind == "movie" else "tv"
|
||||||
url = ("https://api.themoviedb.org/3/search/" + path + "?query="
|
url = ("https://api.themoviedb.org/3/search/" + path + "?query="
|
||||||
+ urllib.parse.quote(query) + "&include_adult=false")
|
+ urllib.parse.quote(query) + "&include_adult=false")
|
||||||
|
req = urllib.request.Request(url, headers={
|
||||||
|
# The key rides a header, never the URL: a token in argv is world-readable.
|
||||||
|
"Authorization": "Bearer " + API_KEY,
|
||||||
|
"Accept": "application/json"})
|
||||||
try:
|
try:
|
||||||
found = tmdb_request(url).get("results") or []
|
with urllib.request.urlopen(req, timeout=20) as r:
|
||||||
|
found = json.load(r).get("results") or []
|
||||||
except Exception as e: # noqa: BLE001
|
except Exception as e: # noqa: BLE001
|
||||||
TMDB["failed"] += 1
|
|
||||||
print(f"tmdb {kind} lookup failed for {query!r}: {e}", file=sys.stderr)
|
print(f"tmdb {kind} lookup failed for {query!r}: {e}", file=sys.stderr)
|
||||||
return None
|
return None
|
||||||
if not found:
|
if not found:
|
||||||
CACHE[ck] = None
|
CACHE[ck] = None
|
||||||
return None
|
return None
|
||||||
top = found[0]
|
top = found[0]
|
||||||
# The two search endpoints date their rows differently: movies carry
|
|
||||||
# `release_date`, tv carries `first_air_date`. Reading only one of them leaves
|
|
||||||
# every film answer year-less, and a year-less answer can never be refused by
|
|
||||||
# the year guard below.
|
|
||||||
when = str(top.get("year") or "") or \
|
|
||||||
str(top.get("release_date") or top.get("first_air_date") or "")[:4]
|
|
||||||
CACHE[ck] = {"name": (top.get("title") or top.get("name") or "").strip(),
|
CACHE[ck] = {"name": (top.get("title") or top.get("name") or "").strip(),
|
||||||
"year": when, "id": top.get("id")}
|
"year": str(top.get("year") or "") or (top.get("first_air_date") or "")[:4],
|
||||||
|
"id": top.get("id")}
|
||||||
return CACHE[ck]
|
return CACHE[ck]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- tvdb, the identity
|
|
||||||
# tvdb names the show and the episode: it is the only provider whose strings are
|
|
||||||
# the ones the operator asked for, so its answer replaces a folder-derived Series
|
|
||||||
# name. The login mints a JWT per run from a mode-600 file; neither the key nor
|
|
||||||
# the token reaches argv, the cache, or the plan. A provider that will not answer
|
|
||||||
# never invents a name: every lookup here returns None, and None keeps the
|
|
||||||
# provider's own words and lands in the exceptions.
|
|
||||||
TVDB = {"jwt": None, "dead": False}
|
|
||||||
|
|
||||||
|
|
||||||
def http_json(url, body=None, headers=None):
|
|
||||||
req = urllib.request.Request(
|
|
||||||
url, data=json.dumps(body).encode() if body is not None else None,
|
|
||||||
headers={"Accept": "application/json", "Content-Type": "application/json",
|
|
||||||
**(headers or {})})
|
|
||||||
with urllib.request.urlopen(req, timeout=30) as r:
|
|
||||||
return json.load(r)
|
|
||||||
|
|
||||||
|
|
||||||
def tvdb_login():
|
|
||||||
"""The run's JWT, or None: a project key needs only api_key=, a user-issued
|
|
||||||
key also needs the pin."""
|
|
||||||
if TVDB["jwt"] is not None:
|
|
||||||
return TVDB["jwt"] or None
|
|
||||||
TVDB["jwt"] = ""
|
|
||||||
try:
|
|
||||||
lines = open(ARGS.tvdb_credentials).read().splitlines()
|
|
||||||
except OSError:
|
|
||||||
print(f"tvdb: no credentials at {ARGS.tvdb_credentials}", file=sys.stderr)
|
|
||||||
return None
|
|
||||||
sec = {}
|
|
||||||
for line in lines:
|
|
||||||
line = line.strip()
|
|
||||||
if line and not line.startswith("#") and "=" in line:
|
|
||||||
k, v = line.split("=", 1)
|
|
||||||
sec[k.strip()] = v.strip()
|
|
||||||
if not sec.get("api_key"):
|
|
||||||
print(f"tvdb: no api_key in {ARGS.tvdb_credentials}", file=sys.stderr)
|
|
||||||
return None
|
|
||||||
body = {"apiKey": sec["api_key"]}
|
|
||||||
if sec.get("pin"):
|
|
||||||
body["token"] = sec["pin"]
|
|
||||||
try:
|
|
||||||
TVDB["jwt"] = (http_json("https://api4.thetvdb.com/v4/login", body)
|
|
||||||
.get("data") or {}).get("token") or ""
|
|
||||||
except Exception as e: # noqa: BLE001
|
|
||||||
print(f"tvdb login failed: {e}", file=sys.stderr)
|
|
||||||
return None
|
|
||||||
return TVDB["jwt"] or None
|
|
||||||
|
|
||||||
|
|
||||||
def tvdb(path):
|
|
||||||
"""One Tvdb v4 GET, or None when it cannot be answered."""
|
|
||||||
if not ARGS.tvdb or TVDB["dead"]:
|
|
||||||
return None
|
|
||||||
jwt = tvdb_login()
|
|
||||||
if not jwt:
|
|
||||||
TVDB["dead"] = True
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return http_json("https://api4.thetvdb.com/v4" + path,
|
|
||||||
headers={"Authorization": "Bearer " + jwt}).get("data")
|
|
||||||
except Exception as e: # noqa: BLE001
|
|
||||||
print(f"tvdb {path} failed: {e}", file=sys.stderr)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def tvdb_series(sid):
|
|
||||||
"""{name, year} for a tvdb series id."""
|
|
||||||
ck = f"tvdb:series:{sid}"
|
|
||||||
if ck in CACHE:
|
|
||||||
return CACHE[ck]
|
|
||||||
data = tvdb(f"/series/{sid}")
|
|
||||||
if not data:
|
|
||||||
return None
|
|
||||||
CACHE[ck] = {"name": (data.get("name") or "").strip(),
|
|
||||||
"year": (data.get("firstAired") or "")[:4]}
|
|
||||||
return CACHE[ck]
|
|
||||||
|
|
||||||
|
|
||||||
def tvdb_id_for_tmdb(mid):
|
|
||||||
"""The tvdb id TMDb records for a series id — the bridge for the bulk of this
|
|
||||||
library, which the server matched by TMDb."""
|
|
||||||
ck = f"tmdb:external:{mid}"
|
|
||||||
if ck in CACHE:
|
|
||||||
return CACHE[ck] or ""
|
|
||||||
if not (ARGS.tmdb and API_KEY):
|
|
||||||
return ""
|
|
||||||
try:
|
|
||||||
got = tmdb_request("https://api.themoviedb.org/3/tv/" + mid + "/external_ids")
|
|
||||||
except Exception as e: # noqa: BLE001
|
|
||||||
TMDB["failed"] += 1
|
|
||||||
print(f"tmdb external ids failed for {mid}: {e}", file=sys.stderr)
|
|
||||||
return ""
|
|
||||||
CACHE[ck] = str(got.get("tvdb_id") or "")
|
|
||||||
return CACHE[ck]
|
|
||||||
|
|
||||||
|
|
||||||
def tvdb_search(name):
|
|
||||||
"""The tvdb series a provider name states, or None: only tvdb's own spelling
|
|
||||||
counts, so a near-miss never renames a show."""
|
|
||||||
ck = f"tvdb:search:{name}"
|
|
||||||
if ck in CACHE:
|
|
||||||
return CACHE[ck]
|
|
||||||
for hit in tvdb("/search?query=" + urllib.parse.quote(name) + "&type=series") or []:
|
|
||||||
if eng.norm(hit.get("name") or "") != eng.norm(name):
|
|
||||||
continue
|
|
||||||
CACHE[ck] = {"name": (hit.get("name") or "").strip(),
|
|
||||||
"year": str(hit.get("year") or ""),
|
|
||||||
"id": str(hit.get("tvdb_id") or "")}
|
|
||||||
return CACHE[ck]
|
|
||||||
CACHE[ck] = None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def tvdb_episodes(sid):
|
|
||||||
"""`season/episode` -> title for one series, in the one call tvdb answers."""
|
|
||||||
ck = f"tvdb:episodes:{sid}"
|
|
||||||
if ck in CACHE:
|
|
||||||
return CACHE[ck]
|
|
||||||
data = tvdb(f"/series/{sid}/episodes/default")
|
|
||||||
if not data:
|
|
||||||
return None
|
|
||||||
CACHE[ck] = {f"{e.get('seasonNumber')}/{e.get('number')}":
|
|
||||||
(e.get("name") or "").strip()
|
|
||||||
for e in data.get("episodes") or []}
|
|
||||||
return CACHE[ck]
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------- the inputs
|
# ------------------------------------------------------------------- the inputs
|
||||||
rows = []
|
rows = []
|
||||||
for line in open(ARGS.enum):
|
for line in open(ARGS.enum):
|
||||||
@@ -430,26 +264,6 @@ for line in open(ARGS.enum):
|
|||||||
rows.append({"bytes": int(size), "ext": ext.lower(), "path": p, "base": base})
|
rows.append({"bytes": int(size), "ext": ext.lower(), "path": p, "base": base})
|
||||||
held = {r["path"]: r for r in rows}
|
held = {r["path"]: r for r in rows}
|
||||||
|
|
||||||
def row_names_the_park(path, name):
|
|
||||||
"""The library's own review pile is not a show. When the park sat INSIDE the
|
|
||||||
library the server indexed the whole folder as one series named after it, and
|
|
||||||
a row like that would rename a real show to the pile — so a row whose series is
|
|
||||||
the hidden top-level folder the file itself sits in is dropped. The server also
|
|
||||||
sanitizes the display name, dropping the leading underscore, so `unsorted` and
|
|
||||||
`_unsorted` are the same pile: compare them without their leading punctuation,
|
|
||||||
or the pile survives as a visible series folder."""
|
|
||||||
top = rel(path).split("/")[0]
|
|
||||||
return top.startswith("_") and eng.norm(name.lstrip("_")) == eng.norm(top.lstrip("_"))
|
|
||||||
|
|
||||||
|
|
||||||
def in_review_pile(path):
|
|
||||||
"""A file under the library's hidden top-level folder is review material by
|
|
||||||
that folder's own convention: its filename may MATCH a show a provider names
|
|
||||||
somewhere else, but it may never INVENT one — inventing one from a file's own
|
|
||||||
words is what filled the pile to begin with."""
|
|
||||||
return rel(path).split("/")[0].startswith("_")
|
|
||||||
|
|
||||||
|
|
||||||
titles = {}
|
titles = {}
|
||||||
with open(ARGS.titles) as fh:
|
with open(ARGS.titles) as fh:
|
||||||
cols = fh.readline().rstrip("\n").split("\t")
|
cols = fh.readline().rstrip("\n").split("\t")
|
||||||
@@ -458,7 +272,7 @@ with open(ARGS.titles) as fh:
|
|||||||
f += [""] * (len(cols) - len(f))
|
f += [""] * (len(cols) - len(f))
|
||||||
row = dict(zip(cols, f))
|
row = dict(zip(cols, f))
|
||||||
p = os.path.abspath(row.get("file", ""))
|
p = os.path.abspath(row.get("file", ""))
|
||||||
if p in held and not row_names_the_park(p, row.get("series", "")):
|
if p in held:
|
||||||
titles[p] = row
|
titles[p] = row
|
||||||
|
|
||||||
dur = {}
|
dur = {}
|
||||||
@@ -565,17 +379,14 @@ def series_name(name):
|
|||||||
return clean(re.sub(r"\s{2,}", " ", PACK_IDX_RE.sub("", name)).strip(" ."))
|
return clean(re.sub(r"\s{2,}", " ", PACK_IDX_RE.sub("", name)).strip(" ."))
|
||||||
|
|
||||||
|
|
||||||
def note(name, year="", tmdb_id="", tvdb_id=""):
|
def note(name, year="", tmdb_id=""):
|
||||||
"""Register a series; the first name wins, the first year and ids stick."""
|
"""Register a series; the first name wins, the first year and id stick."""
|
||||||
k = eng.norm(name)
|
k = eng.norm(name)
|
||||||
s = series.setdefault(k, {"name": series_name(name), "year": "", "tmdb": "",
|
s = series.setdefault(k, {"name": series_name(name), "year": "", "tmdb": ""})
|
||||||
"tvdb": ""})
|
|
||||||
if year and not s["year"]:
|
if year and not s["year"]:
|
||||||
s["year"] = str(year)
|
s["year"] = str(year)
|
||||||
if tmdb_id and not s["tmdb"]:
|
if tmdb_id and not s["tmdb"]:
|
||||||
s["tmdb"] = str(tmdb_id)
|
s["tmdb"] = str(tmdb_id)
|
||||||
if tvdb_id and not s["tvdb"]:
|
|
||||||
s["tvdb"] = str(tvdb_id)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -602,9 +413,7 @@ for r in video:
|
|||||||
# every file under SNL 16-20/Season 16..19 carries season 16 episode 20.
|
# every file under SNL 16-20/Season 16..19 carries season 16 episode 20.
|
||||||
owner = (t["series"] if t and t["series"]
|
owner = (t["series"] if t and t["series"]
|
||||||
else series_of_dir(os.path.dirname(p))
|
else series_of_dir(os.path.dirname(p))
|
||||||
or known_in(r["base"])
|
or title_of(r["base"][:marked[2]])[0])
|
||||||
or ("" if in_review_pile(p)
|
|
||||||
else title_of(r["base"][:marked[2]])[0]))
|
|
||||||
if not eng.norm(owner).replace(" ", ""):
|
if not eng.norm(owner).replace(" ", ""):
|
||||||
unknown.append((r, "an episode marker with no series name to hang it on"))
|
unknown.append((r, "an episode marker with no series name to hang it on"))
|
||||||
continue
|
continue
|
||||||
@@ -618,8 +427,7 @@ for r in video:
|
|||||||
continue
|
continue
|
||||||
eps = [ep]
|
eps = [ep]
|
||||||
claimants.append((r, owner, season, eps))
|
claimants.append((r, owner, season, eps))
|
||||||
note(owner, t["year"] if t else "", t["tmdb"] if t else "",
|
note(owner, t["year"] if t else "", t["tmdb"] if t else "")
|
||||||
t["tvdb"] if t else "")
|
|
||||||
continue
|
continue
|
||||||
owner = series_of_dir(os.path.dirname(p))
|
owner = series_of_dir(os.path.dirname(p))
|
||||||
if fs is not None and owner and leading_index(r["base"]):
|
if fs is not None and owner and leading_index(r["base"]):
|
||||||
@@ -642,84 +450,17 @@ for k, s in series.items():
|
|||||||
if got and eng.norm(got["name"]) == k:
|
if got and eng.norm(got["name"]) == k:
|
||||||
s["year"] = got["year"]
|
s["year"] = got["year"]
|
||||||
|
|
||||||
# tvdb is the identity authority. The row's own tvdb id is the door; a TMDb id
|
if ARGS.tmdb:
|
||||||
# crosses through TMDb's external ids; a bare name counts only when tvdb spells
|
|
||||||
# it the same and the years do not disagree. A series nothing resolves keeps the
|
|
||||||
# provider's words AND says so in the exceptions, which is where the operator
|
|
||||||
# finds the folders the plan did not rename to authority.
|
|
||||||
for s in series.values():
|
|
||||||
got = tvdb_series(s["tvdb"]) if s["tvdb"] else None
|
|
||||||
if not got and s["tmdb"]:
|
|
||||||
sid = tvdb_id_for_tmdb(s["tmdb"])
|
|
||||||
if sid:
|
|
||||||
s["tvdb"] = sid
|
|
||||||
got = tvdb_series(sid)
|
|
||||||
if not got:
|
|
||||||
hit = tvdb_search(s["name"])
|
|
||||||
if hit and hit["id"] and (not s["year"] or not hit["year"]
|
|
||||||
or hit["year"] == s["year"]):
|
|
||||||
s["tvdb"] = hit["id"]
|
|
||||||
got = hit
|
|
||||||
if got and got["name"]:
|
|
||||||
s["canon"], s["canon_year"] = got["name"], got["year"]
|
|
||||||
|
|
||||||
# One identity, one folder: the provider's spellings collapse to the shows tvdb
|
|
||||||
# says they are, so a box set split over five release folders is ONE series and
|
|
||||||
# two files claiming one episode MEET inside the duplicate rule below.
|
|
||||||
alias, canon = {}, {}
|
|
||||||
for k, s in series.items():
|
|
||||||
key = f"id:{s['tvdb']}" if s["tvdb"] else k
|
|
||||||
alias[k] = key
|
|
||||||
keep = canon.setdefault(key, s)
|
|
||||||
if keep is not s:
|
|
||||||
for f in ("year", "tvdb", "canon", "canon_year"):
|
|
||||||
if not keep.get(f) and s.get(f):
|
|
||||||
keep[f] = s[f]
|
|
||||||
|
|
||||||
if ARGS.tmdb or ARGS.tvdb:
|
|
||||||
with open(ARGS.tmdb_cache, "w") as fh:
|
with open(ARGS.tmdb_cache, "w") as fh:
|
||||||
json.dump(CACHE, fh, indent=1, sort_keys=True)
|
json.dump(CACHE, fh, indent=1, sort_keys=True)
|
||||||
|
|
||||||
|
|
||||||
def identity(s):
|
|
||||||
"""(name, year) a folder and its files carry: tvdb's whenever it answered,
|
|
||||||
the provider's own words only for a series nothing resolved."""
|
|
||||||
if s.get("canon"):
|
|
||||||
return s["canon"], s["canon_year"] or s["year"]
|
|
||||||
return s["name"], s["year"]
|
|
||||||
|
|
||||||
|
|
||||||
def series_key(name):
|
|
||||||
"""The merged identity a provider spelling belongs to."""
|
|
||||||
return alias.get(eng.norm(name), eng.norm(name))
|
|
||||||
|
|
||||||
|
|
||||||
def label_of(s):
|
|
||||||
"""`<Series> (<year>)`, the form both matchers read. A provider's name can
|
|
||||||
arrive WITH the year already on it — TVDb hands back `Acapulco (2021)` for the
|
|
||||||
reboot — and appending a second one names a folder nothing will match again."""
|
|
||||||
nm, yr = identity(s)
|
|
||||||
nm = clean(nm)
|
|
||||||
if not yr or re.search(r"\(\d{4}\)\s*$", nm):
|
|
||||||
return nm
|
|
||||||
return f"{nm} ({yr})"
|
|
||||||
|
|
||||||
|
|
||||||
def episode_title(r, s, season, ep):
|
|
||||||
"""The title this episode carries: tvdb's for the id the series resolved to,
|
|
||||||
else the title the matched row states, else nothing at all — a dangling ` - `
|
|
||||||
is not a title, and an invented one is worse than none."""
|
|
||||||
if s["tvdb"]:
|
|
||||||
t = ((tvdb_episodes(s["tvdb"]) or {}).get(f"{season}/{ep}") or "").strip()
|
|
||||||
if t:
|
|
||||||
return clean(t)
|
|
||||||
t = ((titles.get(r["path"]) or {}).get("title") or "").strip()
|
|
||||||
return clean(t) if t else ""
|
|
||||||
|
|
||||||
|
|
||||||
def series_dir(name):
|
def series_dir(name):
|
||||||
s = canon.get(series_key(name))
|
s = series.get(eng.norm(name))
|
||||||
return f"{ROOT}/{label_of(s)}" if s else f"{ROOT}/{clean(series_name(name))}"
|
if not s:
|
||||||
|
s = {"name": clean(name), "year": ""}
|
||||||
|
label = f"{s['name']} ({s['year']})" if s["year"] else s["name"]
|
||||||
|
return f"{ROOT}/{clean(label)}"
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------- the plan
|
# ------------------------------------------------------------------- the plan
|
||||||
@@ -759,28 +500,25 @@ def park(r, reason):
|
|||||||
by_ep = collections.defaultdict(list)
|
by_ep = collections.defaultdict(list)
|
||||||
for r, name, season, eps in claimants:
|
for r, name, season, eps in claimants:
|
||||||
for e in eps:
|
for e in eps:
|
||||||
by_ep[(series_key(name), season, e)].append(r)
|
by_ep[(eng.norm(name), season, e)].append(r)
|
||||||
winners, losers = set(), {}
|
winners, losers = set(), {}
|
||||||
for (key, season, e), rs in by_ep.items():
|
for (key, season, e), rs in by_ep.items():
|
||||||
rs.sort(key=lambda r: (-r["bytes"], r["path"]))
|
rs.sort(key=lambda r: (-r["bytes"], r["path"]))
|
||||||
winners.add(rs[0]["path"])
|
winners.add(rs[0]["path"])
|
||||||
for r in rs[1:]:
|
for r in rs[1:]:
|
||||||
losers.setdefault(r["path"],
|
losers.setdefault(r["path"],
|
||||||
f"duplicate of {canon[key]['name']} S{season:02d}E{e:02d}"
|
f"duplicate of {series[key]['name']} S{season:02d}E{e:02d}"
|
||||||
" (the bigger copy keeps the standard name)")
|
" (the bigger copy keeps the standard name)")
|
||||||
for r, name, season, eps in claimants:
|
for r, name, season, eps in claimants:
|
||||||
if r["path"] in losers:
|
if r["path"] in losers:
|
||||||
park(r, losers[r["path"]])
|
park(r, losers[r["path"]])
|
||||||
continue
|
continue
|
||||||
s = canon[series_key(name)]
|
s = series[eng.norm(name)]
|
||||||
stem = f"{label_of(s)} - S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps)
|
label = f" S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps)
|
||||||
title = episode_title(r, s, season, eps[0])
|
|
||||||
if title:
|
|
||||||
stem += f" - {title}"
|
|
||||||
part = PART_RE.search(os.path.splitext(r["base"])[0])
|
part = PART_RE.search(os.path.splitext(r["base"])[0])
|
||||||
if part and len(eps) == 1:
|
tail = f" Part {part.group(1).upper()}" if part and len(eps) == 1 else ""
|
||||||
stem += f" Part {part.group(1).upper()}"
|
book(r, f"{series_dir(name)}/Season {season:02d}", s["name"] + label + tail + r["ext"],
|
||||||
book(r, f"{series_dir(name)}/Season {season:02d}", stem + r["ext"], "tv")
|
"tv")
|
||||||
|
|
||||||
for r, name, kind in extras:
|
for r, name, kind in extras:
|
||||||
book(r, f"{series_dir(name)}/{kind}",
|
book(r, f"{series_dir(name)}/{kind}",
|
||||||
@@ -792,13 +530,8 @@ for r, name, kind in extras:
|
|||||||
for r in films:
|
for r in films:
|
||||||
parsed, year = title_of(r["base"])
|
parsed, year = title_of(r["base"])
|
||||||
got = tmdb("movie", parsed)
|
got = tmdb("movie", parsed)
|
||||||
# The provider's title must be a run of words the FILE itself states. The query
|
words = eng.norm(got["name"]).split() if got else []
|
||||||
# is only what title_of could salvage from the name, and a short one ("russia")
|
if not got or not eng.mentions(parsed.split(), words) \
|
||||||
# lets the top hit be a different work entirely: a 1963 Bond film is not a 1991
|
|
||||||
# documentary episode, and Special Edition I is not Special Edition III.
|
|
||||||
said = eng.norm(os.path.splitext(r["base"])[0]).split()
|
|
||||||
named = eng.norm(got["name"]).split() if got else []
|
|
||||||
if not got or not eng.mentions(named, said) \
|
|
||||||
or (year != "?" and got["year"] and got["year"] != year):
|
or (year != "?" and got["year"] and got["year"] != year):
|
||||||
park(r, "long file with no episode evidence, and no movie provider names it")
|
park(r, "long file with no episode evidence, and no movie provider names it")
|
||||||
continue
|
continue
|
||||||
@@ -922,30 +655,6 @@ for r, reason in unknown:
|
|||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------- output
|
# --------------------------------------------------------------------- output
|
||||||
def provider_broken():
|
|
||||||
"""TMDb was asked for names and did not answer: the plan is not the plan the
|
|
||||||
operator asked for, whatever the move count says."""
|
|
||||||
return bool(ARGS.tmdb and (not API_KEY or TMDB["failed"]))
|
|
||||||
|
|
||||||
|
|
||||||
def provider_note():
|
|
||||||
"""Provider health on the summary line. The 401s are already on stderr, and a
|
|
||||||
terminal that scrolled past them shows a plan that looks finished: the count
|
|
||||||
belongs where it cannot be missed, because a half-resolved plan renames
|
|
||||||
folders with the folder's own junk words."""
|
|
||||||
if not (ARGS.tmdb or ARGS.tvdb):
|
|
||||||
return ""
|
|
||||||
if not ARGS.tmdb:
|
|
||||||
return ", tmdb off (names from the cache and tvdb only)"
|
|
||||||
if not API_KEY:
|
|
||||||
return ", tmdb NOT ASKED (TMDB_API_KEY unset): NOT provider-complete"
|
|
||||||
if not TMDB["tries"]:
|
|
||||||
return ", tmdb cache-only (every name was already answered)"
|
|
||||||
done = TMDB["tries"] - TMDB["failed"]
|
|
||||||
return (f", tmdb {done}/{TMDB['tries']} answered"
|
|
||||||
+ (" — NOT provider-complete" if TMDB["failed"] else ""))
|
|
||||||
|
|
||||||
|
|
||||||
def write_plan():
|
def write_plan():
|
||||||
"""mkdir rows for every destination folder and its whole chain, then the moves.
|
"""mkdir rows for every destination folder and its whole chain, then the moves.
|
||||||
Parents first: the executor creates directories in the order they are listed."""
|
Parents first: the executor creates directories in the order they are listed."""
|
||||||
@@ -967,23 +676,16 @@ def write_plan():
|
|||||||
sized = sum(held[s]["bytes"] for s, _d, _k in moves if s in held)
|
sized = sum(held[s]["bytes"] for s, _d, _k in moves if s in held)
|
||||||
print(f"plan: {len(moves) - len(parked)} moves, {len(parked)} parked"
|
print(f"plan: {len(moves) - len(parked)} moves, {len(parked)} parked"
|
||||||
f" ({sized / 2**40:.2f} TiB touched), {len(keeps)} already placed,"
|
f" ({sized / 2**40:.2f} TiB touched), {len(keeps)} already placed,"
|
||||||
f" {len(dirs)} folders{provider_note()} -> {ARGS.out}")
|
f" {len(dirs)} folders -> {ARGS.out}")
|
||||||
|
|
||||||
|
|
||||||
def write_exceptions():
|
def write_exceptions():
|
||||||
no_year = sorted({s["name"] for s in canon.values() if not s["year"]})
|
no_year = sorted(s["name"] for s in series.values() if not s["year"])
|
||||||
no_tvdb = sorted({identity(s)[0] for s in canon.values() if not s.get("canon")})
|
|
||||||
with open(ARGS.exceptions, "w") as fh:
|
with open(ARGS.exceptions, "w") as fh:
|
||||||
fh.write(f"TV relayout exceptions for {ROOT}\n\n")
|
fh.write(f"TV relayout exceptions for {ROOT}\n\n")
|
||||||
fh.write(f"Everything below is parked under {PARK}, keeping the relative path it\n")
|
fh.write(f"Everything below is parked under {PARK}, keeping the relative path it\n")
|
||||||
fh.write("has today. Nothing is deleted and nothing is overwritten: a parked file\n")
|
fh.write("has today. Nothing is deleted and nothing is overwritten: a parked file\n")
|
||||||
fh.write("is one rename from wherever the plan is pointed next.\n\n")
|
fh.write("is one rename from wherever the plan is pointed next.\n\n")
|
||||||
if provider_broken():
|
|
||||||
fh.write("WARNING: " + provider_note().strip(", ").replace(": NOT", " — NOT")
|
|
||||||
+ "\n Names and episode titles that should have come from TMDb are\n"
|
|
||||||
" missing, so folders keep the metadata provider's words. Fix the\n"
|
|
||||||
" credential and re-plan: the cache keeps every answer that did\n"
|
|
||||||
" arrive, so the re-run costs only the lookups that failed.\n\n")
|
|
||||||
for reason in sorted(reasons, key=lambda r: -len(reasons[r])):
|
for reason in sorted(reasons, key=lambda r: -len(reasons[r])):
|
||||||
rs = reasons[reason]
|
rs = reasons[reason]
|
||||||
fh.write(f"{len(rs):>6} files {reason}\n")
|
fh.write(f"{len(rs):>6} files {reason}\n")
|
||||||
@@ -996,12 +698,6 @@ def write_exceptions():
|
|||||||
fh.write(f"{len(no_year):>6} series carry no year from any provider, so their\n")
|
fh.write(f"{len(no_year):>6} series carry no year from any provider, so their\n")
|
||||||
fh.write(" folders are named without one; Jellyfin still matches them.\n")
|
fh.write(" folders are named without one; Jellyfin still matches them.\n")
|
||||||
fh.write(" " + ", ".join(no_year[:300]) + "\n")
|
fh.write(" " + ", ".join(no_year[:300]) + "\n")
|
||||||
if no_tvdb:
|
|
||||||
fh.write(f"\n{len(no_tvdb):>6} series carry no tvdb identity, so their folders\n")
|
|
||||||
fh.write(" keep the name the metadata provider gave them and no episode\n")
|
|
||||||
fh.write(" title was taken. Nothing was invented: give these a resolvable\n")
|
|
||||||
fh.write(" id (--tvdb) and re-plan; a rename is idempotent.\n")
|
|
||||||
fh.write(" " + ", ".join(no_tvdb[:300]) + "\n")
|
|
||||||
print(f"exceptions -> {ARGS.exceptions}")
|
print(f"exceptions -> {ARGS.exceptions}")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+47
-12
@@ -9,8 +9,13 @@ its destination, and NO row may still name its source — a source row is a scan
|
|||||||
that has not caught up, and it is the one signature of a library pointing at
|
that has not caught up, and it is the one signature of a library pointing at
|
||||||
files that no longer exist.
|
files that no longer exist.
|
||||||
|
|
||||||
Two doors, both read-only, both without a lock while the server runs
|
Two doors, both read-only, both without a lock while the server runs (there is
|
||||||
(`mode=ro&immutable=1`; there is no `sqlite3` binary on these hosts):
|
no `sqlite3` binary on these hosts). Open `mode=ro` PLAIN: both servers keep
|
||||||
|
their libraries in WAL mode, and `immutable=1` — which claims the file never
|
||||||
|
changes — makes sqlite skip the write-ahead log entirely, so every row the
|
||||||
|
server has committed but not yet checkpointed reads as MISSING, and a read
|
||||||
|
beside a running scanner can even fail as "database disk image is malformed".
|
||||||
|
`immutable=1` is only the fallback, for a database whose `-shm` cannot be read.
|
||||||
|
|
||||||
jellyfin `BaseItems.Path`. The container binds the share at `/data`, so the
|
jellyfin `BaseItems.Path`. The container binds the share at `/data`, so the
|
||||||
database holds `/data/<library>/…` where the journal holds
|
database holds `/data/<library>/…` where the journal holds
|
||||||
@@ -22,9 +27,14 @@ Two doors, both read-only, both without a lock while the server runs
|
|||||||
|
|
||||||
Neither server holds a row for everything the journal renames: a `.log`, `.cue`
|
Neither server holds a row for everything the journal renames: a `.log`, `.cue`
|
||||||
or cover image has no row to land in, so its absence is not a fault. The report
|
or cover image has no row to land in, so its absence is not a fault. The report
|
||||||
therefore breaks the missing rows down by extension, and `--ignore-ext` names
|
therefore breaks the missing rows down by extension, and `--ignore-ext` names the
|
||||||
the extensions this server does not index, which are then counted separately and
|
extensions this server does not index, which are then counted separately and kept
|
||||||
kept out of the verdict.
|
out of the verdict. What is left is split by what the filesystem says, because
|
||||||
|
the database alone cannot: a file still at its `src` is `REAL` (this pass's own
|
||||||
|
damage), a file at neither path is `gone` (a later pass or the server renamed it),
|
||||||
|
and a file sitting at its `dst` with no row naming it is simply something the
|
||||||
|
server does not index. A row that still names `src` is none of these: it is the
|
||||||
|
server's own stale pointer — a phantom — and it costs a re-scan, never a restore.
|
||||||
|
|
||||||
Exit codes: 0 every move landed; 1 something is stale or missing (the details are
|
Exit codes: 0 every move landed; 1 something is stale or missing (the details are
|
||||||
printed); 2 the journal or the database is not usable at all.
|
printed); 2 the journal or the database is not usable at all.
|
||||||
@@ -92,10 +102,18 @@ def read_moves(path):
|
|||||||
def server_paths(db, kind):
|
def server_paths(db, kind):
|
||||||
"""Every path the server holds, or a refusal that names what is missing."""
|
"""Every path the server holds, or a refusal that names what is missing."""
|
||||||
spec = KINDS[kind]
|
spec = KINDS[kind]
|
||||||
try:
|
con = None
|
||||||
con = sqlite3.connect(f"file:{db}?mode=ro&immutable=1", uri=True)
|
err = None
|
||||||
except sqlite3.Error as e:
|
# Plain ro reads the WAL the server commits through; immutable is the last
|
||||||
die(f"cannot open {db}: {e}")
|
# resort, because it cannot see a row the server has not checkpointed.
|
||||||
|
for opt in ("mode=ro", "mode=ro&immutable=1"):
|
||||||
|
try:
|
||||||
|
con = sqlite3.connect(f"file:{db}?{opt}", uri=True)
|
||||||
|
break
|
||||||
|
except sqlite3.Error as e:
|
||||||
|
err = e
|
||||||
|
if con is None:
|
||||||
|
die(f"cannot open {db}: {err}")
|
||||||
with con:
|
with con:
|
||||||
have = {r[1] for r in con.execute(f"pragma table_info({spec['table']})")}
|
have = {r[1] for r in con.execute(f"pragma table_info({spec['table']})")}
|
||||||
if not spec["columns"] <= have:
|
if not spec["columns"] <= have:
|
||||||
@@ -161,20 +179,37 @@ def main():
|
|||||||
if at_src:
|
if at_src:
|
||||||
stale.append(src)
|
stale.append(src)
|
||||||
if not at_dst:
|
if not at_dst:
|
||||||
missing.append(dst)
|
missing.append((src, dst))
|
||||||
|
|
||||||
|
# The database cannot tell a lost file from one it never indexes: both read
|
||||||
|
# as "no row at the destination". Stat both paths, so the report separates
|
||||||
|
# the classes the runbook names — `REAL` is the only one that is this pass's
|
||||||
|
# own damage, and `gone` is a later pass or the server's own rename.
|
||||||
|
on_disk, real, gone = [], [], []
|
||||||
|
for src, dst in missing:
|
||||||
|
if os.path.exists(dst):
|
||||||
|
on_disk.append(dst)
|
||||||
|
elif os.path.exists(src):
|
||||||
|
real.append(src)
|
||||||
|
else:
|
||||||
|
gone.append(dst)
|
||||||
|
|
||||||
print(f"{args.db_kind}: {len(moves)} moves checked")
|
print(f"{args.db_kind}: {len(moves)} moves checked")
|
||||||
print(f"landed: {landed}")
|
print(f"landed: {landed}")
|
||||||
print(f"still at src: {len(stale)}")
|
print(f"still at src: {len(stale)}")
|
||||||
print(f"missing from the DB: {len(missing)}")
|
print(f"missing from the DB: {len(missing)}")
|
||||||
|
print(f" on disk at dst, no row names it: {len(on_disk)}")
|
||||||
|
print(f" REAL — file still at src: {len(real)}")
|
||||||
|
print(f" gone — at neither path: {len(gone)}")
|
||||||
if ignored:
|
if ignored:
|
||||||
print(f"not indexed (extension ignored): {ignored}")
|
print(f"not indexed (extension ignored): {ignored}")
|
||||||
if missing:
|
if missing:
|
||||||
exts = Counter(os.path.splitext(d)[1].lower() or "(no extension)"
|
exts = Counter(os.path.splitext(d)[1].lower() or "(no extension)"
|
||||||
for d in missing)
|
for _, d in missing)
|
||||||
print("missing by extension: "
|
print("missing by extension: "
|
||||||
+ ", ".join(f"{e} {n}" for e, n in sorted(exts.items())))
|
+ ", ".join(f"{e} {n}" for e, n in sorted(exts.items())))
|
||||||
for label, items in (("stale", stale), ("missing", missing)):
|
for label, items in (("real", real), ("gone", gone), ("missing", on_disk),
|
||||||
|
("stale", stale)):
|
||||||
for path in items[:args.show]:
|
for path in items[:args.show]:
|
||||||
print(f" {label}: {path}")
|
print(f" {label}: {path}")
|
||||||
if len(items) > args.show:
|
if len(items) > args.show:
|
||||||
|
|||||||
@@ -48,14 +48,6 @@ Use the `Authorization: token` header for everything.
|
|||||||
- **Read an issue**: `curl -s -K "$cfg" $API/issues/<number>` (fields: `number`, `title`, `body`, `labels[].name`, `state`)
|
- **Read an issue**: `curl -s -K "$cfg" $API/issues/<number>` (fields: `number`, `title`, `body`, `labels[].name`, `state`)
|
||||||
- **List issues**: `curl -s -K "$cfg" "$API/issues?state=open&limit=50"` — add `&labels=<id>` to filter
|
- **List issues**: `curl -s -K "$cfg" "$API/issues?state=open&limit=50"` — add `&labels=<id>` to filter
|
||||||
- **Comment**: `curl -s -K "$cfg" -X POST -d "$(jq -n --rawfile b "$BODY" '{body:$b}')" $API/issues/<number>/comments`
|
- **Comment**: `curl -s -K "$cfg" -X POST -d "$(jq -n --rawfile b "$BODY" '{body:$b}')" $API/issues/<number>/comments`
|
||||||
- **Comment on a PR**: the conversation thread is `POST $API/issues/<PR-number>/comments`
|
|
||||||
(201, and the answer's `html_url` points at `pulls/<n>`; Gitea's issue index is
|
|
||||||
unified, so the PR's number is the same number). `$API/pulls/<n>/comments` is the
|
|
||||||
**review**-comment store and answers `405` to a plain body.
|
|
||||||
- **A body built by `jq --rawfile b /dev/stdin` inside `$(…)` is empty**: the command
|
|
||||||
substitution does not inherit the heredoc, so Gitea receives `{}` and answers 422 —
|
|
||||||
and `jq -r '.id'` on the error prints nothing, which reads as success. Write the
|
|
||||||
body to a file and pass `--rawfile` that file.
|
|
||||||
- **Apply labels**: `POST $API/issues/<number>/labels` with `{"labels":[<label-ID numbers>]}` — Gitea wants numeric label **IDs**, not names; resolve them from `GET $API/labels` (see [[Triage Labels]] for which labels exist)
|
- **Apply labels**: `POST $API/issues/<number>/labels` with `{"labels":[<label-ID numbers>]}` — Gitea wants numeric label **IDs**, not names; resolve them from `GET $API/labels` (see [[Triage Labels]] for which labels exist)
|
||||||
- **Remove a label**: `DELETE $API/issues/<number>/labels/<label-id>`
|
- **Remove a label**: `DELETE $API/issues/<number>/labels/<label-id>`
|
||||||
- **Close**: `PATCH $API/issues/<number>` with `{"state":"closed"}`
|
- **Close**: `PATCH $API/issues/<number>` with `{"state":"closed"}`
|
||||||
|
|||||||
@@ -182,48 +182,6 @@ scripts/jellyfin-providers.sh verify # plugin + key LENGTHS, never key bytes
|
|||||||
2. Set metadata/language preferences (`LANG=en_US.UTF-8` pinned in the unit).
|
2. Set metadata/language preferences (`LANG=en_US.UTF-8` pinned in the unit).
|
||||||
3. Configure clients to use `http://192.168.0.235:8096` or the guarded vhost.
|
3. Configure clients to use `http://192.168.0.235:8096` or the guarded vhost.
|
||||||
|
|
||||||
## Triggering a scan
|
|
||||||
|
|
||||||
The scan key lives in `~/.config/secrets/jellyfin` (mode 600; the same value is
|
|
||||||
`JELLYFIN_API_KEY` in the ignored `dankserver/.env`). It rides a mode-600 curl
|
|
||||||
config, never a header argument, in the `MediaBrowser` form: 12.4 authenticates
|
|
||||||
only `Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token`
|
|
||||||
for every key, valid or not. Before believing a 401, prove the key against the
|
|
||||||
server's own table — `select count(*) from ApiKeys where AccessToken = ?` on
|
|
||||||
`jellyfin.db` opened `mode=ro&immutable=1`. The backslashes below matter:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
umask 077
|
|
||||||
{ printf 'header = "Authorization: MediaBrowser Token=\\"%s\\""\n' \
|
|
||||||
"$(cat ~/.config/secrets/jellyfin)"
|
|
||||||
echo 'url = "http://127.0.0.1:8096/Library/Refresh"'
|
|
||||||
echo 'request = "POST"'; } >/tmp/jf.hdr
|
|
||||||
curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n"
|
|
||||||
shred -u /tmp/jf.hdr
|
|
||||||
```
|
|
||||||
|
|
||||||
`204` is acceptance; anything else queued no scan, so re-issue the identical
|
|
||||||
request instead of diagnosing. A `401` from a key that answers `200` on a
|
|
||||||
hand-typed request is this block's quoting, not the server: bash `printf`
|
|
||||||
collapses `\"` to `"`, curl ends the quoted value at that quote, and Jellyfin
|
|
||||||
receives `Authorization: MediaBrowser Token=` with nothing after it (`\\"` answers
|
|
||||||
204, `\"` answers 401 — both run on this host).
|
|
||||||
|
|
||||||
One accepted refresh is the whole procedure. A second one while a scan runs moves
|
|
||||||
`Scan Media Library` to `Cancelling`, and once the runner holds `Cancelling` every
|
|
||||||
later `204` is accepted and then discarded — the status code stops meaning
|
|
||||||
anything. The fix is `systemctl --user restart jellyfin.service` (never
|
|
||||||
`deploy.sh`: that runs the never-tested metadata-view cutover), one POST after
|
|
||||||
health answers, and then believe `ScheduledTasks` State at 130 % CPU, not the
|
|
||||||
response code.
|
|
||||||
|
|
||||||
**A scan that must see a new folder has to be recursive.** `POST /Library/Refresh`
|
|
||||||
is the incremental pass, and inotify does not cross a rootless podman bind, so
|
|
||||||
folders created on the host stay invisible to it. The move is
|
|
||||||
`POST /Items/{id}/Refresh?Recursive=true` (204) per library, the id from
|
|
||||||
`GET /Library/VirtualFolders` — its field is `ItemId`, not `Id` — and that same
|
|
||||||
record's `RefreshStatus` is the progress read.
|
|
||||||
|
|
||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
- [[Plex]] — co-resident media server on the same volume
|
- [[Plex]] — co-resident media server on the same volume
|
||||||
|
|||||||
@@ -1,74 +0,0 @@
|
|||||||
---
|
|
||||||
type: reference
|
|
||||||
title: Media Audiobook Relayout
|
|
||||||
description: "Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals."
|
|
||||||
tags: [dankserver, media-server, plex, media-relayout]
|
|
||||||
timestamp: 2026-10-31T00:00:00Z
|
|
||||||
---
|
|
||||||
|
|
||||||
# Media Audiobook Relayout
|
|
||||||
|
|
||||||
`/mnt/entropy/Audiobooks` is the one library here that [[Jellyfin]] does not read
|
|
||||||
at all — its music scanner has no audiobook concept — while [[Plex]] indexes all
|
|
||||||
of it as Artist → Album → Track: the album is the BOOK, that book's single artist
|
|
||||||
is the NARRATOR, both matched out of the files' own tags. Today one book is split
|
|
||||||
across `… [Part 1]`, `… [Part 2]` folders three or four deep inside a collection
|
|
||||||
pack, so a scan serves three quarters of a novel; `scripts/media-audiobooks.py`
|
|
||||||
re-files the tree so the property both servers' book scanners assume is true —
|
|
||||||
|
|
||||||
Audiobooks/<Narrator>/<Book> (<year>)/<the file keeps its own name>
|
|
||||||
|
|
||||||
The shared gates, the executor, the journal law and the rollback live in
|
|
||||||
[[Media Relayout]]; this page is the audiobook arm only.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. The authority: Plex's chapter rows for this root (the DB opens read-only).
|
|
||||||
python3 scripts/media-census.py audiobooks --root /mnt/entropy/Audiobooks \
|
|
||||||
--out /tmp/ab_rows.tsv
|
|
||||||
# 2. What is on disk (stat only).
|
|
||||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Audiobooks \
|
|
||||||
--out /tmp/ab_enum.tsv
|
|
||||||
# 3. The plan.
|
|
||||||
python3 scripts/media-audiobooks.py --root /mnt/entropy/Audiobooks \
|
|
||||||
--enum /tmp/ab_enum.tsv --rows /tmp/ab_rows.tsv --out /tmp/ab_plan.tsv \
|
|
||||||
--exceptions /tmp/ab_exceptions.txt
|
|
||||||
# 4. Read the exceptions file FIRST. Then the same executor, journal and rollback
|
|
||||||
# as [[Media Relayout]] shows, on its own journal, and refresh the Plex section
|
|
||||||
# as in step 5 of the movie Procedure — from inside the container, never over
|
|
||||||
# the host port.
|
|
||||||
scripts/media-apply.py apply --plan /tmp/ab_plan.tsv --root /mnt/entropy \
|
|
||||||
--journal /mnt/entropy/media-relayout-audiobooks.jsonl --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
The group is the identity Plex asserts in its own database — **(narrator, book,
|
|
||||||
year)** — so one book arriving in four part folders becomes one folder, and a
|
|
||||||
1965 and a 2020 recording that share a title stay two books.
|
|
||||||
|
|
||||||
Two laws are forced by what the rows actually hold:
|
|
||||||
|
|
||||||
+- **A chapter is never renumbered.** Most books contain chapters whose index Plex
|
|
||||||
does not know, so a number written into a name would be invented — and it would
|
|
||||||
buy nothing, because both servers order playback by the embedded track tag, not
|
|
||||||
by the file name. A wrong number in a name is the number a human later trusts.
|
|
||||||
Fix order in the tags: a Plex scan re-reads tags and never renames a file.
|
|
||||||
+- **The shelf is the narrator, not the author.** Every book has exactly one artist
|
|
||||||
row and it is the narrator; the author survives only inside some album titles
|
|
||||||
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving it would
|
|
||||||
mean parsing titles where a ripper wrote one and inventing it everywhere else.
|
|
||||||
The author stays visible in the folder name Plex itself shows.
|
|
||||||
|
|
||||||
What it refuses, the music arm's list with one addition: a video container, audio
|
|
||||||
no chapter row names, and a loose file in a folder holding two books all park under
|
|
||||||
`_unsorted` with the path they have today — and so does a chapter whose row names
|
|
||||||
**no book or no narrator**: a placeholder (`[Unknown Album]`, `[Unknown Artist]`)
|
|
||||||
that unrelated books share, or a chapter number mistagged as the artist (`0.4`).
|
|
||||||
`Various Artists` is not a placeholder — that is a multi-cast read.
|
|
||||||
|
|
||||||
The reviewed plan lives at `/mnt/entropy/media-relayout-audiobooks-plan.tsv`; it
|
|
||||||
applies on its own journal, behind the music writer, and then refreshes the Plex
|
|
||||||
section. Verify it with `--db-kind plex` — Jellyfin holds no rows here, so a
|
|
||||||
Jellyfin pass proves nothing either way.
|
|
||||||
|
|
||||||
## See Also
|
|
||||||
|
|
||||||
[[Media Relayout]], [[Media Music Relayout]], [[Plex]]
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
---
|
|
||||||
type: reference
|
|
||||||
title: Media Music Relayout
|
|
||||||
description: "Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals."
|
|
||||||
tags: [dankserver, media-server, jellyfin, media-relayout]
|
|
||||||
timestamp: 2026-10-31T00:00:00Z
|
|
||||||
---
|
|
||||||
|
|
||||||
# Media Music Relayout
|
|
||||||
|
|
||||||
`/mnt/entropy/Music` is rips and discography packs, and Jellyfin has matched and
|
|
||||||
tagged almost all of it. `scripts/media-music.py` re-files it into the shape
|
|
||||||
Jellyfin's music scanner reads —
|
|
||||||
|
|
||||||
Music/<album artist>/<Album> (<year>)/NN - <Title>.ext
|
|
||||||
|
|
||||||
— and the authority is the tag row of the track, never the name the ripper gave
|
|
||||||
the file. The shared gates, the executor, the journal law and the rollback live in
|
|
||||||
[[Media Relayout]]; this page is the music arm only.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. The authority: Jellyfin's audio rows, one per track, with the album and
|
|
||||||
# artist ids its own database carries (the DB opens read-only).
|
|
||||||
python3 scripts/media-census.py music --root /mnt/entropy/Music \
|
|
||||||
--out /tmp/music_rows.tsv
|
|
||||||
# 2. What is on disk (stat only).
|
|
||||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Music \
|
|
||||||
--out /tmp/music_enum.tsv
|
|
||||||
# 3. The plan.
|
|
||||||
python3 scripts/media-music.py --root /mnt/entropy/Music --enum /tmp/music_enum.tsv \
|
|
||||||
--rows /tmp/music_rows.tsv --out /tmp/music_plan.tsv \
|
|
||||||
--exceptions /tmp/music_exceptions.txt
|
|
||||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Then the same
|
|
||||||
# executor, journal and rollback as [[Media Relayout]] shows, on its own journal.
|
|
||||||
scripts/media-apply.py apply --plan /tmp/music_plan.tsv --root /mnt/entropy \
|
|
||||||
--journal /mnt/entropy/media-relayout-music.jsonl --apply
|
|
||||||
```
|
|
||||||
|
|
||||||
The group is what the tags assert, not where the files sit: **(album-artist tag,
|
|
||||||
album, year)**. One album arrives as three disc folders and as two copies of one
|
|
||||||
rip, and one MusicBrainz release group is one album. A compilation is one album
|
|
||||||
whose tracks name many performers, so it files once under `Various Artists`:
|
|
||||||
keying on the performer would file one album once per artist. Two different
|
|
||||||
album-artist tags on one album name are two albums, which is what Jellyfin's two
|
|
||||||
album items say.
|
|
||||||
|
|
||||||
What it refuses:
|
|
||||||
|
|
||||||
+- **A sidecar is placed by the audio around it, never by its own name.** A
|
|
||||||
`.lrc`, `.txt` or picture that shares a track's stem follows that track and
|
|
||||||
takes the name the track now has, or Jellyfin cannot find it; anything else
|
|
||||||
goes to the album its own folder was placed in, or to the folder whose discs
|
|
||||||
sit one level below it. A folder whose audio belongs to two albums parks its
|
|
||||||
sidecar rather than guessing.
|
|
||||||
+- **An image album keeps its name.** A folder holding one audio file and a
|
|
||||||
`.cue` is a whole album in one file; numbering it `01 - …` tells Jellyfin to
|
|
||||||
index it as one track.
|
|
||||||
+- **A video container parks**, because Jellyfin will not index it as music and a
|
|
||||||
rename to `.m4a` would claim it is; **audio no provider row names parks** too,
|
|
||||||
because unmatched audio is a tagging job, not a renaming job.
|
|
||||||
|
|
||||||
ONE writer at a time on this array: two rename passes race, and neither journal
|
|
||||||
tells. Verify the run against Jellyfin's own tables — see *Proving the move
|
|
||||||
landed* in [[Media Relayout]].
|
|
||||||
|
|
||||||
## See Also
|
|
||||||
|
|
||||||
[[Media Relayout]], [[Jellyfin]], [[Media Audiobook Relayout]]
|
|
||||||
@@ -47,10 +47,8 @@ $C plex --root /mnt/entropy/Movies --out /tmp/plex-movies.tsv
|
|||||||
```
|
```
|
||||||
|
|
||||||
`plex` reads the Plex metadata cache (`--db` defaults to the path the
|
`plex` reads the Plex metadata cache (`--db` defaults to the path the
|
||||||
[[Plex]] container bind-mounts at `/var/lib/plexmediaserver`) with plain
|
[[Plex]] container bind-mounts at `/var/lib/plexmediaserver`) with
|
||||||
`mode=ro`: a live server is never locked or written, and plain ro is also what
|
`mode=ro&immutable=1`: a live server is never locked or written.
|
||||||
makes a scan's uncheckpointed rows visible. The same rule as
|
|
||||||
`scripts/media-verify.py` — see *Proving the move landed*.
|
|
||||||
|
|
||||||
Then plan — this reads only the three TSVs, so it costs seconds and can run
|
Then plan — this reads only the three TSVs, so it costs seconds and can run
|
||||||
anywhere, including a laptop:
|
anywhere, including a laptop:
|
||||||
@@ -81,12 +79,41 @@ An apply also leaves the release folders it emptied standing where they were —
|
|||||||
hundreds of them at the library root — which is what `sweep` removes, dry-run
|
hundreds of them at the library root — which is what `sweep` removes, dry-run
|
||||||
first, from the same journal.
|
first, from the same journal.
|
||||||
|
|
||||||
After it lands, both servers need a scan or the change is invisible. [[Plex]] and
|
After it lands, both servers need a scan or the change is invisible.
|
||||||
[[Jellyfin]] own the exact requests; two laws hold at this step: `204` is
|
|
||||||
acceptance and anything else queued nothing — re-issue the identical request
|
```bash
|
||||||
rather than diagnosing — and ONE accepted refresh is the whole procedure, because
|
# Run this from the host shell through `podman exec`, never from a neighbour
|
||||||
a second while a scan runs moves `Scan Media Library` to `Cancelling` and every
|
# container: Plex's HTTPD closes without a single byte on any request that
|
||||||
later `204` is accepted and then discarded.
|
# arrives from another container's netns (10.89.x.x reads as "empty reply", not
|
||||||
|
# as a 401). The token rides stdin, never argv, where any process on the host
|
||||||
|
# could read it out of /proc/<pid>/cmdline.
|
||||||
|
umask 077
|
||||||
|
podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \
|
||||||
|
'/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok
|
||||||
|
podman exec -i plex sh -c 'read -r T
|
||||||
|
for k in $(curl -s -H "X-Plex-Token: $T" -H "Accept: application/json" \
|
||||||
|
http://127.0.0.1:32400/library/sections | grep -o "\"key\":\"[0-9]*\"" | cut -d\" -f4); do
|
||||||
|
curl -s -o /dev/null -w "plex section $k: %{http_code}\n" -X POST \
|
||||||
|
-H "X-Plex-Token: $T" "http://127.0.0.1:32400/library/sections/$k/refresh"
|
||||||
|
done' </tmp/px.tok
|
||||||
|
```
|
||||||
|
|
||||||
|
[[Jellyfin]]'s scan key lives in `~/.config/secrets/jellyfin` (mode 600; the
|
||||||
|
same value is `JELLYFIN_API_KEY` in the ignored `dankserver/.env`). It rides a
|
||||||
|
mode-600 curl config, never a header argument, in the `MediaBrowser` form —
|
||||||
|
Jellyfin 12.4 refuses a bare `X-Emby-Token`, and the backslashes below matter:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
umask 077
|
||||||
|
{ printf 'header = "Authorization: MediaBrowser Token=\\"%s\\""\n' \
|
||||||
|
"$(cat ~/.config/secrets/jellyfin)"
|
||||||
|
echo 'url = "http://127.0.0.1:8096/Library/Refresh"'
|
||||||
|
echo 'request = "POST"'; } >/tmp/jf.hdr
|
||||||
|
curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n"
|
||||||
|
shred -u /tmp/jf.hdr
|
||||||
|
```
|
||||||
|
|
||||||
|
`204` is acceptance; anything else queued no scan. A `401` from a key that answers `200` on a hand-typed request is this block's quoting, not the server — bash `printf` collapses `\"` to `"`, curl ends the quoted value at that quote, and Jellyfin receives `Authorization: MediaBrowser Token=` with nothing after it (`\\"` answers 204, `\"` answers 401, both run on this host). One accepted refresh is the whole procedure: a second one while a scan runs moves `Scan Media Library` to `Cancelling`, and once the runner holds `Cancelling` every later `204` is accepted and then dropped — the fix is `systemctl --user restart jellyfin.service` (never `deploy.sh`: that runs the metadata-view cutover), one POST after health answers, and then believe `ScheduledTasks` State `Running`, not the status code.
|
||||||
|
|
||||||
### Proving the move landed
|
### Proving the move landed
|
||||||
|
|
||||||
@@ -96,30 +123,14 @@ replayed against both servers' own tables: every `move` row has a row at its
|
|||||||
replay: it opens each database read-only (there is no `sqlite3` binary on these
|
replay: it opens each database read-only (there is no `sqlite3` binary on these
|
||||||
hosts or inside either container) and exits non-zero while a row is stale.
|
hosts or inside either container) and exits non-zero while a row is stale.
|
||||||
|
|
||||||
Both libraries live in WAL mode, so the tool opens them plain `mode=ro`:
|
|
||||||
`immutable=1` promises sqlite the file never changes, which skips the
|
|
||||||
write-ahead log — a scanner's own uncheckpointed rows then read as missing, and
|
|
||||||
a read beside a running scan can fail as "database disk image is malformed".
|
|
||||||
`immutable` stays only the fallback for a database whose `-shm` cannot be read.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
scripts/media-verify.py --journal /mnt/entropy/media-relayout-music.jsonl \
|
scripts/media-verify.py --journal /mnt/entropy/media-relayout-music.jsonl \
|
||||||
--db-kind jellyfin --ignore-ext jpg,jpeg,png,log,cue,toc,nfo,txt,m3u,accurip
|
--db-kind jellyfin --ignore-ext jpg,jpeg,png,log,cue,toc,nfo,txt,m3u,accurip
|
||||||
```
|
```
|
||||||
|
|
||||||
"No row at the destination" is not one verdict but four. **never indexed**: the
|
Ignore only what a server cannot index. A missing `.flac` or `.mkv` is the
|
||||||
server indexes no file of that extension (its own extension census says so) — a
|
fault; a row still naming its `src` means the scan has not followed the rename
|
||||||
sidecar, not a lost film. **gone**: the file sits at neither src nor dst, because
|
yet, so re-run the check after the scan, never before.
|
||||||
a later pass or the server's own extension sweep renamed it. **phantom**: a row
|
|
||||||
still naming src while the file has left src. **REAL**: no row at dst *and* the
|
|
||||||
file still at src — the only class that is this relayout's own damage. Phantoms
|
|
||||||
are normal for both servers (Plex carries thousands that predate any relayout);
|
|
||||||
re-scan to clear them and never read them as loss.
|
|
||||||
|
|
||||||
The owning server decides whether a pass can convict anything: Plex's roots are
|
|
||||||
its `sections` rows joined to their location rows (`Movies`, `TVShows`, `Music`,
|
|
||||||
`Audiobooks`, `LiveRecording`) and Jellyfin holds no audiobook library at all, so
|
|
||||||
the audiobook journal verifies with `--db-kind plex` or it proves nothing.
|
|
||||||
|
|
||||||
Three doors that look right and are not: `GET /search?query=` matches metadata
|
Three doors that look right and are not: `GET /search?query=` matches metadata
|
||||||
titles, never file paths; `?path=` on `/library/sections/N/all` makes Plex's
|
titles, never file paths; `?path=` on `/library/sections/N/all` makes Plex's
|
||||||
@@ -156,15 +167,16 @@ from the box-set folder. What it refuses it prints and leaves alone:
|
|||||||
- a destination two files both claim: the executor refuses the whole plan
|
- a destination two files both claim: the executor refuses the whole plan
|
||||||
before its first rename, and only Frank decides which copy is the keeper.
|
before its first rename, and only Frank decides which copy is the keeper.
|
||||||
- an episode the destination library already holds at the same byte count. The
|
- an episode the destination library already holds at the same byte count. The
|
||||||
movie share once carried a second copy of a whole box set — same episode list,
|
movie share carried a second copy of the whole Monk box set — 124 episodes
|
||||||
every byte count identical. The mover leaves such a series in place and says
|
both sides, the same episode list, every byte count identical, 31.5 GiB each
|
||||||
so; a duplicate is deleted or kept by Frank, never copied into one library by
|
way, and head+tail blake2b identical on a 6-episode sample. The mover leaves
|
||||||
a tool.
|
such a series in place and says so; a duplicate is deleted or kept by Frank,
|
||||||
|
never copied into one library by a tool.
|
||||||
|
|
||||||
A sidecar that shares an episode's own name (`.srt`, `.nfo`) moves with it.
|
A sidecar that shares an episode's own name (`.srt`, `.nfo`) moves with it.
|
||||||
|
|
||||||
Below the threshold the mover prints and leaves, so a lone downloaded episode of
|
Below the threshold the mover prints and leaves, so a lone downloaded episode of
|
||||||
a real show stays in the movie library on purpose.
|
a real show stays in the movie library on purpose — six did on 2026-10-30.
|
||||||
Filing one by hand is the same executor on a hand-written plan and the same
|
Filing one by hand is the same executor on a hand-written plan and the same
|
||||||
journal; the trap is the series year. Read it from the file's own name or the
|
journal; the trap is the series year. Read it from the file's own name or the
|
||||||
folder it owns, never from the title whose bonus folder it happens to be parked
|
folder it owns, never from the title whose bonus folder it happens to be parked
|
||||||
@@ -175,16 +187,12 @@ show. A name whose marker is `Part 2` is a film and stays put.
|
|||||||
|
|
||||||
`/mnt/entropy/TVShows` is scene packs rather than a library, and Jellyfin has
|
`/mnt/entropy/TVShows` is scene packs rather than a library, and Jellyfin has
|
||||||
already matched and numbered most of it. `scripts/media-shows.py` re-files that
|
already matched and numbered most of it. `scripts/media-shows.py` re-files that
|
||||||
library into the shape Jellyfin reads, with **tvdb as the name authority**:
|
library into the same shape, and it never guesses: every series name and every
|
||||||
|
episode number it writes traces to an authority, and what has none is parked for
|
||||||
<Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.ext
|
a human to read.
|
||||||
|
|
||||||
It never guesses: every series name, episode number and title it writes traces to
|
|
||||||
an authority, and what has none is parked for a human to read.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 1. The authority: Jellyfin's own matched rows — each carries the series' TMDb
|
# 1. The authority: Jellyfin's own matched rows (the DB opens read-only).
|
||||||
# and Tvdb ids and the title the row holds (the DB opens read-only).
|
|
||||||
python3 scripts/media-census.py jellyfin --root /mnt/entropy/TVShows \
|
python3 scripts/media-census.py jellyfin --root /mnt/entropy/TVShows \
|
||||||
--out /tmp/tv_titles.tsv
|
--out /tmp/tv_titles.tsv
|
||||||
# 2. What is on disk (stat only), and the runtimes the film arm needs.
|
# 2. What is on disk (stat only), and the runtimes the film arm needs.
|
||||||
@@ -192,33 +200,16 @@ python3 scripts/media-census.py enumerate --root /mnt/entropy/TVShows \
|
|||||||
--out /tmp/tv_enum.tsv
|
--out /tmp/tv_enum.tsv
|
||||||
python3 scripts/media-census.py probe --root /mnt/entropy/TVShows \
|
python3 scripts/media-census.py probe --root /mnt/entropy/TVShows \
|
||||||
--enum /tmp/tv_enum.tsv --jobs 6 --out /tmp/tv_probed.tsv
|
--enum /tmp/tv_enum.tsv --jobs 6 --out /tmp/tv_probed.tsv
|
||||||
# The probe is the slow step: ffprobe reads both ends of every file. So re-use
|
# The probe is the slow step: ffprobe reads both ends of every file.
|
||||||
# the last one — a duration belongs to the file, not to its name: re-key the
|
# 3. The plan. --tmdb needs TMDB_API_KEY; without it a series no provider dated
|
||||||
# previous census through that pass's own journal (`src` -> `dst`) instead of
|
# is simply named without a year, which Jellyfin accepts.
|
||||||
# seeking the array again. It is also the only column the film arm reads, and
|
|
||||||
# only for a file with no marker and no provider row (4,794 of 58,551 today,
|
|
||||||
# all of them covered by a re-key that costs seconds, not an hour).
|
|
||||||
# 3. The plan. --tmdb needs TMDB_API_KEY — the v3 key in ~/.config/secrets/tmdb,
|
|
||||||
# which travels as the `api_key` query parameter. Only a v4 request token is
|
|
||||||
# accepted as a Bearer header, and the wrong shape answers 401 on EVERY call,
|
|
||||||
# which the summary line flags as NOT provider-complete. --tvdb reads its
|
|
||||||
# `api_key=` line from ~/.config/secrets/tvdb, never argv. Both write the SAME
|
|
||||||
# cache file, so the plan is reproducible offline afterwards: run it once to
|
|
||||||
# fill the cache, and quote the second run.
|
|
||||||
# Paths are only strings to a dry run, so this step can run on the workstation
|
|
||||||
# against scp'd TSVs — which is where the two provider secrets live.
|
|
||||||
python3 scripts/media-shows.py --root /mnt/entropy/TVShows --enum /tmp/tv_enum.tsv \
|
python3 scripts/media-shows.py --root /mnt/entropy/TVShows --enum /tmp/tv_enum.tsv \
|
||||||
--titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv \
|
--titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv --out /mnt/entropy/media-relayout-shows-plan.tsv \
|
||||||
--tmdb-cache /mnt/entropy/media-tv-names.json --tmdb --tvdb \
|
--exceptions /mnt/entropy/media-relayout-shows-exceptions.txt
|
||||||
--park /mnt/entropy/media-park/TVShows \
|
|
||||||
--out /mnt/entropy/media-relayout-shows2-plan.tsv \
|
|
||||||
--exceptions /mnt/entropy/media-relayout-shows2-exceptions.txt
|
|
||||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Same executor
|
# 4. Read the exceptions file FIRST — it is the review, not a log. Same executor
|
||||||
# as above, but on this arm's OWN journal, NAMED PER PASS: the journal of an
|
# as above, but on this arm's OWN journal: a shared journal cannot be unwound.
|
||||||
# applied pass is its unwind record, so a second pass that reuses the name
|
scripts/media-apply.py apply --plan /mnt/entropy/media-relayout-shows-plan.tsv \
|
||||||
# destroys the only map back.
|
--root /mnt/entropy --journal /mnt/entropy/media-relayout-shows.jsonl --apply
|
||||||
scripts/media-apply.py apply --plan /mnt/entropy/media-relayout-shows2-plan.tsv \
|
|
||||||
--root /mnt/entropy --journal /mnt/entropy/media-relayout-shows2.jsonl --apply
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Evidence, strongest first — the order the planner believes things:
|
Evidence, strongest first — the order the planner believes things:
|
||||||
@@ -226,19 +217,12 @@ Evidence, strongest first — the order the planner believes things:
|
|||||||
- the **episode marker the file's own name states** (`S01E02-E03`, `1x04`);
|
- the **episode marker the file's own name states** (`S01E02-E03`, `1x04`);
|
||||||
- the **`Season NN` folder it sits in and the number its name leads with inside
|
- the **`Season NN` folder it sits in and the number its name leads with inside
|
||||||
it**: `Season 19/01 Charles Barkley, Nirvana.avi` is S19E01;
|
it**: `Season 19/01 Charles Barkley, Nirvana.avi` is S19E01;
|
||||||
- the **provider row**: which episode the file IS, and the numbering of a file
|
- the **provider row**: identity (the series name and its year), and the
|
||||||
whose own name says nothing;
|
numbering of a file whose own name says nothing;
|
||||||
- the **row's tvdb id, else its TMDb id carried through TMDb's external ids, else
|
|
||||||
an exact tvdb name search**: that id names the SERIES FOLDER and supplies the
|
|
||||||
episode title, because a name the server read off a folder is not a show name;
|
|
||||||
- a **bonus folder** anywhere above the file, or Jellyfin's `ExtraType` → the
|
- a **bonus folder** anywhere above the file, or Jellyfin's `ExtraType` → the
|
||||||
series' canonical extra folder;
|
series' canonical extra folder;
|
||||||
- a runtime over `SHORT_MAX` makes it film-shaped, and it leaves for `--movies` only
|
- a runtime over `SHORT_MAX` makes it film-shaped, and it leaves for `--movies`
|
||||||
when the movie provider's title is a run of words the FILE itself states and its
|
only when a movie provider names it — otherwise it parks.
|
||||||
year agrees with the file's: the query is only what `title_of` salvaged from a
|
|
||||||
release name, and the first hit for a short one is a near miss — a 1963 Bond film
|
|
||||||
for the word "russia", *Special Edition I* for a file saying *Edition III*. Else it
|
|
||||||
parks.
|
|
||||||
|
|
||||||
Two refusals carry the design, and a dry run on the real library found both:
|
Two refusals carry the design, and a dry run on the real library found both:
|
||||||
|
|
||||||
@@ -253,44 +237,125 @@ Two refusals carry the design, and a dry run on the real library found both:
|
|||||||
the file's own leading number, win; a row that still insists on season 1989
|
the file's own leading number, win; a row that still insists on season 1989
|
||||||
parks its file instead of creating `Season 1989`.
|
parks its file instead of creating `Season 1989`.
|
||||||
|
|
||||||
**A provider name is evidence only when an id backs it.** Jellyfin names a Series
|
The provider's name is copied verbatim, warts included: Jellyfin carries a Series
|
||||||
after the folder it scanned, so a library filed by release folders ends up with
|
named `Black Clover 001` (92 episodes, no year) because it matched a pack off a
|
||||||
one Series item per box set — `24 (2001) Season 1`, `Modern Family S05` — and
|
filename. Renaming a show is Jellyfin's work — a folder rename never touches its
|
||||||
copying those names verbatim launders folder junk into the contract. The planner
|
provider ids — so the mover keeps the name it is handed and strips only the pack
|
||||||
resolves each series to its tvdb id and writes tvdb's name and year; the merged
|
index a release leads with (`4a. Seed DESTINY` → `Seed DESTINY`).
|
||||||
identity is also what turns a box set split across five release folders into ONE
|
|
||||||
folder, and what makes two files claiming one episode MEET in the duplicate rule.
|
|
||||||
A series no id resolves keeps the provider's words and is listed under *no tvdb
|
|
||||||
identity* in the exceptions — that list, not a rename, is the review.
|
|
||||||
|
|
||||||
**The review pile lives outside the library.** Files parked inside
|
## The music library's own relayout
|
||||||
`/mnt/entropy/TVShows` were indexed as one show named `_unsorted`, and that
|
|
||||||
show's rows then looked like evidence. So a row whose series is the hidden
|
|
||||||
top-level folder the file itself sits in is dropped — and the folder is matched
|
|
||||||
with its leading underscore stripped, because the server *displays* `_unsorted` as
|
|
||||||
`unsorted`: compare the spelled names and the pile survives the relayout as a
|
|
||||||
visible series folder that real files get renamed into. `--park` outside the root
|
|
||||||
makes the next plan re-file the pile instead of skipping it. Inside a hidden
|
|
||||||
top-level folder a file's own words may MATCH a registered show, never invent one.
|
|
||||||
|
|
||||||
Release-group folder names are the one thing no provider can read, so the
|
`/mnt/entropy/Music` is rips and discography packs, and Jellyfin has matched and
|
||||||
pending TVShows naming plan files 179 top-level directories (9,931 files) under
|
tagged almost all of it. `scripts/media-music.py` re-files it into the shape
|
||||||
the junk name they arrived with. A TVDB id map can name them, under the ladder's
|
Jellyfin's music scanner reads —
|
||||||
own acceptance rule: a hit counts only when TVDB's spelling equals the cleaned
|
|
||||||
query, or differs from it by punctuation alone — the same letters are not a
|
|
||||||
guess. Everything else is a choice for Frank, and the map is never applied
|
|
||||||
unreviewed: `/mnt/entropy/media-relayout-shows-tvdb-proposal.tsv` carries one
|
|
||||||
row per folder with `verdict` = `EXACT` / `PUNCT` / `NEAR` / `NONE` and a `note`
|
|
||||||
naming the two traps — a folder marked `(US)` that matches the UK original, and
|
|
||||||
several folders mapping to one id, which is a MERGE whose duplicate episodes the
|
|
||||||
executor will refuse until a keeper is chosen.
|
|
||||||
|
|
||||||
## The music and audiobook arms
|
Music/<album artist>/<Album> (<year>)/NN - <Title>.ext
|
||||||
|
|
||||||
Each is a page of its own, because each reads a different authority and refuses
|
— and the authority is the tag row of the track, never the name the ripper gave
|
||||||
for a different reason: [[Media Music Relayout]] files by Jellyfin's track tags,
|
the file.
|
||||||
[[Media Audiobook Relayout]] by Plex's chapter rows. Both share this page's
|
|
||||||
executor, the one-journal-per-arm law, the sweep and the rollback.
|
```bash
|
||||||
|
# 1. The authority: Jellyfin's audio rows, one per track, with the album and
|
||||||
|
# artist ids its own database carries (the DB opens read-only).
|
||||||
|
python3 scripts/media-census.py music --root /mnt/entropy/Music \
|
||||||
|
--out /tmp/music_rows.tsv
|
||||||
|
# 2. What is on disk (stat only).
|
||||||
|
python3 scripts/media-census.py enumerate --root /mnt/entropy/Music \
|
||||||
|
--out /tmp/music_enum.tsv
|
||||||
|
# 3. The plan.
|
||||||
|
python3 scripts/media-music.py --root /mnt/entropy/Music --enum /tmp/music_enum.tsv \
|
||||||
|
--rows /tmp/music_rows.tsv --out /tmp/music_plan.tsv \
|
||||||
|
--exceptions /tmp/music_exceptions.txt
|
||||||
|
# 4. Read the exceptions file FIRST — it is the review, not a log. Then the same
|
||||||
|
# executor, journal and rollback as above, on its own journal.
|
||||||
|
scripts/media-apply.py apply --plan /tmp/music_plan.tsv --root /mnt/entropy \
|
||||||
|
--journal /mnt/entropy/media-relayout-music.jsonl --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
The group is what the tags assert, not where the files sit: **(album-artist tag,
|
||||||
|
album, year)**. One album arrives as three disc folders and as two copies of one
|
||||||
|
rip, and one MusicBrainz release group is one album. A compilation is one album
|
||||||
|
whose tracks name many performers, so it files once under `Various Artists`:
|
||||||
|
keying on the performer would file one album once per artist. Two different
|
||||||
|
album-artist tags on one album name are two albums, which is what Jellyfin's two
|
||||||
|
album items say.
|
||||||
|
|
||||||
|
What it refuses:
|
||||||
|
|
||||||
|
+- **A sidecar is placed by the audio around it, never by its own name.** A
|
||||||
|
`.lrc`, `.txt` or picture that shares a track's stem follows that track and
|
||||||
|
takes the name the track now has, or Jellyfin cannot find it; anything else
|
||||||
|
goes to the album its own folder was placed in, or to the folder whose discs
|
||||||
|
sit one level below it. A folder whose audio belongs to two albums parks its
|
||||||
|
sidecar rather than guessing.
|
||||||
|
+- **An image album keeps its name.** A folder holding one audio file and a
|
||||||
|
`.cue` is a whole album in one file; numbering it `01 - …` tells Jellyfin to
|
||||||
|
index it as one track.
|
||||||
|
+- **A video container parks**, because Jellyfin will not index it as music and a
|
||||||
|
rename to `.m4a` would claim it is; **audio no provider row names parks** too,
|
||||||
|
because unmatched audio is a tagging job, not a renaming job.
|
||||||
|
|
||||||
|
The plan applies on 2026-10-31 through `/mnt/entropy/media-relayout-music.jsonl`;
|
||||||
|
its rollback is that journal run through `media-apply.py rollback --apply`. ONE
|
||||||
|
writer at a time on this array: two rename passes race, and neither journal tells.
|
||||||
|
|
||||||
|
## The audiobook library's own relayout
|
||||||
|
|
||||||
|
`/mnt/entropy/Audiobooks` is the one library here that [[Jellyfin]] does not read
|
||||||
|
at all — its music scanner has no audiobook concept — while [[Plex]] indexes all
|
||||||
|
of it as Artist → Album → Track: the album is the BOOK, that book's single artist
|
||||||
|
is the NARRATOR, both matched out of the files' own tags. Today one book is split
|
||||||
|
across `… [Part 1]`, `… [Part 2]` folders three or four deep inside a collection
|
||||||
|
pack, so a scan serves three quarters of a novel; `scripts/media-audiobooks.py`
|
||||||
|
re-files the tree so the property both servers' book scanners assume is true —
|
||||||
|
|
||||||
|
Audiobooks/<Narrator>/<Book> (<year>)/<the file keeps its own name>
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. The authority: Plex's chapter rows for this root (the DB opens read-only).
|
||||||
|
python3 scripts/media-census.py audiobooks --root /mnt/entropy/Audiobooks \
|
||||||
|
--out /tmp/ab_rows.tsv
|
||||||
|
# 2. What is on disk (stat only).
|
||||||
|
python3 scripts/media-census.py enumerate --root /mnt/entropy/Audiobooks \
|
||||||
|
--out /tmp/ab_enum.tsv
|
||||||
|
# 3. The plan.
|
||||||
|
python3 scripts/media-audiobooks.py --root /mnt/entropy/Audiobooks \
|
||||||
|
--enum /tmp/ab_enum.tsv --rows /tmp/ab_rows.tsv --out /tmp/ab_plan.tsv \
|
||||||
|
--exceptions /tmp/ab_exceptions.txt
|
||||||
|
# 4. Read the exceptions file FIRST. Then the same executor, journal and rollback
|
||||||
|
# as above, on its own journal, and refresh the Plex section as in step 5 of
|
||||||
|
# the movie Procedure — from inside the container, never over the host port.
|
||||||
|
scripts/media-apply.py apply --plan /tmp/ab_plan.tsv --root /mnt/entropy \
|
||||||
|
--journal /mnt/entropy/media-relayout-audiobooks.jsonl --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
The group is the identity Plex asserts in its own database — **(narrator, book,
|
||||||
|
year)** — so one book arriving in four part folders becomes one folder, and a
|
||||||
|
1965 and a 2020 recording that share a title stay two books.
|
||||||
|
|
||||||
|
Two laws are forced by what the rows actually hold:
|
||||||
|
|
||||||
|
+- **A chapter is never renumbered.** Most books contain chapters whose index Plex
|
||||||
|
does not know, so a number written into a name would be invented — and it would
|
||||||
|
buy nothing, because both servers order playback by the embedded track tag, not
|
||||||
|
by the file name. A wrong number in a name is the number a human later trusts.
|
||||||
|
Fix order in the tags: a Plex scan re-reads tags and never renames a file.
|
||||||
|
+- **The shelf is the narrator, not the author.** Every book has exactly one artist
|
||||||
|
row and it is the narrator; the author survives only inside some album titles
|
||||||
|
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving it would
|
||||||
|
mean parsing titles where a ripper wrote one and inventing it everywhere else.
|
||||||
|
The author stays visible in the folder name Plex itself shows.
|
||||||
|
|
||||||
|
What it refuses, the music arm's list with one addition: a video container, audio
|
||||||
|
no chapter row names, and a loose file in a folder holding two books all park under
|
||||||
|
`_unsorted` with the path they have today — and so does a chapter whose row names
|
||||||
|
**no book or no narrator**: a placeholder (`[Unknown Album]`, `[Unknown Artist]`)
|
||||||
|
that unrelated books share, or a chapter number mistagged as the artist (`0.4`).
|
||||||
|
`Various Artists` is not a placeholder — that is a multi-cast read.
|
||||||
|
|
||||||
|
The plan stands reviewed at `/mnt/entropy/media-relayout-audiobooks-plan.tsv`
|
||||||
|
(55,114 moves, 7,764 parked, executor preflight refused nothing); it applies on its
|
||||||
|
own journal, behind the music writer, and then refreshes the Plex section.
|
||||||
|
|
||||||
## Laws
|
## Laws
|
||||||
|
|
||||||
@@ -331,6 +396,19 @@ executor, the one-journal-per-arm law, the sweep and the rollback.
|
|||||||
free space, and `rename(2)` refuses it anyway. Trash goes beside the item —
|
free space, and `rename(2)` refuses it anyway. Trash goes beside the item —
|
||||||
`/mnt/entropy/.AGENTTRASH/<date>-<topic>/` — so it costs one inode change and
|
`/mnt/entropy/.AGENTTRASH/<date>-<topic>/` — so it costs one inode change and
|
||||||
rolls back exactly like the move that made it.
|
rolls back exactly like the move that made it.
|
||||||
|
- **A Jellyfin 401 is the header, not the key.** 12.4 authenticates only
|
||||||
|
`Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token` for
|
||||||
|
every key, valid or not. This page carried "no credential on this box can
|
||||||
|
trigger a scan, the two Seerr keys answer 401" for weeks because of that
|
||||||
|
header — the keys were innocent. Before believing a 401, prove the key against
|
||||||
|
the server's own table: `select count(*) from ApiKeys where AccessToken = ?`
|
||||||
|
on `jellyfin.db` opened `mode=ro&immutable=1`.
|
||||||
|
- **A scan that must see a new folder has to be recursive.** `POST
|
||||||
|
/Library/Refresh` is the incremental pass, and inotify does not cross a
|
||||||
|
rootless podman bind, so folders created on the host stay invisible to it. The
|
||||||
|
move is `POST /Items/{id}/Refresh?Recursive=true` (204) per library, the id
|
||||||
|
from `GET /Library/VirtualFolders` — its field is `ItemId`, not `Id` — and
|
||||||
|
that same record's `RefreshStatus` is the progress read.
|
||||||
- **A file whose name carries no words is bonus material, not a folder.**
|
- **A file whose name carries no words is bonus material, not a folder.**
|
||||||
`.Extra 1 ( 1H 27min multisub ).mkv` parses to `1`; a movie folder named `1`
|
`.Extra 1 ( 1H 27min multisub ).mkv` parses to `1`; a movie folder named `1`
|
||||||
is not a movie. It becomes a review line telling a human to name the file.
|
is not a movie. It becomes a review line telling a human to name the file.
|
||||||
@@ -359,15 +437,16 @@ executor, the one-journal-per-arm law, the sweep and the rollback.
|
|||||||
recreates the folder, so the sweep is undone by the same command as the moves.
|
recreates the folder, so the sweep is undone by the same command as the moves.
|
||||||
Proof carries downward: inside a folder the journal proves emptied, every
|
Proof carries downward: inside a folder the journal proves emptied, every
|
||||||
empty directory goes with it. A release's `Screens/` or `Sample/` is never a
|
empty directory goes with it. A release's `Screens/` or `Sample/` is never a
|
||||||
move source itself, so it can hold an emptied parent until the next sweep.
|
move source itself, so 146 of them held their emptied parents on disk after
|
||||||
|
the first sweep.
|
||||||
A failed `rmdir` is not a removal: only a folder that is really gone leaves
|
A failed `rmdir` is not a removal: only a folder that is really gone leaves
|
||||||
scope, so the next sweep finishes what a permission problem stopped.
|
scope, so the next sweep finishes what a permission problem stopped.
|
||||||
- **An episode-shaped name marks a SERIES, not a film — and the series is the
|
- **An episode-shaped name marks a SERIES, not a film — and the series is the
|
||||||
name BEFORE the marker.** `Monk - S05E03 - Mr. Monk and the Big Game` is series
|
name BEFORE the marker.** `Monk - S05E03 - Mr. Monk and the Big Game` is series
|
||||||
`Monk`. Keyed on the whole title instead, every episode became a group of one,
|
`Monk`. Keyed on the whole title instead, every episode became a group of one
|
||||||
never reaching the five-episode size that calls something a series — a whole
|
and never reached the five-episode size that calls something a series, so a
|
||||||
box set filed as one folder per episode. The threshold still earns its keep:
|
box set was filed as 124 movies in 124 folders. The threshold still earns its
|
||||||
a lone film named `Part 2` stays a film. Episodes of a series are parked
|
keep: a lone film named `Part 2` stays a film. Episodes of a series are parked
|
||||||
in `_TV needs a TV library - <series>` for a human to move to a TV source —
|
in `_TV needs a TV library - <series>` for a human to move to a TV source —
|
||||||
the movie tool never files TV.
|
the movie tool never files TV.
|
||||||
- **One journal is one undo.** The TV move runs on its own journal, and with
|
- **One journal is one undo.** The TV move runs on its own journal, and with
|
||||||
@@ -410,14 +489,6 @@ any folder a sweep removed, restores every source path, removes only the folders
|
|||||||
that run created *and that are now empty*, and refuses everything if any moved
|
that run created *and that are now empty*, and refuses everything if any moved
|
||||||
file cannot be accounted for. A rolled-back journal refuses to roll back twice.
|
file cannot be accounted for. A rolled-back journal refuses to roll back twice.
|
||||||
|
|
||||||
**Roll a chain back newest journal first, or the tool refuses.** Each pass
|
|
||||||
re-files what an earlier one moved, so a journal is a clean inverse only until
|
|
||||||
something downstream touches its files; `ls -l /mnt/entropy/media-relayout*.jsonl`
|
|
||||||
is the order. Refusing is the tool working: on 2026-10-31 the movie journal could
|
|
||||||
not restore files the later TV passes had re-filed, and the stray journal could
|
|
||||||
not restore files Jellyfin's extension sweep had renamed. Roll back what still
|
|
||||||
inverts and leave the rest as the current state — never force it.
|
|
||||||
|
|
||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
[[Jellyfin]], [[Plex]], [[Media Metadata View]], [[Entropy Volume]], [[Testing Policy]], [[Storage Policy]]
|
[[Jellyfin]], [[Plex]], [[Media Metadata View]], [[Entropy Volume]], [[Testing Policy]], [[Storage Policy]]
|
||||||
|
|||||||
@@ -87,45 +87,3 @@ root runs as the `dank` user, and the deployer adopts the data root with
|
|||||||
while the HTTP layer finishes loading; the first boot of a years-old
|
while the HTTP layer finishes loading; the first boot of a years-old
|
||||||
library took ~10 minutes (2026-09 cutover). The deployer gate now allows
|
library took ~10 minutes (2026-09 cutover). The deployer gate now allows
|
||||||
600 s before failing; later boots are much faster (warm page cache).
|
600 s before failing; later boots are much faster (warm page cache).
|
||||||
|
|
||||||
## Triggering a scan
|
|
||||||
|
|
||||||
A rename on disk is invisible until the section is rescanned, and a moved file is
|
|
||||||
a `Couldn't find the file to stream` 404 until that scan passes it. Run this from
|
|
||||||
the host shell through `podman exec`, never from a neighbour container: PMS's
|
|
||||||
HTTPD closes without a single byte on any request that arrives from another
|
|
||||||
container's netns (10.89.x.x reads as "empty reply", not as a 401). The token
|
|
||||||
rides stdin, never argv, where any process on the host could read it out of
|
|
||||||
`/proc/<pid>/cmdline`; it lives in `Preferences.xml` inside the RW data bind.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
umask 077
|
|
||||||
podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \
|
|
||||||
'/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok
|
|
||||||
cat >/tmp/pxscan.sh <<'EOS'
|
|
||||||
read -r T
|
|
||||||
cfg=$(mktemp)
|
|
||||||
mk() { { printf 'header = "X-Plex-Token: %s"\nheader = "Accept: application/json"\n' "$T"
|
|
||||||
printf 'url = "%s"\n' "$1"; } > "$cfg"; }
|
|
||||||
mk "http://127.0.0.1:32400/library/sections"
|
|
||||||
for k in $(curl -s -K "$cfg" | grep -o '"key":"[0-9]*"' | cut -d\" -f4); do
|
|
||||||
mk "http://127.0.0.1:32400/library/sections/$k/refresh"
|
|
||||||
printf 'request = "POST"\n' >> "$cfg"
|
|
||||||
echo "plex section $k: $(curl -s -o /dev/null -w '%{http_code}' -K "$cfg")"
|
|
||||||
done
|
|
||||||
shred -u "$cfg"
|
|
||||||
EOS
|
|
||||||
podman cp /tmp/pxscan.sh plex:/tmp/pxscan.sh
|
|
||||||
podman exec -i plex sh /tmp/pxscan.sh </tmp/px.tok
|
|
||||||
shred -u /tmp/px.tok
|
|
||||||
```
|
|
||||||
|
|
||||||
Every section answers `204` — or `200` with a Directory body when
|
|
||||||
`Accept: application/json` is sent. Inside the quoted `printf` format a
|
|
||||||
backslash-newline is LITERAL, not a continuation, so `mk` writes two `printf`s:
|
|
||||||
one glued line and one stray backslash is a config file curl rejects, and the
|
|
||||||
failure prints as `option -K: is unknown`, never as a bad config line.
|
|
||||||
|
|
||||||
Proof that a rename landed is not a scan response: `scripts/media-verify.py`
|
|
||||||
replays the run's journal against the database — the only read that
|
|
||||||
distinguishes "not scanned yet" from "gone"; see [[Media Relayout]].
|
|
||||||
@@ -51,9 +51,7 @@ is a locator: one row, one clause, no numbers.
|
|||||||
| [[Entropy Volume]] | The Dankserver entropy volume is a RAID-backed LUKS filesystem mounted at |
|
| [[Entropy Volume]] | The Dankserver entropy volume is a RAID-backed LUKS filesystem mounted at |
|
||||||
| [[Jellyfin]] | Jellyfin on Dankserver: quadlet unit, overlay metadata view, provider keys, ports, access, transcode status. |
|
| [[Jellyfin]] | Jellyfin on Dankserver: quadlet unit, overlay metadata view, provider keys, ports, access, transcode status. |
|
||||||
| [[mariadb]] | The MariaDB backend for RomM: quadlet unit, network alias, env-file secret handling, and the backup path. |
|
| [[mariadb]] | The MariaDB backend for RomM: quadlet unit, network alias, env-file secret handling, and the backup path. |
|
||||||
| [[Media Audiobook Relayout]] | Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals. |
|
|
||||||
| [[Media Metadata View]] | The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback. |
|
| [[Media Metadata View]] | The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback. |
|
||||||
| [[Media Music Relayout]] | Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals. |
|
|
||||||
| [[Media Relayout]] | Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback. |
|
| [[Media Relayout]] | Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback. |
|
||||||
| [[Plex]] | Plex Media Server is a **containerized media server on Dankserver**: a podman |
|
| [[Plex]] | Plex Media Server is a **containerized media server on Dankserver**: a podman |
|
||||||
| [[redis]] | The Redis cache and task queue for RomM: quadlet unit, network alias topology, and the backup path. |
|
| [[redis]] | The Redis cache and task queue for RomM: quadlet unit, network alias topology, and the backup path. |
|
||||||
|
|||||||
@@ -228,6 +228,51 @@ PY
|
|||||||
[ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 1 ] # header only
|
[ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 1 ] # header only
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# wal_fresh DB ROOT OUT — a row Plex matched during a scan and committed, but
|
||||||
|
# has not checkpointed: the normal state of a library being walked. The holder
|
||||||
|
# connection stops the WAL folding into the main file when the writer closes,
|
||||||
|
# and the census runs as a real subprocess against those same files.
|
||||||
|
wal_fresh() {
|
||||||
|
local db="$1" root="$2" out="$3"
|
||||||
|
python3 - "$db" "$root" "$out" "$CENSUS" <<'PY'
|
||||||
|
import sqlite3, subprocess, sys
|
||||||
|
db, root, out, census = sys.argv[1:5]
|
||||||
|
w = sqlite3.connect(db)
|
||||||
|
w.execute("pragma journal_mode=wal")
|
||||||
|
w.execute("pragma wal_autocheckpoint=0")
|
||||||
|
w.commit()
|
||||||
|
w.execute("pragma wal_checkpoint(truncate)") # the rows so far are on disk
|
||||||
|
holder = sqlite3.connect(db) # last close would checkpoint
|
||||||
|
ar = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||||
|
' year, "index", deleted_at) VALUES (NULL, 8, ?, NULL, NULL, NULL)',
|
||||||
|
("Fresh Narrator",)).lastrowid
|
||||||
|
al = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||||
|
' year, "index", deleted_at) VALUES (?, 9, ?, ?, NULL, NULL)',
|
||||||
|
(ar, "Fresh Book", 2026)).lastrowid
|
||||||
|
mi = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||||
|
' year, "index", deleted_at) VALUES (?, 10, ?, NULL, ?, NULL)',
|
||||||
|
(al, "Fresh Chapter", 1)).lastrowid
|
||||||
|
it = w.execute("insert into media_items (metadata_item_id, deleted_at)"
|
||||||
|
" VALUES (?, NULL)", (mi,)).lastrowid
|
||||||
|
w.execute("insert into media_parts (media_item_id, file, deleted_at)"
|
||||||
|
" VALUES (?, ?, NULL)", (it, root + "/Fresh/01.mp3"))
|
||||||
|
w.commit()
|
||||||
|
sys.exit(subprocess.run([sys.executable, census, "audiobooks", "--root", root,
|
||||||
|
"--db", db, "--out", out]).returncode)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "the census exports a row the scanner committed but has not checkpointed" {
|
||||||
|
DB="$BATS_TEST_TMPDIR/plex.db"
|
||||||
|
db_with_part "$DB" "$BATS_TEST_TMPDIR/Audiobooks/Old/01.mp3" \
|
||||||
|
"Old Narrator" "Old Book" 1990 1 "Old Chapter"
|
||||||
|
run wal_fresh "$DB" "$BATS_TEST_TMPDIR/Audiobooks" "$BATS_TEST_TMPDIR/exp.tsv"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 3 ] # header + both rows
|
||||||
|
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||||
|
| grep -q "^$BATS_TEST_TMPDIR/Audiobooks/Fresh/01\.mp3 Fresh Narrator Fresh Book 2026 1 Fresh Chapter$"
|
||||||
|
}
|
||||||
|
|
||||||
# -------------------------------------------------------------------- the plan
|
# -------------------------------------------------------------------- the plan
|
||||||
|
|
||||||
@test "a matched chapter files under its narrator and its book, name untouched" {
|
@test "a matched chapter files under its narrator and its book, name untouched" {
|
||||||
|
|||||||
+25
-177
@@ -8,14 +8,7 @@
|
|||||||
# file name. So every test here states which authority a file had — name only,
|
# file name. So every test here states which authority a file had — name only,
|
||||||
# provider only, both, neither — and asserts where the PLAN files it, never how
|
# provider only, both, neither — and asserts where the PLAN files it, never how
|
||||||
# the planner decided. A fixture Jellyfin database stands in for the live one and
|
# the planner decided. A fixture Jellyfin database stands in for the live one and
|
||||||
# the name caches are input files, so no test touches the network.
|
# the TMDb cache is an input file, so no test touches the network.
|
||||||
#
|
|
||||||
# The target contract is tvdb-grade naming:
|
|
||||||
# <Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.<ext>
|
|
||||||
# tvdb is the IDENTITY authority: a provider name a server derived from a folder
|
|
||||||
# is not a series name, so a polluted name is undone through the series' tvdb id.
|
|
||||||
# The review pile parks OUTSIDE the library — a parked folder inside it indexes
|
|
||||||
# as one junk show, and that show's rows are then mistaken for evidence.
|
|
||||||
#
|
#
|
||||||
# The two outcomes that cost a re-run of a 50k-file plan are a file filed as an
|
# The two outcomes that cost a re-run of a 50k-file plan are a file filed as an
|
||||||
# episode that is not one (it steals an episode number from its real owner) and a
|
# episode that is not one (it steals an episode number from its real owner) and a
|
||||||
@@ -32,11 +25,11 @@ APPLY="$REPO_ROOT/scripts/media-apply.py"
|
|||||||
# at PATH SIZE — a file with deterministic content.
|
# at PATH SIZE — a file with deterministic content.
|
||||||
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
|
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
|
||||||
|
|
||||||
# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] [TMDB] [TITLE] [TVDB]
|
# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] — one row of the authority
|
||||||
# — one row of the authority export scripts/media-census.py jellyfin writes.
|
# export scripts/media-census.py jellyfin writes.
|
||||||
title_row() {
|
title_row() {
|
||||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "${4:-}" \
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t\n' "$1" "$2" "$3" "${4:-}" "${5:-}" "${6:-}" \
|
||||||
"${5:-}" "${6:-}" "${7:-}" "${8:-}" "${9:-}" >> "$TITLES"
|
>> "$TITLES"
|
||||||
}
|
}
|
||||||
|
|
||||||
# dst SRC — the destination the plan gives SRC, or nothing when it does not move.
|
# dst SRC — the destination the plan gives SRC, or nothing when it does not move.
|
||||||
@@ -47,7 +40,6 @@ dst() {
|
|||||||
setup() {
|
setup() {
|
||||||
LIB="$BATS_TEST_TMPDIR/TVShows"
|
LIB="$BATS_TEST_TMPDIR/TVShows"
|
||||||
PARK="$LIB/_unsorted"
|
PARK="$LIB/_unsorted"
|
||||||
OUTSIDE="$BATS_TEST_TMPDIR/park"
|
|
||||||
MOVIES="$BATS_TEST_TMPDIR/Movies"
|
MOVIES="$BATS_TEST_TMPDIR/Movies"
|
||||||
ENUM="$BATS_TEST_TMPDIR/enum.tsv"
|
ENUM="$BATS_TEST_TMPDIR/enum.tsv"
|
||||||
PROBED="$BATS_TEST_TMPDIR/probed.tsv"
|
PROBED="$BATS_TEST_TMPDIR/probed.tsv"
|
||||||
@@ -85,15 +77,8 @@ setup() {
|
|||||||
# A season folder whose rows are stale: the database numbers every file of this
|
# A season folder whose rows are stale: the database numbers every file of this
|
||||||
# pack season 16 episode 20, four seasons' worth of them.
|
# pack season 16 episode 20, four seasons' worth of them.
|
||||||
at "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" 6000
|
at "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" 6000
|
||||||
|
# One episode-shaped file loose at the library root.
|
||||||
at "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" 4000
|
at "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" 4000
|
||||||
# A box-set folder whose Series item the server named AFTER THE FOLDER: the
|
|
||||||
# row's own name is junk, and only a provider id can undo it.
|
|
||||||
BOX="$LIB/Show Box (2016) Season 1 S01 (1080p WEB-DL)"
|
|
||||||
at "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" 6000
|
|
||||||
# The park inside the library: the server indexed the whole pile as ONE show
|
|
||||||
# named after the folder, so its rows state a series that is not a show.
|
|
||||||
at "$PARK/Park Show S01E01.mkv" 5000
|
|
||||||
at "$PARK/Twisted.Metal.S01E01.1080p.WEB-DL.mkv" 5200
|
|
||||||
|
|
||||||
# The enum census lists every file; the planner reads that instead of walking
|
# The enum census lists every file; the planner reads that instead of walking
|
||||||
# the tree, so the fixture writes the same three columns.
|
# the tree, so the fixture writes the same three columns.
|
||||||
@@ -107,21 +92,17 @@ with open(out, "w") as fh:
|
|||||||
fh.write(f"{os.stat(p).st_size}\t{os.path.splitext(f)[1]}\t{p}\n")
|
fh.write(f"{os.stat(p).st_size}\t{os.path.splitext(f)[1]}\t{p}\n")
|
||||||
PY
|
PY
|
||||||
|
|
||||||
printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\ttitle\ttvdb\n' > "$TITLES"
|
printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\n' > "$TITLES"
|
||||||
printf '{}\n' > "$CACHE"
|
printf '{}\n' > "$CACHE"
|
||||||
title_row "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" \
|
title_row "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" \
|
||||||
"The Examples" 2010 1 1 "" "" "Pilot"
|
"The Examples" 2010 1 1
|
||||||
title_row "$ANIME/[grp] Anime Pack - 05.mkv" "Anime Pack" "" 1 5
|
title_row "$ANIME/[grp] Anime Pack - 05.mkv" "Anime Pack" "" 1 5
|
||||||
title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots"
|
title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1
|
||||||
title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots"
|
title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1
|
||||||
title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2 "" "" "Double Trouble"
|
title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2
|
||||||
title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3 "" "" "Loose Ends"
|
title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3
|
||||||
title_row "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" \
|
title_row "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" \
|
||||||
"SNL Pack" "" 16 20
|
"SNL Pack" "" 16 20
|
||||||
title_row "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" \
|
|
||||||
"Show Box (2016) Season 1" 2016 1 1 "" "2334" "Pilot" "78850"
|
|
||||||
# The row the server wrote for a parked file: the series is the PARK FOLDER.
|
|
||||||
title_row "$PARK/Park Show S01E01.mkv" "_unsorted" "" 1 1
|
|
||||||
}
|
}
|
||||||
|
|
||||||
plan() {
|
plan() {
|
||||||
@@ -130,28 +111,6 @@ plan() {
|
|||||||
--out "$OUT" --exceptions "$EXC"
|
--out "$OUT" --exceptions "$EXC"
|
||||||
}
|
}
|
||||||
|
|
||||||
plan_park() { # the review pile outside the library
|
|
||||||
run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --titles "$TITLES" \
|
|
||||||
--probed "$PROBED" --tmdb-cache "$CACHE" --movies "$MOVIES" \
|
|
||||||
--park "$OUTSIDE" --out "$OUT" --exceptions "$EXC"
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "--tmdb with no key is reported as an unfinished plan, not a finished one" {
|
|
||||||
# A dead provider is invisible where it happens: every 401 is one line on
|
|
||||||
# stderr, and a terminal that scrolled past them shows a plan that reads as
|
|
||||||
# complete while every folder still carries the metadata provider's junk. The
|
|
||||||
# plan is still written — the answers that DID arrive are worth their rename —
|
|
||||||
# so the state has to ride in the summary line and in the review file.
|
|
||||||
# env -u keeps this offline no matter what the developer's shell exports.
|
|
||||||
run env -u TMDB_API_KEY python3 "$PLAN" --root "$LIB" --enum "$ENUM" \
|
|
||||||
--titles "$TITLES" --probed "$PROBED" --tmdb-cache "$CACHE" \
|
|
||||||
--movies "$MOVIES" --tmdb --out "$OUT" --exceptions "$EXC"
|
|
||||||
[ "$status" -eq 0 ]
|
|
||||||
grep -q 'tmdb NOT ASKED (TMDB_API_KEY unset): NOT provider-complete' <<<"$output"
|
|
||||||
grep -q 'WARNING: tmdb NOT ASKED (TMDB_API_KEY unset) — NOT provider-complete' "$EXC"
|
|
||||||
grep -qE '^plan: [1-9][0-9]* moves' <<<"$output"
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "the census exports the provider's episode numbering as a TSV" {
|
@test "the census exports the provider's episode numbering as a TSV" {
|
||||||
DB="$BATS_TEST_TMPDIR/jellyfin.db"
|
DB="$BATS_TEST_TMPDIR/jellyfin.db"
|
||||||
python3 - "$DB" "$SEASON" <<'PY'
|
python3 - "$DB" "$SEASON" <<'PY'
|
||||||
@@ -166,7 +125,7 @@ sid = str(uuid.uuid4())
|
|||||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
||||||
(sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples",
|
(sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples",
|
||||||
pre, None, None, 2010, None, 0,
|
pre, None, None, 2010, None, 0,
|
||||||
'{"ProviderIds":{"Tmdb":"1234","Tvdb":"78850"}}'))
|
'{"ProviderIds":{"Tmdb":"1234"}}'))
|
||||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
||||||
(str(uuid.uuid4()), sid, "MediaBrowser.Controller.Entities.TV.Episode",
|
(str(uuid.uuid4()), sid, "MediaBrowser.Controller.Entities.TV.Episode",
|
||||||
"Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1,
|
"Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1,
|
||||||
@@ -177,113 +136,34 @@ PY
|
|||||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||||
[ "$status" -eq 0 ]
|
[ "$status" -eq 0 ]
|
||||||
run head -1 "$BATS_TEST_TMPDIR/exp.tsv"
|
run head -1 "$BATS_TEST_TMPDIR/exp.tsv"
|
||||||
[ "${lines[0]}" = "file series year season episode extra tmdb title tvdb" ]
|
[ "${lines[0]}" = "file series year season episode extra tmdb" ]
|
||||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||||
| grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234 Pilot 78850$"
|
| grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234$"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "the census reads provider ids from the table newer Jellyfin keeps" {
|
@test "an episode its own name states files as Show SxxEyy under Season NN" {
|
||||||
DB="$BATS_TEST_TMPDIR/jellyfin12.db"
|
|
||||||
python3 - "$DB" "$SEASON" <<'PY'
|
|
||||||
import os, sqlite3, sys, uuid
|
|
||||||
db, season = sys.argv[1], sys.argv[2]
|
|
||||||
c = sqlite3.connect(db)
|
|
||||||
pre = "/data/TVShows/" + os.path.basename(season)
|
|
||||||
c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber,
|
|
||||||
ParentIndexNumber, ProductionYear, ExtraType,
|
|
||||||
IsVirtualItem, Data)""")
|
|
||||||
# Jellyfin 12 moved the ids out of the Data blob and into their own table.
|
|
||||||
c.execute("create table BaseItemProviders (ItemId, ProviderId, ProviderValue)")
|
|
||||||
sid = str(uuid.uuid4())
|
|
||||||
eid = str(uuid.uuid4())
|
|
||||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
|
||||||
(sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples",
|
|
||||||
pre, None, None, 2010, None, 0, '{}'))
|
|
||||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
|
||||||
(eid, sid, "MediaBrowser.Controller.Entities.TV.Episode",
|
|
||||||
"Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1,
|
|
||||||
None, None, 0, '{}'))
|
|
||||||
c.execute("insert into BaseItemProviders values (?,?,?)", (sid, "Tvdb", "9999"))
|
|
||||||
c.commit()
|
|
||||||
PY
|
|
||||||
run python3 "$CENSUS" jellyfin --root "$LIB" --db "$DB" \
|
|
||||||
--out "$BATS_TEST_TMPDIR/exp12.tsv"
|
|
||||||
[ "$status" -eq 0 ]
|
|
||||||
tail -1 "$BATS_TEST_TMPDIR/exp12.tsv" \
|
|
||||||
| grep -q " Pilot 9999$"
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "an episode its own name states files as the titled standard name" {
|
|
||||||
plan
|
plan
|
||||||
[ "$status" -eq 0 ]
|
[ "$status" -eq 0 ]
|
||||||
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv")" \
|
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv")" \
|
||||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.mkv" ]
|
= "$LIB/The Examples (2010)/Season 01/The Examples S01E01.mkv" ]
|
||||||
}
|
|
||||||
|
|
||||||
@test "the tvdb identity names the series folder, not the name a folder left behind" {
|
|
||||||
printf '{"tvdb:series:78850": {"name": "Show Box", "year": "2016"}}\n' > "$CACHE"
|
|
||||||
plan
|
|
||||||
[ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" \
|
|
||||||
= "$LIB/Show Box (2016)/Season 01/Show Box (2016) - S01E01 - Pilot.mkv" ]
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "a provider name that already carries the year is not given it twice" {
|
|
||||||
# TVDb answers the reboot as `Acapulco (2021)`. The folder contract is ONE year,
|
|
||||||
# and `Acapulco (2021) (2021)` is a folder neither matcher nor human reuses.
|
|
||||||
printf '{"tvdb:series:78850": {"name": "Show Box (2016)", "year": "2016"}}\n' > "$CACHE"
|
|
||||||
plan
|
|
||||||
[ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" \
|
|
||||||
= "$LIB/Show Box (2016)/Season 01/Show Box (2016) - S01E01 - Pilot.mkv" ]
|
|
||||||
run grep -F '(2016) (2016)' "$OUT"
|
|
||||||
[ -z "$output" ]
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "a series no authority resolves keeps the provider's name and is reported" {
|
|
||||||
plan
|
|
||||||
[ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \
|
|
||||||
= "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ]
|
|
||||||
grep -A9 -i "no tvdb" "$EXC" | grep -q "SNL Pack"
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "a row whose series is the park folder is not evidence of a show" {
|
|
||||||
plan_park
|
|
||||||
run awk -F '\t' -v p="$PARK/" '$1 == "move" && index($3, p) == 1 {print $3}' "$OUT"
|
|
||||||
[ -z "$output" ]
|
|
||||||
run awk -F '\t' '$1 == "move" && $2 ~ /Park Show S01E01/ {print $3}' "$OUT"
|
|
||||||
[[ "$output" == "$OUTSIDE/"* ]]
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "the pile's sanitized display name is still the pile, not a series" {
|
|
||||||
# Jellyfin drops the leading underscore when it shows the hidden folder, so the
|
|
||||||
# row says the series is "unsorted" while the file sits in `_unsorted`. Match
|
|
||||||
# only the underscored spelling and the plan renames real files into a visible
|
|
||||||
# folder named after the review pile — which is how 617 of them got there.
|
|
||||||
title_row "$PARK/Twisted.Metal.S01E01.1080p.WEB-DL.mkv" "unsorted" "" 1 1 \
|
|
||||||
"" "" "Twisted Metal"
|
|
||||||
plan_park
|
|
||||||
run awk -F '\t' -v l="$LIB" \
|
|
||||||
'$1 == "move" && index($3, l "/unsorted/") == 1 {print $3}' "$OUT"
|
|
||||||
[ -z "$output" ]
|
|
||||||
run awk -F '\t' '$1 == "move" && $2 ~ /Twisted\.Metal\.S01E01/ {print $3}' "$OUT"
|
|
||||||
[[ "$output" == "$OUTSIDE/"* ]]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "a subtitle follows its episode and keeps its language flag" {
|
@test "a subtitle follows its episode and keeps its language flag" {
|
||||||
plan
|
plan
|
||||||
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt")" \
|
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt")" \
|
||||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.en.srt" ]
|
= "$LIB/The Examples (2010)/Season 01/The Examples S01E01.en.srt" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "a subtitle two folders down inside Subs/<release-name>/ still follows its episode" {
|
@test "a subtitle two folders down inside Subs/<release-name>/ still follows its episode" {
|
||||||
plan
|
plan
|
||||||
[ "$(dst "$SEASON/Subs/The.Examples.S01E01.Pilot.1080p.x265/The.Examples.S01E01.Pilot.1080p.x265.jpn.srt")" \
|
[ "$(dst "$SEASON/Subs/The.Examples.S01E01.Pilot.1080p.x265/The.Examples.S01E01.Pilot.1080p.x265.jpn.srt")" \
|
||||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.jpn.srt" ]
|
= "$LIB/The Examples (2010)/Season 01/The Examples S01E01.jpn.srt" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "a subtitle that carries the container's word in its own name still follows the episode" {
|
@test "a subtitle that carries the container's word in its own name still follows the episode" {
|
||||||
plan
|
plan
|
||||||
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv.forced.srt")" \
|
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv.forced.srt")" \
|
||||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.forced.srt" ]
|
= "$LIB/The Examples (2010)/Season 01/The Examples S01E01.forced.srt" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "a season image keeps its own name; a series nfo keeps the series folder" {
|
@test "a season image keeps its own name; a series nfo keeps the series folder" {
|
||||||
@@ -298,10 +178,10 @@ PY
|
|||||||
[ -z "$(awk -F '\t' '$3 ~ /Season .*teaser/ {print $3}' "$OUT")" ]
|
[ -z "$(awk -F '\t' '$3 ~ /Season .*teaser/ {print $3}' "$OUT")" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "a file only the provider numbers files at the episode it is matched to, title or no title" {
|
@test "a file only the provider numbers files at the episode it is matched to" {
|
||||||
plan
|
plan
|
||||||
[ "$(dst "$ANIME/[grp] Anime Pack - 05.mkv")" \
|
[ "$(dst "$ANIME/[grp] Anime Pack - 05.mkv")" \
|
||||||
= "$LIB/Anime Pack/Season 01/Anime Pack - S01E05.mkv" ]
|
= "$LIB/Anime Pack/Season 01/Anime Pack S01E05.mkv" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "a bonus folder inside a release pack belongs to the pack's show, not to its own name" {
|
@test "a bonus folder inside a release pack belongs to the pack's show, not to its own name" {
|
||||||
@@ -313,19 +193,19 @@ PY
|
|||||||
@test "a Season folder outvotes the stale row its files carry" {
|
@test "a Season folder outvotes the stale row its files carry" {
|
||||||
plan
|
plan
|
||||||
[ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \
|
[ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \
|
||||||
= "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ]
|
= "$LIB/SNL Pack/Season 19/SNL Pack S19E01.avi" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@test "one file holding two episodes keeps both numbers in its name" {
|
@test "one file holding two episodes keeps both numbers in its name" {
|
||||||
plan
|
plan
|
||||||
[ "$(dst "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv")" \
|
[ "$(dst "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv")" \
|
||||||
= "$LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E02-E03 - Double Trouble.mkv" ]
|
= "$LIB/Show Two (2016)/Season 01/Show Two S01E02-E03.mkv" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "two files claiming one episode: one keeps the name, the other parks" {
|
@test "two files claiming one episode: one keeps the name, the other parks" {
|
||||||
plan
|
plan
|
||||||
n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E01 - Two Pilots.mkv " "$OUT")
|
n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two S01E01.mkv " "$OUT")
|
||||||
[ "$n" -eq 1 ]
|
[ "$n" -eq 1 ]
|
||||||
run dst "$LIB/Dupes/Show Two S01E01.mkv"
|
run dst "$LIB/Dupes/Show Two S01E01.mkv"
|
||||||
[[ "$output" == "$PARK/"* ]]
|
[[ "$output" == "$PARK/"* ]]
|
||||||
@@ -360,38 +240,6 @@ json.dump({"movie:some documentary":
|
|||||||
[[ "$output" == "$PARK/"* ]]
|
[[ "$output" == "$PARK/"* ]]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "a movie answer the file does not itself name stays in the review pile" {
|
|
||||||
# TMDb's top hit for a truncated query is a NEAR miss, not an answer: "mobile
|
|
||||||
# suit gundam 00" comes back as Special Edition I while the file is Special
|
|
||||||
# Edition III, and a one-word query "russia" comes back as a 1963 Bond film. A
|
|
||||||
# file leaves the TV library only when it states the title the provider names.
|
|
||||||
printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \
|
|
||||||
"$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED"
|
|
||||||
python3 -c 'import json, sys
|
|
||||||
json.dump({"movie:some documentary":
|
|
||||||
{"name": "Some Documentary Special Part Two", "year": "2015", "id": 55}},
|
|
||||||
open(sys.argv[1], "w"))' "$CACHE"
|
|
||||||
plan
|
|
||||||
run dst "$LIB/Longform/Some Documentary Special (2015).mkv"
|
|
||||||
[[ "$output" == "$PARK/"* ]]
|
|
||||||
run awk -F '\t' -v m="$MOVIES/" '$1 == "move" && index($3, m) == 1 {print $3}' "$OUT"
|
|
||||||
[ -z "$output" ]
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "a movie answer from another year is not that film" {
|
|
||||||
# The year the file states outranks a provider's eagerness: the 2015 special is
|
|
||||||
# not the 1999 one, however alike their names are.
|
|
||||||
printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \
|
|
||||||
"$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED"
|
|
||||||
python3 -c 'import json, sys
|
|
||||||
json.dump({"movie:some documentary":
|
|
||||||
{"name": "Some Documentary Special", "year": "1999", "id": 55}},
|
|
||||||
open(sys.argv[1], "w"))' "$CACHE"
|
|
||||||
plan
|
|
||||||
run dst "$LIB/Longform/Some Documentary Special (2015).mkv"
|
|
||||||
[[ "$output" == "$PARK/"* ]]
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "the plan creates every folder before the move that needs it" {
|
@test "the plan creates every folder before the move that needs it" {
|
||||||
plan
|
plan
|
||||||
python3 - "$OUT" "$LIB" "$MOVIES" "$PARK" <<'PY'
|
python3 - "$OUT" "$LIB" "$MOVIES" "$PARK" <<'PY'
|
||||||
|
|||||||
+72
-2
@@ -195,7 +195,7 @@ setup() {
|
|||||||
--share "$SHARE" --container /data
|
--share "$SHARE" --container /data
|
||||||
[ "$status" -eq 1 ]
|
[ "$status" -eq 1 ]
|
||||||
[ "${lines[3]}" = "missing from the DB: 2" ]
|
[ "${lines[3]}" = "missing from the DB: 2" ]
|
||||||
[ "${lines[4]}" = "missing by extension: .log 1, .mkv 1" ]
|
[[ "$output" == *"missing by extension: .log 1, .mkv 1"* ]]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "--ignore-ext keeps a file the server never indexes out of the verdict" {
|
@test "--ignore-ext keeps a file the server never indexes out of the verdict" {
|
||||||
@@ -208,5 +208,75 @@ setup() {
|
|||||||
[ "$status" -eq 0 ]
|
[ "$status" -eq 0 ]
|
||||||
[ "${lines[1]}" = "landed: 1" ]
|
[ "${lines[1]}" = "landed: 1" ]
|
||||||
[ "${lines[3]}" = "missing from the DB: 0" ]
|
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||||
[ "${lines[4]}" = "not indexed (extension ignored): 1" ]
|
[[ "$output" == *"not indexed (extension ignored): 1"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
# wal_case <db> — a Jellyfin database whose NEW row lives only in the
|
||||||
|
# write-ahead log: the server has committed the rename but not checkpointed it,
|
||||||
|
# which is the normal state of a library a scan is writing. The holder
|
||||||
|
# connection keeps the WAL from folding into the main file, and the verifier
|
||||||
|
# runs as a real subprocess so it faces the same files Plex and Jellyfin leave.
|
||||||
|
wal_case() {
|
||||||
|
python3 - "$JFDB" "$JOURNAL" "$SHARE" "$VERIFY" <<'PY'
|
||||||
|
import sqlite3, subprocess, sys
|
||||||
|
db, journal, share, verify = sys.argv[1:5]
|
||||||
|
w = sqlite3.connect(db)
|
||||||
|
w.execute("pragma journal_mode=wal")
|
||||||
|
w.execute("pragma wal_autocheckpoint=0")
|
||||||
|
w.execute("insert into BaseItems (Path, Type) values ('/data/Movies/Old.mkv', 'x')")
|
||||||
|
w.commit()
|
||||||
|
w.execute("pragma wal_checkpoint(truncate)")
|
||||||
|
holder = sqlite3.connect(db) # last-connection close would checkpoint
|
||||||
|
w.execute("update BaseItems set Path='/data/Movies/Old/Old.mkv'")
|
||||||
|
w.commit()
|
||||||
|
r = subprocess.run([sys.executable, verify, "--journal", journal,
|
||||||
|
"--db-kind", "jellyfin", "--db", db,
|
||||||
|
"--share", share, "--container", "/data"],
|
||||||
|
capture_output=True, text=True)
|
||||||
|
print(r.stdout, end="")
|
||||||
|
sys.exit(r.returncode)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a row the server committed but has not checkpointed is proof, not a miss" {
|
||||||
|
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||||
|
run wal_case
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ "${lines[1]}" = "landed: 1" ]
|
||||||
|
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
# The DB alone cannot tell a lost file from a sidecar: both are "no row at the
|
||||||
|
# destination". The tool therefore stats the two paths and separates the classes
|
||||||
|
# the runbook names, so an operator never reads a `.pdf` as a lost film or,
|
||||||
|
# worse, reads a real loss as a sidecar.
|
||||||
|
@test "a file the database does not name that still sits at src is the pass's own damage" {
|
||||||
|
mkdir -p "$SHARE/Movies"
|
||||||
|
echo film > "$SHARE/Movies/Old.mkv"
|
||||||
|
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||||
|
|
||||||
|
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||||
|
--share "$SHARE" --container /data
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[ "${lines[3]}" = "missing from the DB: 1" ]
|
||||||
|
[ "${lines[4]}" = " on disk at dst, no row names it: 0" ]
|
||||||
|
[ "${lines[5]}" = " REAL — file still at src: 1" ]
|
||||||
|
[ "${lines[6]}" = " gone — at neither path: 0" ]
|
||||||
|
[[ "$output" == *"real: $SHARE/Movies/Old.mkv"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a file at neither path is gone, and a file at dst the server ignores is not a loss" {
|
||||||
|
move_row "$SHARE/Movies/Gone.mkv" "$SHARE/Movies/Gone/Gone.mkv"
|
||||||
|
mkdir -p "$SHARE/Movies/Sidecar"
|
||||||
|
echo cover > "$SHARE/Movies/Sidecar/cover.jpg"
|
||||||
|
move_row "$SHARE/Movies/Album/cover.jpg" "$SHARE/Movies/Sidecar/cover.jpg"
|
||||||
|
|
||||||
|
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||||
|
--share "$SHARE" --container /data
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
|
[ "${lines[3]}" = "missing from the DB: 2" ]
|
||||||
|
[ "${lines[4]}" = " on disk at dst, no row names it: 1" ]
|
||||||
|
[ "${lines[5]}" = " REAL — file still at src: 0" ]
|
||||||
|
[ "${lines[6]}" = " gone — at neither path: 1" ]
|
||||||
|
[[ "$output" == *"gone: $SHARE/Movies/Gone/Gone.mkv"* ]]
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user