Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c95ae20bfb | ||
|
|
a0dfa61623 | ||
|
|
7ee831adee | ||
|
|
f430e2264d | ||
|
|
82a2183d40 | ||
|
|
c70f9b3f6d | ||
|
|
32b8901bff | ||
|
|
6d03bbc8e7 | ||
|
|
32b58dd787 | ||
|
|
0a365274b6 | ||
|
|
e531e90278 | ||
|
|
c3ab565d9b |
No files matched your search
+58
-45
@@ -21,11 +21,12 @@ scripts/media-plan.py and scripts/media-shows.py read them instead of touching m
|
||||
the titles Plex already matched — the authority over any filename guess
|
||||
|
||||
jellyfin file <TAB> series <TAB> year <TAB> season <TAB> episode <TAB> extra
|
||||
<TAB> tmdb (see JF_COLS)
|
||||
<TAB> tmdb <TAB> title <TAB> tvdb (see JF_COLS)
|
||||
one row per episode Jellyfin has matched and numbered, its path mapped
|
||||
from the container's view back to the host; `year` and `tmdb` are the
|
||||
SERIES' values, `extra` is Jellyfin's own extra type when the row is
|
||||
bonus material rather than an episode
|
||||
from the container's view back to the host; `year`, `tmdb` and `tvdb`
|
||||
are the SERIES' values, `title` is the episode title the row carries,
|
||||
`extra` is Jellyfin's own extra type when the row is bonus material
|
||||
rather than an episode
|
||||
|
||||
music file <TAB> artist <TAB> album <TAB> year <TAB> disc <TAB> track <TAB> title
|
||||
<TAB> mb <TAB> performer (see MUSIC_COLS)
|
||||
@@ -83,13 +84,14 @@ left join metadata_items par on par.id = mi.parent_id
|
||||
where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
||||
"""
|
||||
JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db"
|
||||
JF_COLS = ["file", "series", "year", "season", "episode", "extra", "tmdb"]
|
||||
JF_COLS = ["file", "series", "year", "season", "episode", "extra",
|
||||
"tmdb", "title", "tvdb"]
|
||||
# Jellyfin stores the entity's .NET class as its type; the TV tree is these three.
|
||||
JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode",
|
||||
"MediaBrowser.Controller.Entities.TV.Season",
|
||||
"MediaBrowser.Controller.Entities.TV.Series")
|
||||
JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, IsVirtualItem
|
||||
from BaseItems where Type = ? and Path like ?""")
|
||||
JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType,
|
||||
IsVirtualItem, Name from BaseItems where Type = ? and Path like ?""")
|
||||
|
||||
# The music tree is these two entity classes. The track row carries the tags
|
||||
# Jellyfin read out of the file; the album row carries what the provider matched,
|
||||
@@ -128,26 +130,6 @@ where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
||||
and al.deleted_at is null
|
||||
"""
|
||||
|
||||
def open_plex_ro(db):
|
||||
"""Plex's library, read the way Plex writes it.
|
||||
|
||||
`immutable=1` tells sqlite the file never changes, so it skips the
|
||||
write-ahead log: every row a scan has committed but not yet checkpointed
|
||||
is invisible, and reading a live database that way can fail outright as
|
||||
"database disk image is malformed". Plain ro reads the WAL through its
|
||||
-shm; immutable is only the fallback for a database whose -shm cannot be
|
||||
read. The same law scripts/media-verify.py states.
|
||||
"""
|
||||
last = None
|
||||
for opt in ("mode=ro", "mode=ro&immutable=1"):
|
||||
try:
|
||||
return sqlite3.connect(f"file:{db}?{opt}", uri=True)
|
||||
except sqlite3.Error as e:
|
||||
last = e
|
||||
sys.exit(f"cannot open the Plex database read-only ({db}): {last}\n"
|
||||
"Plex runs rootless here: run this as the container's owner, never "
|
||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||
|
||||
|
||||
def enumerate_lib(root, out):
|
||||
"""One row per real file under root: size, extension, path."""
|
||||
@@ -247,7 +229,13 @@ def probe(enum, out, jobs):
|
||||
|
||||
def plex(root, db, out):
|
||||
"""Plex's matched titles for files under root, from its DB, read-only."""
|
||||
con = open_plex_ro(db)
|
||||
uri = f"file:{db}?mode=ro&immutable=1"
|
||||
try:
|
||||
con = sqlite3.connect(uri, uri=True)
|
||||
except sqlite3.Error as e:
|
||||
sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n"
|
||||
"Plex runs rootless here: run this as the container's owner, never "
|
||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||
with open(out, "w", newline="") as fh:
|
||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||
w.writerow(PLEX_COLS)
|
||||
@@ -268,9 +256,10 @@ def jellyfin(root, db, container, out):
|
||||
|
||||
The numbering is what matters, not the name: a release group's file name says
|
||||
nothing about which episode a file is, and Jellyfin has already answered that
|
||||
question against a provider. Season/episode come from the episode row; the
|
||||
series name, its year and its TMDb id come from the Series row the episode
|
||||
climbs to, because those are the only place Jellyfin keeps them.
|
||||
question against a provider. Season/episode and the episode's own title come
|
||||
from the episode row; the series name, its year and its TMDb and Tvdb ids come
|
||||
from the Series row the episode climbs to, because those are the only place
|
||||
Jellyfin keeps them.
|
||||
"""
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||
@@ -291,21 +280,38 @@ def jellyfin(root, db, container, out):
|
||||
" where Type in (?,?,?)", JF_TV):
|
||||
node[i] = (pid, typ, name, year, data)
|
||||
|
||||
# Jellyfin 12 keeps provider ids in their own table; older builds kept them in
|
||||
# the row's Data blob. Both shapes are read, because the tvdb id the series
|
||||
# naming depends on is wherever this build happened to write it.
|
||||
prov = {}
|
||||
if "BaseItemProviders" in {r[0] for r in con.execute(
|
||||
"select name from sqlite_master where type='table'")}:
|
||||
for iid, pid, val in con.execute(
|
||||
"select ItemId, ProviderId, ProviderValue from BaseItemProviders"
|
||||
" where ProviderId in ('Tmdb', 'Tvdb')"):
|
||||
prov[(iid, pid)] = val
|
||||
|
||||
def provider_of(item_id, want):
|
||||
got = prov.get((item_id, want))
|
||||
if got:
|
||||
return str(got)
|
||||
try:
|
||||
blob = json.loads(node[item_id][4] or "{}").get("ProviderIds") or {}
|
||||
except (KeyError, ValueError, TypeError):
|
||||
return ""
|
||||
return str(blob.get(want, "") or "")
|
||||
|
||||
def series_of(item_id):
|
||||
"""Climb episode -> season -> series; empty when the chain is broken."""
|
||||
for _ in range(4):
|
||||
if item_id not in node:
|
||||
return "", "", ""
|
||||
pid, typ, name, year, data = node[item_id]
|
||||
return "", "", "", ""
|
||||
pid, typ = node[item_id][0], node[item_id][1]
|
||||
if typ.endswith(".Series"):
|
||||
try:
|
||||
tmdb = (json.loads(data or "{}")
|
||||
.get("ProviderIds") or {}).get("Tmdb", "")
|
||||
except ValueError:
|
||||
tmdb = ""
|
||||
return name or "", year or "", tmdb
|
||||
return (node[item_id][2] or "", node[item_id][3] or "",
|
||||
provider_of(item_id, "Tmdb"), provider_of(item_id, "Tvdb"))
|
||||
item_id = pid
|
||||
return "", "", ""
|
||||
return "", "", "", ""
|
||||
|
||||
# The fleet's quadlets bind the share at /data, so the container's view of
|
||||
# this library is /data/<the library's own folder name> unless told otherwise.
|
||||
@@ -316,17 +322,18 @@ def jellyfin(root, db, container, out):
|
||||
with open(out, "w", newline="") as fh:
|
||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||
w.writerow(JF_COLS)
|
||||
for iid, path, idx, pidx, extra, virt in con.execute(JF_SQL,
|
||||
(JF_TV[0], prefix + "%")):
|
||||
for iid, path, idx, pidx, extra, virt, epname in con.execute(
|
||||
JF_SQL, (JF_TV[0], prefix + "%")):
|
||||
if virt or not path or not path.startswith(prefix):
|
||||
skipped += 1 # a virtual item is a variant, not a file
|
||||
continue
|
||||
sname, syear, tmdb = series_of(iid)
|
||||
sname, syear, tmdb, tvdb = series_of(iid)
|
||||
w.writerow([root + "/" + path[len(prefix):],
|
||||
sname.replace("\t", " "), syear,
|
||||
"" if pidx is None else pidx,
|
||||
"" if idx is None else idx,
|
||||
(extra or "").replace("\t", " "), tmdb])
|
||||
(extra or "").replace("\t", " "), tmdb,
|
||||
(epname or "").replace("\t", " "), tvdb])
|
||||
written += 1
|
||||
print(f"episodes={written} virtual-or-blank={skipped} -> {out}")
|
||||
|
||||
@@ -419,7 +426,13 @@ def audiobooks(root, db, out):
|
||||
at least one un-indexed chapter, so the relayout that reads this file must
|
||||
not renumber anything from it.
|
||||
"""
|
||||
con = open_plex_ro(db)
|
||||
uri = f"file:{db}?mode=ro&immutable=1"
|
||||
try:
|
||||
con = sqlite3.connect(uri, uri=True)
|
||||
except sqlite3.Error as e:
|
||||
sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n"
|
||||
"Plex runs rootless here: run this as the container's owner, never "
|
||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||
tables = {r[0] for r in con.execute(
|
||||
"select name from sqlite_master where type='table'")}
|
||||
missing = {"media_parts", "media_items", "metadata_items"} - tables
|
||||
|
||||
+349
-45
@@ -6,29 +6,40 @@ r"""Plan a Jellyfin-format relayout of a TV library. DRY RUN ONLY: nothing moves
|
||||
[--out TSV] [--exceptions TXT]
|
||||
|
||||
Inputs are the TSVs scripts/media-census.py exports: `enumerate` lists what
|
||||
exists, `jellyfin` says which episode each file IS, and `probe` is optional
|
||||
duration evidence for the movie rule. The TMDb cache is a plain JSON file so a
|
||||
plan stays reproducible: answers are read from it, and a miss is looked up only
|
||||
with --tmdb and TMDB_API_KEY in the environment (the key never enters argv, and
|
||||
exists, `jellyfin` says which episode each file IS (its series' TMDb and Tvdb
|
||||
ids, and the episode title the row carries), and `probe` is optional duration
|
||||
evidence for the movie rule. Every name and title answer lives in the cache JSON
|
||||
so a plan stays reproducible: answers are read from it, and a miss is looked up
|
||||
only with --tmdb/--tvdb and their credentials (a key never enters argv, and
|
||||
never the cache).
|
||||
|
||||
Target contract (jellyfin.org/docs/general/server/media/shows):
|
||||
|
||||
<root>/<Series> (<year>)/Season <NN>/<Series> SxxEyy.ext
|
||||
<root>/<Series> (<year>)/Season <NN>/<Series> (<year>) - SxxEyy - <Title>.ext
|
||||
one file holding two episodes SxxE01-E02 specials Season 00
|
||||
a documented part of an episode SxxE03 Part 1 sidecars <base>.<lang>.srt
|
||||
bonus material <series>/trailers|featurettes|… — never a season
|
||||
|
||||
The series name and the episode title are the ones the IDENTITY authority — tvdb
|
||||
— states for the id the metadata server matched, not the words a release folder
|
||||
or a server's folder-derived Series name happens to carry.
|
||||
|
||||
PLAN is the TSV scripts/media-apply.py replays: `action src dst kind`, every
|
||||
mkdir row before any move, and `keep` for a file already where the contract puts
|
||||
it. EXCEPTIONS is the review list: everything parked, and why.
|
||||
|
||||
The authority chain, strongest first, for one video file:
|
||||
|
||||
1. a provider row — Jellyfin matched this exact path and numbered it.
|
||||
1. a provider row — Jellyfin matched this exact path and numbered it. The
|
||||
row's NAME is only evidence once an id backs it: the server names a Series
|
||||
after the folder it scanned, so `24 (2001) Season 1` is a folder echo, and
|
||||
the series' tvdb id is what undoes it.
|
||||
2. the file's own SxxEyy / 1x04 marker, the series taken from a sibling's
|
||||
provider row, else from the nearest ancestor folder that has one, else from
|
||||
the release folder's own words.
|
||||
a registered show its own words name, else — outside the review pile — from
|
||||
the release folder's words. Inside a hidden top-level folder a file may
|
||||
MATCH a registered show but never INVENT one: invention is what filled the
|
||||
pile to begin with.
|
||||
3. nothing. An unnumbered file is PARKED under --park with its relative path
|
||||
kept: an invented `Season 01` steals an episode number from the file that
|
||||
really is S01E01, and a wrong number is worse than no number.
|
||||
@@ -47,8 +58,11 @@ Guards, and why each is here:
|
||||
* Series art and the show nfo follow the SERIES, a season image follows its
|
||||
SEASON, a subtitle or episode nfo follows the episode's new base — an episode
|
||||
moved without its subtitles is a half-move.
|
||||
* Hidden files, macOS forks, qBittorrent resume parts and the park pile itself
|
||||
are never moved. Skipping the park is what makes re-planning stable.
|
||||
* Hidden files, macOS forks, qBittorrent resume parts are never moved. A file
|
||||
already inside --park is never moved either: that is what makes re-planning
|
||||
stable. --park belongs OUTSIDE the library — parked files inside it get
|
||||
indexed as one show named after the park folder, and a row like that is a
|
||||
folder echo, never evidence (the guard in `row_names_the_park`).
|
||||
|
||||
The plan crosses three trees (the library, its park, the movie library), so the
|
||||
executor runs with --root at the share that holds them, on its own journal —
|
||||
@@ -56,7 +70,8 @@ one journal is one undo:
|
||||
|
||||
python3 scripts/media-shows.py --root /mnt/entropy/TVShows \
|
||||
--enum /tmp/tv-enum.tsv --titles /tmp/jellyfin-tv.tsv \
|
||||
--out /mnt/entropy/media-tv-plan.tsv
|
||||
--tvdb --tmdb --out /mnt/entropy/media-tv-plan.tsv \
|
||||
--park /mnt/entropy/media-park/TVShows
|
||||
python3 scripts/media-apply.py apply --plan /mnt/entropy/media-tv-plan.tsv \
|
||||
--root /mnt/entropy --journal /mnt/entropy/media-tv-relayout.jsonl
|
||||
"""
|
||||
@@ -80,14 +95,21 @@ ARGS.add_argument("--titles", default="/tmp/jellyfin-tv.tsv",
|
||||
help="the provider's numbering (media-census.py jellyfin)")
|
||||
ARGS.add_argument("--probed", default="",
|
||||
help="ffprobe census for the movie rule; absent = no durations")
|
||||
ARGS.add_argument("--tmdb-cache", default="/tmp/tmdb-cache.json",
|
||||
help="TMDb answers as an input file, so a plan is reproducible")
|
||||
ARGS.add_argument("--tmdb-cache", default="/tmp/tv-names.json",
|
||||
help="name answers (TMDb and Tvdb) as ONE input file, so a plan "
|
||||
"is reproducible; keys carry the provider they came from")
|
||||
ARGS.add_argument("--tmdb", action="store_true",
|
||||
help="look up cache misses (needs TMDB_API_KEY)")
|
||||
help="look up cache misses at TMDb (needs TMDB_API_KEY)")
|
||||
ARGS.add_argument("--tvdb", action="store_true",
|
||||
help="look up cache misses at Tvdb (needs the credentials file)")
|
||||
ARGS.add_argument("--tvdb-credentials",
|
||||
default=os.path.expanduser("~/.config/secrets/tvdb"),
|
||||
help="file with an api_key= line (and a pin= line for a "
|
||||
"user-issued key); mode 600, and never the cache")
|
||||
ARGS.add_argument("--movies", default="/mnt/entropy/Movies",
|
||||
help="movie library a mis-filed film moves into")
|
||||
ARGS.add_argument("--park", default="",
|
||||
help="review pile (default: <root>/_unsorted)")
|
||||
help="review pile OUTSIDE the library (default: <root>/_unsorted)")
|
||||
ARGS.add_argument("--out", default="/tmp/tv-plan.tsv")
|
||||
ARGS.add_argument("--exceptions", default="/tmp/tv-exceptions.txt")
|
||||
ARGS = ARGS.parse_args()
|
||||
@@ -105,6 +127,7 @@ try:
|
||||
except (OSError, ValueError):
|
||||
CACHE = {}
|
||||
API_KEY = os.environ.get("TMDB_API_KEY", "")
|
||||
TMDB = {"tries": 0, "failed": 0}
|
||||
|
||||
VIDEO_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg",
|
||||
".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".flv", ".vob", ".rm",
|
||||
@@ -218,6 +241,20 @@ def leading_index(name):
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def tmdb_request(url):
|
||||
"""One TMDb call, authenticated. The two credential shapes are NOT
|
||||
interchangeable and the difference is a 401 on every single call: a v3 key
|
||||
travels as the `api_key` query parameter, and only a v4 request token is
|
||||
accepted as a Bearer header. A token in argv is world-readable in
|
||||
/proc/<pid>/cmdline — this URL is built inside the process and never reaches
|
||||
argv, so the query form is safe here."""
|
||||
TMDB["tries"] += 1
|
||||
if API_KEY.count(".") == 2: # three segments: a v4 JWT
|
||||
return http_json(url, headers={"Authorization": "Bearer " + API_KEY})
|
||||
return http_json(url + ("&" if "?" in url else "?") + "api_key="
|
||||
+ urllib.parse.quote(API_KEY))
|
||||
|
||||
|
||||
def tmdb(kind, query):
|
||||
"""A provider answer for a query: the cache first, TMDb once, then never again."""
|
||||
ck = f"{kind}:{query}"
|
||||
@@ -231,26 +268,155 @@ def tmdb(kind, query):
|
||||
path = "movie" if kind == "movie" else "tv"
|
||||
url = ("https://api.themoviedb.org/3/search/" + path + "?query="
|
||||
+ urllib.parse.quote(query) + "&include_adult=false")
|
||||
req = urllib.request.Request(url, headers={
|
||||
# The key rides a header, never the URL: a token in argv is world-readable.
|
||||
"Authorization": "Bearer " + API_KEY,
|
||||
"Accept": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as r:
|
||||
found = json.load(r).get("results") or []
|
||||
found = tmdb_request(url).get("results") or []
|
||||
except Exception as e: # noqa: BLE001
|
||||
TMDB["failed"] += 1
|
||||
print(f"tmdb {kind} lookup failed for {query!r}: {e}", file=sys.stderr)
|
||||
return None
|
||||
if not found:
|
||||
CACHE[ck] = None
|
||||
return None
|
||||
top = found[0]
|
||||
# The two search endpoints date their rows differently: movies carry
|
||||
# `release_date`, tv carries `first_air_date`. Reading only one of them leaves
|
||||
# every film answer year-less, and a year-less answer can never be refused by
|
||||
# the year guard below.
|
||||
when = str(top.get("year") or "") or \
|
||||
str(top.get("release_date") or top.get("first_air_date") or "")[:4]
|
||||
CACHE[ck] = {"name": (top.get("title") or top.get("name") or "").strip(),
|
||||
"year": str(top.get("year") or "") or (top.get("first_air_date") or "")[:4],
|
||||
"id": top.get("id")}
|
||||
"year": when, "id": top.get("id")}
|
||||
return CACHE[ck]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- tvdb, the identity
|
||||
# tvdb names the show and the episode: it is the only provider whose strings are
|
||||
# the ones the operator asked for, so its answer replaces a folder-derived Series
|
||||
# name. The login mints a JWT per run from a mode-600 file; neither the key nor
|
||||
# the token reaches argv, the cache, or the plan. A provider that will not answer
|
||||
# never invents a name: every lookup here returns None, and None keeps the
|
||||
# provider's own words and lands in the exceptions.
|
||||
TVDB = {"jwt": None, "dead": False}
|
||||
|
||||
|
||||
def http_json(url, body=None, headers=None):
|
||||
req = urllib.request.Request(
|
||||
url, data=json.dumps(body).encode() if body is not None else None,
|
||||
headers={"Accept": "application/json", "Content-Type": "application/json",
|
||||
**(headers or {})})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def tvdb_login():
|
||||
"""The run's JWT, or None: a project key needs only api_key=, a user-issued
|
||||
key also needs the pin."""
|
||||
if TVDB["jwt"] is not None:
|
||||
return TVDB["jwt"] or None
|
||||
TVDB["jwt"] = ""
|
||||
try:
|
||||
lines = open(ARGS.tvdb_credentials).read().splitlines()
|
||||
except OSError:
|
||||
print(f"tvdb: no credentials at {ARGS.tvdb_credentials}", file=sys.stderr)
|
||||
return None
|
||||
sec = {}
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
sec[k.strip()] = v.strip()
|
||||
if not sec.get("api_key"):
|
||||
print(f"tvdb: no api_key in {ARGS.tvdb_credentials}", file=sys.stderr)
|
||||
return None
|
||||
body = {"apiKey": sec["api_key"]}
|
||||
if sec.get("pin"):
|
||||
body["token"] = sec["pin"]
|
||||
try:
|
||||
TVDB["jwt"] = (http_json("https://api4.thetvdb.com/v4/login", body)
|
||||
.get("data") or {}).get("token") or ""
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"tvdb login failed: {e}", file=sys.stderr)
|
||||
return None
|
||||
return TVDB["jwt"] or None
|
||||
|
||||
|
||||
def tvdb(path):
|
||||
"""One Tvdb v4 GET, or None when it cannot be answered."""
|
||||
if not ARGS.tvdb or TVDB["dead"]:
|
||||
return None
|
||||
jwt = tvdb_login()
|
||||
if not jwt:
|
||||
TVDB["dead"] = True
|
||||
return None
|
||||
try:
|
||||
return http_json("https://api4.thetvdb.com/v4" + path,
|
||||
headers={"Authorization": "Bearer " + jwt}).get("data")
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"tvdb {path} failed: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def tvdb_series(sid):
|
||||
"""{name, year} for a tvdb series id."""
|
||||
ck = f"tvdb:series:{sid}"
|
||||
if ck in CACHE:
|
||||
return CACHE[ck]
|
||||
data = tvdb(f"/series/{sid}")
|
||||
if not data:
|
||||
return None
|
||||
CACHE[ck] = {"name": (data.get("name") or "").strip(),
|
||||
"year": (data.get("firstAired") or "")[:4]}
|
||||
return CACHE[ck]
|
||||
|
||||
|
||||
def tvdb_id_for_tmdb(mid):
|
||||
"""The tvdb id TMDb records for a series id — the bridge for the bulk of this
|
||||
library, which the server matched by TMDb."""
|
||||
ck = f"tmdb:external:{mid}"
|
||||
if ck in CACHE:
|
||||
return CACHE[ck] or ""
|
||||
if not (ARGS.tmdb and API_KEY):
|
||||
return ""
|
||||
try:
|
||||
got = tmdb_request("https://api.themoviedb.org/3/tv/" + mid + "/external_ids")
|
||||
except Exception as e: # noqa: BLE001
|
||||
TMDB["failed"] += 1
|
||||
print(f"tmdb external ids failed for {mid}: {e}", file=sys.stderr)
|
||||
return ""
|
||||
CACHE[ck] = str(got.get("tvdb_id") or "")
|
||||
return CACHE[ck]
|
||||
|
||||
|
||||
def tvdb_search(name):
|
||||
"""The tvdb series a provider name states, or None: only tvdb's own spelling
|
||||
counts, so a near-miss never renames a show."""
|
||||
ck = f"tvdb:search:{name}"
|
||||
if ck in CACHE:
|
||||
return CACHE[ck]
|
||||
for hit in tvdb("/search?query=" + urllib.parse.quote(name) + "&type=series") or []:
|
||||
if eng.norm(hit.get("name") or "") != eng.norm(name):
|
||||
continue
|
||||
CACHE[ck] = {"name": (hit.get("name") or "").strip(),
|
||||
"year": str(hit.get("year") or ""),
|
||||
"id": str(hit.get("tvdb_id") or "")}
|
||||
return CACHE[ck]
|
||||
CACHE[ck] = None
|
||||
return None
|
||||
|
||||
|
||||
def tvdb_episodes(sid):
|
||||
"""`season/episode` -> title for one series, in the one call tvdb answers."""
|
||||
ck = f"tvdb:episodes:{sid}"
|
||||
if ck in CACHE:
|
||||
return CACHE[ck]
|
||||
data = tvdb(f"/series/{sid}/episodes/default")
|
||||
if not data:
|
||||
return None
|
||||
CACHE[ck] = {f"{e.get('seasonNumber')}/{e.get('number')}":
|
||||
(e.get("name") or "").strip()
|
||||
for e in data.get("episodes") or []}
|
||||
return CACHE[ck]
|
||||
|
||||
# ------------------------------------------------------------------- the inputs
|
||||
rows = []
|
||||
for line in open(ARGS.enum):
|
||||
@@ -264,6 +430,26 @@ for line in open(ARGS.enum):
|
||||
rows.append({"bytes": int(size), "ext": ext.lower(), "path": p, "base": base})
|
||||
held = {r["path"]: r for r in rows}
|
||||
|
||||
def row_names_the_park(path, name):
|
||||
"""The library's own review pile is not a show. When the park sat INSIDE the
|
||||
library the server indexed the whole folder as one series named after it, and
|
||||
a row like that would rename a real show to the pile — so a row whose series is
|
||||
the hidden top-level folder the file itself sits in is dropped. The server also
|
||||
sanitizes the display name, dropping the leading underscore, so `unsorted` and
|
||||
`_unsorted` are the same pile: compare them without their leading punctuation,
|
||||
or the pile survives as a visible series folder."""
|
||||
top = rel(path).split("/")[0]
|
||||
return top.startswith("_") and eng.norm(name.lstrip("_")) == eng.norm(top.lstrip("_"))
|
||||
|
||||
|
||||
def in_review_pile(path):
|
||||
"""A file under the library's hidden top-level folder is review material by
|
||||
that folder's own convention: its filename may MATCH a show a provider names
|
||||
somewhere else, but it may never INVENT one — inventing one from a file's own
|
||||
words is what filled the pile to begin with."""
|
||||
return rel(path).split("/")[0].startswith("_")
|
||||
|
||||
|
||||
titles = {}
|
||||
with open(ARGS.titles) as fh:
|
||||
cols = fh.readline().rstrip("\n").split("\t")
|
||||
@@ -272,7 +458,7 @@ with open(ARGS.titles) as fh:
|
||||
f += [""] * (len(cols) - len(f))
|
||||
row = dict(zip(cols, f))
|
||||
p = os.path.abspath(row.get("file", ""))
|
||||
if p in held:
|
||||
if p in held and not row_names_the_park(p, row.get("series", "")):
|
||||
titles[p] = row
|
||||
|
||||
dur = {}
|
||||
@@ -379,14 +565,17 @@ def series_name(name):
|
||||
return clean(re.sub(r"\s{2,}", " ", PACK_IDX_RE.sub("", name)).strip(" ."))
|
||||
|
||||
|
||||
def note(name, year="", tmdb_id=""):
|
||||
"""Register a series; the first name wins, the first year and id stick."""
|
||||
def note(name, year="", tmdb_id="", tvdb_id=""):
|
||||
"""Register a series; the first name wins, the first year and ids stick."""
|
||||
k = eng.norm(name)
|
||||
s = series.setdefault(k, {"name": series_name(name), "year": "", "tmdb": ""})
|
||||
s = series.setdefault(k, {"name": series_name(name), "year": "", "tmdb": "",
|
||||
"tvdb": ""})
|
||||
if year and not s["year"]:
|
||||
s["year"] = str(year)
|
||||
if tmdb_id and not s["tmdb"]:
|
||||
s["tmdb"] = str(tmdb_id)
|
||||
if tvdb_id and not s["tvdb"]:
|
||||
s["tvdb"] = str(tvdb_id)
|
||||
|
||||
|
||||
|
||||
@@ -413,7 +602,9 @@ for r in video:
|
||||
# every file under SNL 16-20/Season 16..19 carries season 16 episode 20.
|
||||
owner = (t["series"] if t and t["series"]
|
||||
else series_of_dir(os.path.dirname(p))
|
||||
or title_of(r["base"][:marked[2]])[0])
|
||||
or known_in(r["base"])
|
||||
or ("" if in_review_pile(p)
|
||||
else title_of(r["base"][:marked[2]])[0]))
|
||||
if not eng.norm(owner).replace(" ", ""):
|
||||
unknown.append((r, "an episode marker with no series name to hang it on"))
|
||||
continue
|
||||
@@ -427,7 +618,8 @@ for r in video:
|
||||
continue
|
||||
eps = [ep]
|
||||
claimants.append((r, owner, season, eps))
|
||||
note(owner, t["year"] if t else "", t["tmdb"] if t else "")
|
||||
note(owner, t["year"] if t else "", t["tmdb"] if t else "",
|
||||
t["tvdb"] if t else "")
|
||||
continue
|
||||
owner = series_of_dir(os.path.dirname(p))
|
||||
if fs is not None and owner and leading_index(r["base"]):
|
||||
@@ -450,17 +642,84 @@ for k, s in series.items():
|
||||
if got and eng.norm(got["name"]) == k:
|
||||
s["year"] = got["year"]
|
||||
|
||||
if ARGS.tmdb:
|
||||
# tvdb is the identity authority. The row's own tvdb id is the door; a TMDb id
|
||||
# crosses through TMDb's external ids; a bare name counts only when tvdb spells
|
||||
# it the same and the years do not disagree. A series nothing resolves keeps the
|
||||
# provider's words AND says so in the exceptions, which is where the operator
|
||||
# finds the folders the plan did not rename to authority.
|
||||
for s in series.values():
|
||||
got = tvdb_series(s["tvdb"]) if s["tvdb"] else None
|
||||
if not got and s["tmdb"]:
|
||||
sid = tvdb_id_for_tmdb(s["tmdb"])
|
||||
if sid:
|
||||
s["tvdb"] = sid
|
||||
got = tvdb_series(sid)
|
||||
if not got:
|
||||
hit = tvdb_search(s["name"])
|
||||
if hit and hit["id"] and (not s["year"] or not hit["year"]
|
||||
or hit["year"] == s["year"]):
|
||||
s["tvdb"] = hit["id"]
|
||||
got = hit
|
||||
if got and got["name"]:
|
||||
s["canon"], s["canon_year"] = got["name"], got["year"]
|
||||
|
||||
# One identity, one folder: the provider's spellings collapse to the shows tvdb
|
||||
# says they are, so a box set split over five release folders is ONE series and
|
||||
# two files claiming one episode MEET inside the duplicate rule below.
|
||||
alias, canon = {}, {}
|
||||
for k, s in series.items():
|
||||
key = f"id:{s['tvdb']}" if s["tvdb"] else k
|
||||
alias[k] = key
|
||||
keep = canon.setdefault(key, s)
|
||||
if keep is not s:
|
||||
for f in ("year", "tvdb", "canon", "canon_year"):
|
||||
if not keep.get(f) and s.get(f):
|
||||
keep[f] = s[f]
|
||||
|
||||
if ARGS.tmdb or ARGS.tvdb:
|
||||
with open(ARGS.tmdb_cache, "w") as fh:
|
||||
json.dump(CACHE, fh, indent=1, sort_keys=True)
|
||||
|
||||
|
||||
def identity(s):
|
||||
"""(name, year) a folder and its files carry: tvdb's whenever it answered,
|
||||
the provider's own words only for a series nothing resolved."""
|
||||
if s.get("canon"):
|
||||
return s["canon"], s["canon_year"] or s["year"]
|
||||
return s["name"], s["year"]
|
||||
|
||||
|
||||
def series_key(name):
|
||||
"""The merged identity a provider spelling belongs to."""
|
||||
return alias.get(eng.norm(name), eng.norm(name))
|
||||
|
||||
|
||||
def label_of(s):
|
||||
"""`<Series> (<year>)`, the form both matchers read. A provider's name can
|
||||
arrive WITH the year already on it — TVDb hands back `Acapulco (2021)` for the
|
||||
reboot — and appending a second one names a folder nothing will match again."""
|
||||
nm, yr = identity(s)
|
||||
nm = clean(nm)
|
||||
if not yr or re.search(r"\(\d{4}\)\s*$", nm):
|
||||
return nm
|
||||
return f"{nm} ({yr})"
|
||||
|
||||
|
||||
def episode_title(r, s, season, ep):
|
||||
"""The title this episode carries: tvdb's for the id the series resolved to,
|
||||
else the title the matched row states, else nothing at all — a dangling ` - `
|
||||
is not a title, and an invented one is worse than none."""
|
||||
if s["tvdb"]:
|
||||
t = ((tvdb_episodes(s["tvdb"]) or {}).get(f"{season}/{ep}") or "").strip()
|
||||
if t:
|
||||
return clean(t)
|
||||
t = ((titles.get(r["path"]) or {}).get("title") or "").strip()
|
||||
return clean(t) if t else ""
|
||||
|
||||
|
||||
def series_dir(name):
|
||||
s = series.get(eng.norm(name))
|
||||
if not s:
|
||||
s = {"name": clean(name), "year": ""}
|
||||
label = f"{s['name']} ({s['year']})" if s["year"] else s["name"]
|
||||
return f"{ROOT}/{clean(label)}"
|
||||
s = canon.get(series_key(name))
|
||||
return f"{ROOT}/{label_of(s)}" if s else f"{ROOT}/{clean(series_name(name))}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- the plan
|
||||
@@ -500,25 +759,28 @@ def park(r, reason):
|
||||
by_ep = collections.defaultdict(list)
|
||||
for r, name, season, eps in claimants:
|
||||
for e in eps:
|
||||
by_ep[(eng.norm(name), season, e)].append(r)
|
||||
by_ep[(series_key(name), season, e)].append(r)
|
||||
winners, losers = set(), {}
|
||||
for (key, season, e), rs in by_ep.items():
|
||||
rs.sort(key=lambda r: (-r["bytes"], r["path"]))
|
||||
winners.add(rs[0]["path"])
|
||||
for r in rs[1:]:
|
||||
losers.setdefault(r["path"],
|
||||
f"duplicate of {series[key]['name']} S{season:02d}E{e:02d}"
|
||||
f"duplicate of {canon[key]['name']} S{season:02d}E{e:02d}"
|
||||
" (the bigger copy keeps the standard name)")
|
||||
for r, name, season, eps in claimants:
|
||||
if r["path"] in losers:
|
||||
park(r, losers[r["path"]])
|
||||
continue
|
||||
s = series[eng.norm(name)]
|
||||
label = f" S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps)
|
||||
s = canon[series_key(name)]
|
||||
stem = f"{label_of(s)} - S{season:02d}E" + "-E".join(f"{e:02d}" for e in eps)
|
||||
title = episode_title(r, s, season, eps[0])
|
||||
if title:
|
||||
stem += f" - {title}"
|
||||
part = PART_RE.search(os.path.splitext(r["base"])[0])
|
||||
tail = f" Part {part.group(1).upper()}" if part and len(eps) == 1 else ""
|
||||
book(r, f"{series_dir(name)}/Season {season:02d}", s["name"] + label + tail + r["ext"],
|
||||
"tv")
|
||||
if part and len(eps) == 1:
|
||||
stem += f" Part {part.group(1).upper()}"
|
||||
book(r, f"{series_dir(name)}/Season {season:02d}", stem + r["ext"], "tv")
|
||||
|
||||
for r, name, kind in extras:
|
||||
book(r, f"{series_dir(name)}/{kind}",
|
||||
@@ -530,8 +792,13 @@ for r, name, kind in extras:
|
||||
for r in films:
|
||||
parsed, year = title_of(r["base"])
|
||||
got = tmdb("movie", parsed)
|
||||
words = eng.norm(got["name"]).split() if got else []
|
||||
if not got or not eng.mentions(parsed.split(), words) \
|
||||
# The provider's title must be a run of words the FILE itself states. The query
|
||||
# is only what title_of could salvage from the name, and a short one ("russia")
|
||||
# lets the top hit be a different work entirely: a 1963 Bond film is not a 1991
|
||||
# documentary episode, and Special Edition I is not Special Edition III.
|
||||
said = eng.norm(os.path.splitext(r["base"])[0]).split()
|
||||
named = eng.norm(got["name"]).split() if got else []
|
||||
if not got or not eng.mentions(named, said) \
|
||||
or (year != "?" and got["year"] and got["year"] != year):
|
||||
park(r, "long file with no episode evidence, and no movie provider names it")
|
||||
continue
|
||||
@@ -655,6 +922,30 @@ for r, reason in unknown:
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- output
|
||||
def provider_broken():
|
||||
"""TMDb was asked for names and did not answer: the plan is not the plan the
|
||||
operator asked for, whatever the move count says."""
|
||||
return bool(ARGS.tmdb and (not API_KEY or TMDB["failed"]))
|
||||
|
||||
|
||||
def provider_note():
|
||||
"""Provider health on the summary line. The 401s are already on stderr, and a
|
||||
terminal that scrolled past them shows a plan that looks finished: the count
|
||||
belongs where it cannot be missed, because a half-resolved plan renames
|
||||
folders with the folder's own junk words."""
|
||||
if not (ARGS.tmdb or ARGS.tvdb):
|
||||
return ""
|
||||
if not ARGS.tmdb:
|
||||
return ", tmdb off (names from the cache and tvdb only)"
|
||||
if not API_KEY:
|
||||
return ", tmdb NOT ASKED (TMDB_API_KEY unset): NOT provider-complete"
|
||||
if not TMDB["tries"]:
|
||||
return ", tmdb cache-only (every name was already answered)"
|
||||
done = TMDB["tries"] - TMDB["failed"]
|
||||
return (f", tmdb {done}/{TMDB['tries']} answered"
|
||||
+ (" — NOT provider-complete" if TMDB["failed"] else ""))
|
||||
|
||||
|
||||
def write_plan():
|
||||
"""mkdir rows for every destination folder and its whole chain, then the moves.
|
||||
Parents first: the executor creates directories in the order they are listed."""
|
||||
@@ -676,16 +967,23 @@ def write_plan():
|
||||
sized = sum(held[s]["bytes"] for s, _d, _k in moves if s in held)
|
||||
print(f"plan: {len(moves) - len(parked)} moves, {len(parked)} parked"
|
||||
f" ({sized / 2**40:.2f} TiB touched), {len(keeps)} already placed,"
|
||||
f" {len(dirs)} folders -> {ARGS.out}")
|
||||
f" {len(dirs)} folders{provider_note()} -> {ARGS.out}")
|
||||
|
||||
|
||||
def write_exceptions():
|
||||
no_year = sorted(s["name"] for s in series.values() if not s["year"])
|
||||
no_year = sorted({s["name"] for s in canon.values() if not s["year"]})
|
||||
no_tvdb = sorted({identity(s)[0] for s in canon.values() if not s.get("canon")})
|
||||
with open(ARGS.exceptions, "w") as fh:
|
||||
fh.write(f"TV relayout exceptions for {ROOT}\n\n")
|
||||
fh.write(f"Everything below is parked under {PARK}, keeping the relative path it\n")
|
||||
fh.write("has today. Nothing is deleted and nothing is overwritten: a parked file\n")
|
||||
fh.write("is one rename from wherever the plan is pointed next.\n\n")
|
||||
if provider_broken():
|
||||
fh.write("WARNING: " + provider_note().strip(", ").replace(": NOT", " — NOT")
|
||||
+ "\n Names and episode titles that should have come from TMDb are\n"
|
||||
" missing, so folders keep the metadata provider's words. Fix the\n"
|
||||
" credential and re-plan: the cache keeps every answer that did\n"
|
||||
" arrive, so the re-run costs only the lookups that failed.\n\n")
|
||||
for reason in sorted(reasons, key=lambda r: -len(reasons[r])):
|
||||
rs = reasons[reason]
|
||||
fh.write(f"{len(rs):>6} files {reason}\n")
|
||||
@@ -698,6 +996,12 @@ def write_exceptions():
|
||||
fh.write(f"{len(no_year):>6} series carry no year from any provider, so their\n")
|
||||
fh.write(" folders are named without one; Jellyfin still matches them.\n")
|
||||
fh.write(" " + ", ".join(no_year[:300]) + "\n")
|
||||
if no_tvdb:
|
||||
fh.write(f"\n{len(no_tvdb):>6} series carry no tvdb identity, so their folders\n")
|
||||
fh.write(" keep the name the metadata provider gave them and no episode\n")
|
||||
fh.write(" title was taken. Nothing was invented: give these a resolvable\n")
|
||||
fh.write(" id (--tvdb) and re-plan; a rename is idempotent.\n")
|
||||
fh.write(" " + ", ".join(no_tvdb[:300]) + "\n")
|
||||
print(f"exceptions -> {ARGS.exceptions}")
|
||||
|
||||
|
||||
|
||||
+10
-45
@@ -9,13 +9,8 @@ its destination, and NO row may still name its source — a source row is a scan
|
||||
that has not caught up, and it is the one signature of a library pointing at
|
||||
files that no longer exist.
|
||||
|
||||
Two doors, both read-only, both without a lock while the server runs (there is
|
||||
no `sqlite3` binary on these hosts). Open `mode=ro` PLAIN: both servers keep
|
||||
their libraries in WAL mode, and `immutable=1` — which claims the file never
|
||||
changes — makes sqlite skip the write-ahead log entirely, so every row the
|
||||
server has committed but not yet checkpointed reads as MISSING, and a read
|
||||
beside a running scanner can even fail as "database disk image is malformed".
|
||||
`immutable=1` is only the fallback, for a database whose `-shm` cannot be read.
|
||||
Two doors, both read-only, both without a lock while the server runs
|
||||
(`mode=ro&immutable=1`; there is no `sqlite3` binary on these hosts):
|
||||
|
||||
jellyfin `BaseItems.Path`. The container binds the share at `/data`, so the
|
||||
database holds `/data/<library>/…` where the journal holds
|
||||
@@ -27,14 +22,9 @@ beside a running scanner can even fail as "database disk image is malformed".
|
||||
|
||||
Neither server holds a row for everything the journal renames: a `.log`, `.cue`
|
||||
or cover image has no row to land in, so its absence is not a fault. The report
|
||||
therefore breaks the missing rows down by extension, and `--ignore-ext` names the
|
||||
extensions this server does not index, which are then counted separately and kept
|
||||
out of the verdict. What is left is split by what the filesystem says, because
|
||||
the database alone cannot: a file still at its `src` is `REAL` (this pass's own
|
||||
damage), a file at neither path is `gone` (a later pass or the server renamed it),
|
||||
and a file sitting at its `dst` with no row naming it is simply something the
|
||||
server does not index. A row that still names `src` is none of these: it is the
|
||||
server's own stale pointer — a phantom — and it costs a re-scan, never a restore.
|
||||
therefore breaks the missing rows down by extension, and `--ignore-ext` names
|
||||
the extensions this server does not index, which are then counted separately and
|
||||
kept out of the verdict.
|
||||
|
||||
Exit codes: 0 every move landed; 1 something is stale or missing (the details are
|
||||
printed); 2 the journal or the database is not usable at all.
|
||||
@@ -102,18 +92,10 @@ def read_moves(path):
|
||||
def server_paths(db, kind):
|
||||
"""Every path the server holds, or a refusal that names what is missing."""
|
||||
spec = KINDS[kind]
|
||||
con = None
|
||||
err = None
|
||||
# Plain ro reads the WAL the server commits through; immutable is the last
|
||||
# resort, because it cannot see a row the server has not checkpointed.
|
||||
for opt in ("mode=ro", "mode=ro&immutable=1"):
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db}?{opt}", uri=True)
|
||||
break
|
||||
con = sqlite3.connect(f"file:{db}?mode=ro&immutable=1", uri=True)
|
||||
except sqlite3.Error as e:
|
||||
err = e
|
||||
if con is None:
|
||||
die(f"cannot open {db}: {err}")
|
||||
die(f"cannot open {db}: {e}")
|
||||
with con:
|
||||
have = {r[1] for r in con.execute(f"pragma table_info({spec['table']})")}
|
||||
if not spec["columns"] <= have:
|
||||
@@ -179,37 +161,20 @@ def main():
|
||||
if at_src:
|
||||
stale.append(src)
|
||||
if not at_dst:
|
||||
missing.append((src, dst))
|
||||
|
||||
# The database cannot tell a lost file from one it never indexes: both read
|
||||
# as "no row at the destination". Stat both paths, so the report separates
|
||||
# the classes the runbook names — `REAL` is the only one that is this pass's
|
||||
# own damage, and `gone` is a later pass or the server's own rename.
|
||||
on_disk, real, gone = [], [], []
|
||||
for src, dst in missing:
|
||||
if os.path.exists(dst):
|
||||
on_disk.append(dst)
|
||||
elif os.path.exists(src):
|
||||
real.append(src)
|
||||
else:
|
||||
gone.append(dst)
|
||||
missing.append(dst)
|
||||
|
||||
print(f"{args.db_kind}: {len(moves)} moves checked")
|
||||
print(f"landed: {landed}")
|
||||
print(f"still at src: {len(stale)}")
|
||||
print(f"missing from the DB: {len(missing)}")
|
||||
print(f" on disk at dst, no row names it: {len(on_disk)}")
|
||||
print(f" REAL — file still at src: {len(real)}")
|
||||
print(f" gone — at neither path: {len(gone)}")
|
||||
if ignored:
|
||||
print(f"not indexed (extension ignored): {ignored}")
|
||||
if missing:
|
||||
exts = Counter(os.path.splitext(d)[1].lower() or "(no extension)"
|
||||
for _, d in missing)
|
||||
for d in missing)
|
||||
print("missing by extension: "
|
||||
+ ", ".join(f"{e} {n}" for e, n in sorted(exts.items())))
|
||||
for label, items in (("real", real), ("gone", gone), ("missing", on_disk),
|
||||
("stale", stale)):
|
||||
for label, items in (("stale", stale), ("missing", missing)):
|
||||
for path in items[:args.show]:
|
||||
print(f" {label}: {path}")
|
||||
if len(items) > args.show:
|
||||
|
||||
@@ -48,6 +48,14 @@ Use the `Authorization: token` header for everything.
|
||||
- **Read an issue**: `curl -s -K "$cfg" $API/issues/<number>` (fields: `number`, `title`, `body`, `labels[].name`, `state`)
|
||||
- **List issues**: `curl -s -K "$cfg" "$API/issues?state=open&limit=50"` — add `&labels=<id>` to filter
|
||||
- **Comment**: `curl -s -K "$cfg" -X POST -d "$(jq -n --rawfile b "$BODY" '{body:$b}')" $API/issues/<number>/comments`
|
||||
- **Comment on a PR**: the conversation thread is `POST $API/issues/<PR-number>/comments`
|
||||
(201, and the answer's `html_url` points at `pulls/<n>`; Gitea's issue index is
|
||||
unified, so the PR's number is the same number). `$API/pulls/<n>/comments` is the
|
||||
**review**-comment store and answers `405` to a plain body.
|
||||
- **A body built by `jq --rawfile b /dev/stdin` inside `$(…)` is empty**: the command
|
||||
substitution does not inherit the heredoc, so Gitea receives `{}` and answers 422 —
|
||||
and `jq -r '.id'` on the error prints nothing, which reads as success. Write the
|
||||
body to a file and pass `--rawfile` that file.
|
||||
- **Apply labels**: `POST $API/issues/<number>/labels` with `{"labels":[<label-ID numbers>]}` — Gitea wants numeric label **IDs**, not names; resolve them from `GET $API/labels` (see [[Triage Labels]] for which labels exist)
|
||||
- **Remove a label**: `DELETE $API/issues/<number>/labels/<label-id>`
|
||||
- **Close**: `PATCH $API/issues/<number>` with `{"state":"closed"}`
|
||||
|
||||
@@ -182,6 +182,48 @@ scripts/jellyfin-providers.sh verify # plugin + key LENGTHS, never key bytes
|
||||
2. Set metadata/language preferences (`LANG=en_US.UTF-8` pinned in the unit).
|
||||
3. Configure clients to use `http://192.168.0.235:8096` or the guarded vhost.
|
||||
|
||||
## Triggering a scan
|
||||
|
||||
The scan key lives in `~/.config/secrets/jellyfin` (mode 600; the same value is
|
||||
`JELLYFIN_API_KEY` in the ignored `dankserver/.env`). It rides a mode-600 curl
|
||||
config, never a header argument, in the `MediaBrowser` form: 12.4 authenticates
|
||||
only `Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token`
|
||||
for every key, valid or not. Before believing a 401, prove the key against the
|
||||
server's own table — `select count(*) from ApiKeys where AccessToken = ?` on
|
||||
`jellyfin.db` opened `mode=ro&immutable=1`. The backslashes below matter:
|
||||
|
||||
```bash
|
||||
umask 077
|
||||
{ printf 'header = "Authorization: MediaBrowser Token=\\"%s\\""\n' \
|
||||
"$(cat ~/.config/secrets/jellyfin)"
|
||||
echo 'url = "http://127.0.0.1:8096/Library/Refresh"'
|
||||
echo 'request = "POST"'; } >/tmp/jf.hdr
|
||||
curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n"
|
||||
shred -u /tmp/jf.hdr
|
||||
```
|
||||
|
||||
`204` is acceptance; anything else queued no scan, so re-issue the identical
|
||||
request instead of diagnosing. A `401` from a key that answers `200` on a
|
||||
hand-typed request is this block's quoting, not the server: bash `printf`
|
||||
collapses `\"` to `"`, curl ends the quoted value at that quote, and Jellyfin
|
||||
receives `Authorization: MediaBrowser Token=` with nothing after it (`\\"` answers
|
||||
204, `\"` answers 401 — both run on this host).
|
||||
|
||||
One accepted refresh is the whole procedure. A second one while a scan runs moves
|
||||
`Scan Media Library` to `Cancelling`, and once the runner holds `Cancelling` every
|
||||
later `204` is accepted and then discarded — the status code stops meaning
|
||||
anything. The fix is `systemctl --user restart jellyfin.service` (never
|
||||
`deploy.sh`: that runs the never-tested metadata-view cutover), one POST after
|
||||
health answers, and then believe `ScheduledTasks` State at 130 % CPU, not the
|
||||
response code.
|
||||
|
||||
**A scan that must see a new folder has to be recursive.** `POST /Library/Refresh`
|
||||
is the incremental pass, and inotify does not cross a rootless podman bind, so
|
||||
folders created on the host stay invisible to it. The move is
|
||||
`POST /Items/{id}/Refresh?Recursive=true` (204) per library, the id from
|
||||
`GET /Library/VirtualFolders` — its field is `ItemId`, not `Id` — and that same
|
||||
record's `RefreshStatus` is the progress read.
|
||||
|
||||
## See Also
|
||||
|
||||
- [[Plex]] — co-resident media server on the same volume
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
type: reference
|
||||
title: Media Audiobook Relayout
|
||||
description: "Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals."
|
||||
tags: [dankserver, media-server, plex, media-relayout]
|
||||
timestamp: 2026-10-31T00:00:00Z
|
||||
---
|
||||
|
||||
# Media Audiobook Relayout
|
||||
|
||||
`/mnt/entropy/Audiobooks` is the one library here that [[Jellyfin]] does not read
|
||||
at all — its music scanner has no audiobook concept — while [[Plex]] indexes all
|
||||
of it as Artist → Album → Track: the album is the BOOK, that book's single artist
|
||||
is the NARRATOR, both matched out of the files' own tags. Today one book is split
|
||||
across `… [Part 1]`, `… [Part 2]` folders three or four deep inside a collection
|
||||
pack, so a scan serves three quarters of a novel; `scripts/media-audiobooks.py`
|
||||
re-files the tree so the property both servers' book scanners assume is true —
|
||||
|
||||
Audiobooks/<Narrator>/<Book> (<year>)/<the file keeps its own name>
|
||||
|
||||
The shared gates, the executor, the journal law and the rollback live in
|
||||
[[Media Relayout]]; this page is the audiobook arm only.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Plex's chapter rows for this root (the DB opens read-only).
|
||||
python3 scripts/media-census.py audiobooks --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-audiobooks.py --root /mnt/entropy/Audiobooks \
|
||||
--enum /tmp/ab_enum.tsv --rows /tmp/ab_rows.tsv --out /tmp/ab_plan.tsv \
|
||||
--exceptions /tmp/ab_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST. Then the same executor, journal and rollback
|
||||
# as [[Media Relayout]] shows, on its own journal, and refresh the Plex section
|
||||
# as in step 5 of the movie Procedure — from inside the container, never over
|
||||
# the host port.
|
||||
scripts/media-apply.py apply --plan /tmp/ab_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-audiobooks.jsonl --apply
|
||||
```
|
||||
|
||||
The group is the identity Plex asserts in its own database — **(narrator, book,
|
||||
year)** — so one book arriving in four part folders becomes one folder, and a
|
||||
1965 and a 2020 recording that share a title stay two books.
|
||||
|
||||
Two laws are forced by what the rows actually hold:
|
||||
|
||||
+- **A chapter is never renumbered.** Most books contain chapters whose index Plex
|
||||
does not know, so a number written into a name would be invented — and it would
|
||||
buy nothing, because both servers order playback by the embedded track tag, not
|
||||
by the file name. A wrong number in a name is the number a human later trusts.
|
||||
Fix order in the tags: a Plex scan re-reads tags and never renames a file.
|
||||
+- **The shelf is the narrator, not the author.** Every book has exactly one artist
|
||||
row and it is the narrator; the author survives only inside some album titles
|
||||
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving it would
|
||||
mean parsing titles where a ripper wrote one and inventing it everywhere else.
|
||||
The author stays visible in the folder name Plex itself shows.
|
||||
|
||||
What it refuses, the music arm's list with one addition: a video container, audio
|
||||
no chapter row names, and a loose file in a folder holding two books all park under
|
||||
`_unsorted` with the path they have today — and so does a chapter whose row names
|
||||
**no book or no narrator**: a placeholder (`[Unknown Album]`, `[Unknown Artist]`)
|
||||
that unrelated books share, or a chapter number mistagged as the artist (`0.4`).
|
||||
`Various Artists` is not a placeholder — that is a multi-cast read.
|
||||
|
||||
The reviewed plan lives at `/mnt/entropy/media-relayout-audiobooks-plan.tsv`; it
|
||||
applies on its own journal, behind the music writer, and then refreshes the Plex
|
||||
section. Verify it with `--db-kind plex` — Jellyfin holds no rows here, so a
|
||||
Jellyfin pass proves nothing either way.
|
||||
|
||||
## See Also
|
||||
|
||||
[[Media Relayout]], [[Media Music Relayout]], [[Plex]]
|
||||
@@ -0,0 +1,68 @@
|
||||
---
|
||||
type: reference
|
||||
title: Media Music Relayout
|
||||
description: "Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals."
|
||||
tags: [dankserver, media-server, jellyfin, media-relayout]
|
||||
timestamp: 2026-10-31T00:00:00Z
|
||||
---
|
||||
|
||||
# Media Music Relayout
|
||||
|
||||
`/mnt/entropy/Music` is rips and discography packs, and Jellyfin has matched and
|
||||
tagged almost all of it. `scripts/media-music.py` re-files it into the shape
|
||||
Jellyfin's music scanner reads —
|
||||
|
||||
Music/<album artist>/<Album> (<year>)/NN - <Title>.ext
|
||||
|
||||
— and the authority is the tag row of the track, never the name the ripper gave
|
||||
the file. The shared gates, the executor, the journal law and the rollback live in
|
||||
[[Media Relayout]]; this page is the music arm only.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Jellyfin's audio rows, one per track, with the album and
|
||||
# artist ids its own database carries (the DB opens read-only).
|
||||
python3 scripts/media-census.py music --root /mnt/entropy/Music \
|
||||
--out /tmp/music_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Music \
|
||||
--out /tmp/music_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-music.py --root /mnt/entropy/Music --enum /tmp/music_enum.tsv \
|
||||
--rows /tmp/music_rows.tsv --out /tmp/music_plan.tsv \
|
||||
--exceptions /tmp/music_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Then the same
|
||||
# executor, journal and rollback as [[Media Relayout]] shows, on its own journal.
|
||||
scripts/media-apply.py apply --plan /tmp/music_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-music.jsonl --apply
|
||||
```
|
||||
|
||||
The group is what the tags assert, not where the files sit: **(album-artist tag,
|
||||
album, year)**. One album arrives as three disc folders and as two copies of one
|
||||
rip, and one MusicBrainz release group is one album. A compilation is one album
|
||||
whose tracks name many performers, so it files once under `Various Artists`:
|
||||
keying on the performer would file one album once per artist. Two different
|
||||
album-artist tags on one album name are two albums, which is what Jellyfin's two
|
||||
album items say.
|
||||
|
||||
What it refuses:
|
||||
|
||||
+- **A sidecar is placed by the audio around it, never by its own name.** A
|
||||
`.lrc`, `.txt` or picture that shares a track's stem follows that track and
|
||||
takes the name the track now has, or Jellyfin cannot find it; anything else
|
||||
goes to the album its own folder was placed in, or to the folder whose discs
|
||||
sit one level below it. A folder whose audio belongs to two albums parks its
|
||||
sidecar rather than guessing.
|
||||
+- **An image album keeps its name.** A folder holding one audio file and a
|
||||
`.cue` is a whole album in one file; numbering it `01 - …` tells Jellyfin to
|
||||
index it as one track.
|
||||
+- **A video container parks**, because Jellyfin will not index it as music and a
|
||||
rename to `.m4a` would claim it is; **audio no provider row names parks** too,
|
||||
because unmatched audio is a tagging job, not a renaming job.
|
||||
|
||||
ONE writer at a time on this array: two rename passes race, and neither journal
|
||||
tells. Verify the run against Jellyfin's own tables — see *Proving the move
|
||||
landed* in [[Media Relayout]].
|
||||
|
||||
## See Also
|
||||
|
||||
[[Media Relayout]], [[Jellyfin]], [[Media Audiobook Relayout]]
|
||||
@@ -47,8 +47,10 @@ $C plex --root /mnt/entropy/Movies --out /tmp/plex-movies.tsv
|
||||
```
|
||||
|
||||
`plex` reads the Plex metadata cache (`--db` defaults to the path the
|
||||
[[Plex]] container bind-mounts at `/var/lib/plexmediaserver`) with
|
||||
`mode=ro&immutable=1`: a live server is never locked or written.
|
||||
[[Plex]] container bind-mounts at `/var/lib/plexmediaserver`) with plain
|
||||
`mode=ro`: a live server is never locked or written, and plain ro is also what
|
||||
makes a scan's uncheckpointed rows visible. The same rule as
|
||||
`scripts/media-verify.py` — see *Proving the move landed*.
|
||||
|
||||
Then plan — this reads only the three TSVs, so it costs seconds and can run
|
||||
anywhere, including a laptop:
|
||||
@@ -79,41 +81,12 @@ An apply also leaves the release folders it emptied standing where they were —
|
||||
hundreds of them at the library root — which is what `sweep` removes, dry-run
|
||||
first, from the same journal.
|
||||
|
||||
After it lands, both servers need a scan or the change is invisible.
|
||||
|
||||
```bash
|
||||
# Run this from the host shell through `podman exec`, never from a neighbour
|
||||
# container: Plex's HTTPD closes without a single byte on any request that
|
||||
# arrives from another container's netns (10.89.x.x reads as "empty reply", not
|
||||
# as a 401). The token rides stdin, never argv, where any process on the host
|
||||
# could read it out of /proc/<pid>/cmdline.
|
||||
umask 077
|
||||
podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \
|
||||
'/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok
|
||||
podman exec -i plex sh -c 'read -r T
|
||||
for k in $(curl -s -H "X-Plex-Token: $T" -H "Accept: application/json" \
|
||||
http://127.0.0.1:32400/library/sections | grep -o "\"key\":\"[0-9]*\"" | cut -d\" -f4); do
|
||||
curl -s -o /dev/null -w "plex section $k: %{http_code}\n" -X POST \
|
||||
-H "X-Plex-Token: $T" "http://127.0.0.1:32400/library/sections/$k/refresh"
|
||||
done' </tmp/px.tok
|
||||
```
|
||||
|
||||
[[Jellyfin]]'s scan key lives in `~/.config/secrets/jellyfin` (mode 600; the
|
||||
same value is `JELLYFIN_API_KEY` in the ignored `dankserver/.env`). It rides a
|
||||
mode-600 curl config, never a header argument, in the `MediaBrowser` form —
|
||||
Jellyfin 12.4 refuses a bare `X-Emby-Token`, and the backslashes below matter:
|
||||
|
||||
```bash
|
||||
umask 077
|
||||
{ printf 'header = "Authorization: MediaBrowser Token=\\"%s\\""\n' \
|
||||
"$(cat ~/.config/secrets/jellyfin)"
|
||||
echo 'url = "http://127.0.0.1:8096/Library/Refresh"'
|
||||
echo 'request = "POST"'; } >/tmp/jf.hdr
|
||||
curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n"
|
||||
shred -u /tmp/jf.hdr
|
||||
```
|
||||
|
||||
`204` is acceptance; anything else queued no scan. A `401` from a key that answers `200` on a hand-typed request is this block's quoting, not the server — bash `printf` collapses `\"` to `"`, curl ends the quoted value at that quote, and Jellyfin receives `Authorization: MediaBrowser Token=` with nothing after it (`\\"` answers 204, `\"` answers 401, both run on this host). One accepted refresh is the whole procedure: a second one while a scan runs moves `Scan Media Library` to `Cancelling`, and once the runner holds `Cancelling` every later `204` is accepted and then dropped — the fix is `systemctl --user restart jellyfin.service` (never `deploy.sh`: that runs the metadata-view cutover), one POST after health answers, and then believe `ScheduledTasks` State `Running`, not the status code.
|
||||
After it lands, both servers need a scan or the change is invisible. [[Plex]] and
|
||||
[[Jellyfin]] own the exact requests; two laws hold at this step: `204` is
|
||||
acceptance and anything else queued nothing — re-issue the identical request
|
||||
rather than diagnosing — and ONE accepted refresh is the whole procedure, because
|
||||
a second while a scan runs moves `Scan Media Library` to `Cancelling` and every
|
||||
later `204` is accepted and then discarded.
|
||||
|
||||
### Proving the move landed
|
||||
|
||||
@@ -123,14 +96,30 @@ replayed against both servers' own tables: every `move` row has a row at its
|
||||
replay: it opens each database read-only (there is no `sqlite3` binary on these
|
||||
hosts or inside either container) and exits non-zero while a row is stale.
|
||||
|
||||
Both libraries live in WAL mode, so the tool opens them plain `mode=ro`:
|
||||
`immutable=1` promises sqlite the file never changes, which skips the
|
||||
write-ahead log — a scanner's own uncheckpointed rows then read as missing, and
|
||||
a read beside a running scan can fail as "database disk image is malformed".
|
||||
`immutable` stays only the fallback for a database whose `-shm` cannot be read.
|
||||
|
||||
```bash
|
||||
scripts/media-verify.py --journal /mnt/entropy/media-relayout-music.jsonl \
|
||||
--db-kind jellyfin --ignore-ext jpg,jpeg,png,log,cue,toc,nfo,txt,m3u,accurip
|
||||
```
|
||||
|
||||
Ignore only what a server cannot index. A missing `.flac` or `.mkv` is the
|
||||
fault; a row still naming its `src` means the scan has not followed the rename
|
||||
yet, so re-run the check after the scan, never before.
|
||||
"No row at the destination" is not one verdict but four. **never indexed**: the
|
||||
server indexes no file of that extension (its own extension census says so) — a
|
||||
sidecar, not a lost film. **gone**: the file sits at neither src nor dst, because
|
||||
a later pass or the server's own extension sweep renamed it. **phantom**: a row
|
||||
still naming src while the file has left src. **REAL**: no row at dst *and* the
|
||||
file still at src — the only class that is this relayout's own damage. Phantoms
|
||||
are normal for both servers (Plex carries thousands that predate any relayout);
|
||||
re-scan to clear them and never read them as loss.
|
||||
|
||||
The owning server decides whether a pass can convict anything: Plex's roots are
|
||||
its `sections` rows joined to their location rows (`Movies`, `TVShows`, `Music`,
|
||||
`Audiobooks`, `LiveRecording`) and Jellyfin holds no audiobook library at all, so
|
||||
the audiobook journal verifies with `--db-kind plex` or it proves nothing.
|
||||
|
||||
Three doors that look right and are not: `GET /search?query=` matches metadata
|
||||
titles, never file paths; `?path=` on `/library/sections/N/all` makes Plex's
|
||||
@@ -167,16 +156,15 @@ from the box-set folder. What it refuses it prints and leaves alone:
|
||||
- a destination two files both claim: the executor refuses the whole plan
|
||||
before its first rename, and only Frank decides which copy is the keeper.
|
||||
- an episode the destination library already holds at the same byte count. The
|
||||
movie share carried a second copy of the whole Monk box set — 124 episodes
|
||||
both sides, the same episode list, every byte count identical, 31.5 GiB each
|
||||
way, and head+tail blake2b identical on a 6-episode sample. The mover leaves
|
||||
such a series in place and says so; a duplicate is deleted or kept by Frank,
|
||||
never copied into one library by a tool.
|
||||
movie share once carried a second copy of a whole box set — same episode list,
|
||||
every byte count identical. The mover leaves such a series in place and says
|
||||
so; a duplicate is deleted or kept by Frank, never copied into one library by
|
||||
a tool.
|
||||
|
||||
A sidecar that shares an episode's own name (`.srt`, `.nfo`) moves with it.
|
||||
|
||||
Below the threshold the mover prints and leaves, so a lone downloaded episode of
|
||||
a real show stays in the movie library on purpose — six did on 2026-10-30.
|
||||
a real show stays in the movie library on purpose.
|
||||
Filing one by hand is the same executor on a hand-written plan and the same
|
||||
journal; the trap is the series year. Read it from the file's own name or the
|
||||
folder it owns, never from the title whose bonus folder it happens to be parked
|
||||
@@ -187,12 +175,16 @@ show. A name whose marker is `Part 2` is a film and stays put.
|
||||
|
||||
`/mnt/entropy/TVShows` is scene packs rather than a library, and Jellyfin has
|
||||
already matched and numbered most of it. `scripts/media-shows.py` re-files that
|
||||
library into the same shape, and it never guesses: every series name and every
|
||||
episode number it writes traces to an authority, and what has none is parked for
|
||||
a human to read.
|
||||
library into the shape Jellyfin reads, with **tvdb as the name authority**:
|
||||
|
||||
<Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.ext
|
||||
|
||||
It never guesses: every series name, episode number and title it writes traces to
|
||||
an authority, and what has none is parked for a human to read.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Jellyfin's own matched rows (the DB opens read-only).
|
||||
# 1. The authority: Jellyfin's own matched rows — each carries the series' TMDb
|
||||
# and Tvdb ids and the title the row holds (the DB opens read-only).
|
||||
python3 scripts/media-census.py jellyfin --root /mnt/entropy/TVShows \
|
||||
--out /tmp/tv_titles.tsv
|
||||
# 2. What is on disk (stat only), and the runtimes the film arm needs.
|
||||
@@ -200,16 +192,33 @@ python3 scripts/media-census.py enumerate --root /mnt/entropy/TVShows \
|
||||
--out /tmp/tv_enum.tsv
|
||||
python3 scripts/media-census.py probe --root /mnt/entropy/TVShows \
|
||||
--enum /tmp/tv_enum.tsv --jobs 6 --out /tmp/tv_probed.tsv
|
||||
# The probe is the slow step: ffprobe reads both ends of every file.
|
||||
# 3. The plan. --tmdb needs TMDB_API_KEY; without it a series no provider dated
|
||||
# is simply named without a year, which Jellyfin accepts.
|
||||
# The probe is the slow step: ffprobe reads both ends of every file. So re-use
|
||||
# the last one — a duration belongs to the file, not to its name: re-key the
|
||||
# previous census through that pass's own journal (`src` -> `dst`) instead of
|
||||
# seeking the array again. It is also the only column the film arm reads, and
|
||||
# only for a file with no marker and no provider row (4,794 of 58,551 today,
|
||||
# all of them covered by a re-key that costs seconds, not an hour).
|
||||
# 3. The plan. --tmdb needs TMDB_API_KEY — the v3 key in ~/.config/secrets/tmdb,
|
||||
# which travels as the `api_key` query parameter. Only a v4 request token is
|
||||
# accepted as a Bearer header, and the wrong shape answers 401 on EVERY call,
|
||||
# which the summary line flags as NOT provider-complete. --tvdb reads its
|
||||
# `api_key=` line from ~/.config/secrets/tvdb, never argv. Both write the SAME
|
||||
# cache file, so the plan is reproducible offline afterwards: run it once to
|
||||
# fill the cache, and quote the second run.
|
||||
# Paths are only strings to a dry run, so this step can run on the workstation
|
||||
# against scp'd TSVs — which is where the two provider secrets live.
|
||||
python3 scripts/media-shows.py --root /mnt/entropy/TVShows --enum /tmp/tv_enum.tsv \
|
||||
--titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv --out /mnt/entropy/media-relayout-shows-plan.tsv \
|
||||
--exceptions /mnt/entropy/media-relayout-shows-exceptions.txt
|
||||
--titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv \
|
||||
--tmdb-cache /mnt/entropy/media-tv-names.json --tmdb --tvdb \
|
||||
--park /mnt/entropy/media-park/TVShows \
|
||||
--out /mnt/entropy/media-relayout-shows2-plan.tsv \
|
||||
--exceptions /mnt/entropy/media-relayout-shows2-exceptions.txt
|
||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Same executor
|
||||
# as above, but on this arm's OWN journal: a shared journal cannot be unwound.
|
||||
scripts/media-apply.py apply --plan /mnt/entropy/media-relayout-shows-plan.tsv \
|
||||
--root /mnt/entropy --journal /mnt/entropy/media-relayout-shows.jsonl --apply
|
||||
# as above, but on this arm's OWN journal, NAMED PER PASS: the journal of an
|
||||
# applied pass is its unwind record, so a second pass that reuses the name
|
||||
# destroys the only map back.
|
||||
scripts/media-apply.py apply --plan /mnt/entropy/media-relayout-shows2-plan.tsv \
|
||||
--root /mnt/entropy --journal /mnt/entropy/media-relayout-shows2.jsonl --apply
|
||||
```
|
||||
|
||||
Evidence, strongest first — the order the planner believes things:
|
||||
@@ -217,12 +226,19 @@ Evidence, strongest first — the order the planner believes things:
|
||||
- the **episode marker the file's own name states** (`S01E02-E03`, `1x04`);
|
||||
- the **`Season NN` folder it sits in and the number its name leads with inside
|
||||
it**: `Season 19/01 Charles Barkley, Nirvana.avi` is S19E01;
|
||||
- the **provider row**: identity (the series name and its year), and the
|
||||
numbering of a file whose own name says nothing;
|
||||
- the **provider row**: which episode the file IS, and the numbering of a file
|
||||
whose own name says nothing;
|
||||
- the **row's tvdb id, else its TMDb id carried through TMDb's external ids, else
|
||||
an exact tvdb name search**: that id names the SERIES FOLDER and supplies the
|
||||
episode title, because a name the server read off a folder is not a show name;
|
||||
- a **bonus folder** anywhere above the file, or Jellyfin's `ExtraType` → the
|
||||
series' canonical extra folder;
|
||||
- a runtime over `SHORT_MAX` makes it film-shaped, and it leaves for `--movies`
|
||||
only when a movie provider names it — otherwise it parks.
|
||||
- a runtime over `SHORT_MAX` makes it film-shaped, and it leaves for `--movies` only
|
||||
when the movie provider's title is a run of words the FILE itself states and its
|
||||
year agrees with the file's: the query is only what `title_of` salvaged from a
|
||||
release name, and the first hit for a short one is a near miss — a 1963 Bond film
|
||||
for the word "russia", *Special Edition I* for a file saying *Edition III*. Else it
|
||||
parks.
|
||||
|
||||
Two refusals carry the design, and a dry run on the real library found both:
|
||||
|
||||
@@ -237,125 +253,44 @@ Two refusals carry the design, and a dry run on the real library found both:
|
||||
the file's own leading number, win; a row that still insists on season 1989
|
||||
parks its file instead of creating `Season 1989`.
|
||||
|
||||
The provider's name is copied verbatim, warts included: Jellyfin carries a Series
|
||||
named `Black Clover 001` (92 episodes, no year) because it matched a pack off a
|
||||
filename. Renaming a show is Jellyfin's work — a folder rename never touches its
|
||||
provider ids — so the mover keeps the name it is handed and strips only the pack
|
||||
index a release leads with (`4a. Seed DESTINY` → `Seed DESTINY`).
|
||||
**A provider name is evidence only when an id backs it.** Jellyfin names a Series
|
||||
after the folder it scanned, so a library filed by release folders ends up with
|
||||
one Series item per box set — `24 (2001) Season 1`, `Modern Family S05` — and
|
||||
copying those names verbatim launders folder junk into the contract. The planner
|
||||
resolves each series to its tvdb id and writes tvdb's name and year; the merged
|
||||
identity is also what turns a box set split across five release folders into ONE
|
||||
folder, and what makes two files claiming one episode MEET in the duplicate rule.
|
||||
A series no id resolves keeps the provider's words and is listed under *no tvdb
|
||||
identity* in the exceptions — that list, not a rename, is the review.
|
||||
|
||||
## The music library's own relayout
|
||||
**The review pile lives outside the library.** Files parked inside
|
||||
`/mnt/entropy/TVShows` were indexed as one show named `_unsorted`, and that
|
||||
show's rows then looked like evidence. So a row whose series is the hidden
|
||||
top-level folder the file itself sits in is dropped — and the folder is matched
|
||||
with its leading underscore stripped, because the server *displays* `_unsorted` as
|
||||
`unsorted`: compare the spelled names and the pile survives the relayout as a
|
||||
visible series folder that real files get renamed into. `--park` outside the root
|
||||
makes the next plan re-file the pile instead of skipping it. Inside a hidden
|
||||
top-level folder a file's own words may MATCH a registered show, never invent one.
|
||||
|
||||
`/mnt/entropy/Music` is rips and discography packs, and Jellyfin has matched and
|
||||
tagged almost all of it. `scripts/media-music.py` re-files it into the shape
|
||||
Jellyfin's music scanner reads —
|
||||
Release-group folder names are the one thing no provider can read, so the
|
||||
pending TVShows naming plan files 179 top-level directories (9,931 files) under
|
||||
the junk name they arrived with. A TVDB id map can name them, under the ladder's
|
||||
own acceptance rule: a hit counts only when TVDB's spelling equals the cleaned
|
||||
query, or differs from it by punctuation alone — the same letters are not a
|
||||
guess. Everything else is a choice for Frank, and the map is never applied
|
||||
unreviewed: `/mnt/entropy/media-relayout-shows-tvdb-proposal.tsv` carries one
|
||||
row per folder with `verdict` = `EXACT` / `PUNCT` / `NEAR` / `NONE` and a `note`
|
||||
naming the two traps — a folder marked `(US)` that matches the UK original, and
|
||||
several folders mapping to one id, which is a MERGE whose duplicate episodes the
|
||||
executor will refuse until a keeper is chosen.
|
||||
|
||||
Music/<album artist>/<Album> (<year>)/NN - <Title>.ext
|
||||
## The music and audiobook arms
|
||||
|
||||
— and the authority is the tag row of the track, never the name the ripper gave
|
||||
the file.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Jellyfin's audio rows, one per track, with the album and
|
||||
# artist ids its own database carries (the DB opens read-only).
|
||||
python3 scripts/media-census.py music --root /mnt/entropy/Music \
|
||||
--out /tmp/music_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Music \
|
||||
--out /tmp/music_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-music.py --root /mnt/entropy/Music --enum /tmp/music_enum.tsv \
|
||||
--rows /tmp/music_rows.tsv --out /tmp/music_plan.tsv \
|
||||
--exceptions /tmp/music_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Then the same
|
||||
# executor, journal and rollback as above, on its own journal.
|
||||
scripts/media-apply.py apply --plan /tmp/music_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-music.jsonl --apply
|
||||
```
|
||||
|
||||
The group is what the tags assert, not where the files sit: **(album-artist tag,
|
||||
album, year)**. One album arrives as three disc folders and as two copies of one
|
||||
rip, and one MusicBrainz release group is one album. A compilation is one album
|
||||
whose tracks name many performers, so it files once under `Various Artists`:
|
||||
keying on the performer would file one album once per artist. Two different
|
||||
album-artist tags on one album name are two albums, which is what Jellyfin's two
|
||||
album items say.
|
||||
|
||||
What it refuses:
|
||||
|
||||
+- **A sidecar is placed by the audio around it, never by its own name.** A
|
||||
`.lrc`, `.txt` or picture that shares a track's stem follows that track and
|
||||
takes the name the track now has, or Jellyfin cannot find it; anything else
|
||||
goes to the album its own folder was placed in, or to the folder whose discs
|
||||
sit one level below it. A folder whose audio belongs to two albums parks its
|
||||
sidecar rather than guessing.
|
||||
+- **An image album keeps its name.** A folder holding one audio file and a
|
||||
`.cue` is a whole album in one file; numbering it `01 - …` tells Jellyfin to
|
||||
index it as one track.
|
||||
+- **A video container parks**, because Jellyfin will not index it as music and a
|
||||
rename to `.m4a` would claim it is; **audio no provider row names parks** too,
|
||||
because unmatched audio is a tagging job, not a renaming job.
|
||||
|
||||
The plan applies on 2026-10-31 through `/mnt/entropy/media-relayout-music.jsonl`;
|
||||
its rollback is that journal run through `media-apply.py rollback --apply`. ONE
|
||||
writer at a time on this array: two rename passes race, and neither journal tells.
|
||||
|
||||
## The audiobook library's own relayout
|
||||
|
||||
`/mnt/entropy/Audiobooks` is the one library here that [[Jellyfin]] does not read
|
||||
at all — its music scanner has no audiobook concept — while [[Plex]] indexes all
|
||||
of it as Artist → Album → Track: the album is the BOOK, that book's single artist
|
||||
is the NARRATOR, both matched out of the files' own tags. Today one book is split
|
||||
across `… [Part 1]`, `… [Part 2]` folders three or four deep inside a collection
|
||||
pack, so a scan serves three quarters of a novel; `scripts/media-audiobooks.py`
|
||||
re-files the tree so the property both servers' book scanners assume is true —
|
||||
|
||||
Audiobooks/<Narrator>/<Book> (<year>)/<the file keeps its own name>
|
||||
|
||||
```bash
|
||||
# 1. The authority: Plex's chapter rows for this root (the DB opens read-only).
|
||||
python3 scripts/media-census.py audiobooks --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-audiobooks.py --root /mnt/entropy/Audiobooks \
|
||||
--enum /tmp/ab_enum.tsv --rows /tmp/ab_rows.tsv --out /tmp/ab_plan.tsv \
|
||||
--exceptions /tmp/ab_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST. Then the same executor, journal and rollback
|
||||
# as above, on its own journal, and refresh the Plex section as in step 5 of
|
||||
# the movie Procedure — from inside the container, never over the host port.
|
||||
scripts/media-apply.py apply --plan /tmp/ab_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-audiobooks.jsonl --apply
|
||||
```
|
||||
|
||||
The group is the identity Plex asserts in its own database — **(narrator, book,
|
||||
year)** — so one book arriving in four part folders becomes one folder, and a
|
||||
1965 and a 2020 recording that share a title stay two books.
|
||||
|
||||
Two laws are forced by what the rows actually hold:
|
||||
|
||||
+- **A chapter is never renumbered.** Most books contain chapters whose index Plex
|
||||
does not know, so a number written into a name would be invented — and it would
|
||||
buy nothing, because both servers order playback by the embedded track tag, not
|
||||
by the file name. A wrong number in a name is the number a human later trusts.
|
||||
Fix order in the tags: a Plex scan re-reads tags and never renames a file.
|
||||
+- **The shelf is the narrator, not the author.** Every book has exactly one artist
|
||||
row and it is the narrator; the author survives only inside some album titles
|
||||
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving it would
|
||||
mean parsing titles where a ripper wrote one and inventing it everywhere else.
|
||||
The author stays visible in the folder name Plex itself shows.
|
||||
|
||||
What it refuses, the music arm's list with one addition: a video container, audio
|
||||
no chapter row names, and a loose file in a folder holding two books all park under
|
||||
`_unsorted` with the path they have today — and so does a chapter whose row names
|
||||
**no book or no narrator**: a placeholder (`[Unknown Album]`, `[Unknown Artist]`)
|
||||
that unrelated books share, or a chapter number mistagged as the artist (`0.4`).
|
||||
`Various Artists` is not a placeholder — that is a multi-cast read.
|
||||
|
||||
The plan stands reviewed at `/mnt/entropy/media-relayout-audiobooks-plan.tsv`
|
||||
(55,114 moves, 7,764 parked, executor preflight refused nothing); it applies on its
|
||||
own journal, behind the music writer, and then refreshes the Plex section.
|
||||
Each is a page of its own, because each reads a different authority and refuses
|
||||
for a different reason: [[Media Music Relayout]] files by Jellyfin's track tags,
|
||||
[[Media Audiobook Relayout]] by Plex's chapter rows. Both share this page's
|
||||
executor, the one-journal-per-arm law, the sweep and the rollback.
|
||||
|
||||
## Laws
|
||||
|
||||
@@ -396,19 +331,6 @@ own journal, behind the music writer, and then refreshes the Plex section.
|
||||
free space, and `rename(2)` refuses it anyway. Trash goes beside the item —
|
||||
`/mnt/entropy/.AGENTTRASH/<date>-<topic>/` — so it costs one inode change and
|
||||
rolls back exactly like the move that made it.
|
||||
- **A Jellyfin 401 is the header, not the key.** 12.4 authenticates only
|
||||
`Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token` for
|
||||
every key, valid or not. This page carried "no credential on this box can
|
||||
trigger a scan, the two Seerr keys answer 401" for weeks because of that
|
||||
header — the keys were innocent. Before believing a 401, prove the key against
|
||||
the server's own table: `select count(*) from ApiKeys where AccessToken = ?`
|
||||
on `jellyfin.db` opened `mode=ro&immutable=1`.
|
||||
- **A scan that must see a new folder has to be recursive.** `POST
|
||||
/Library/Refresh` is the incremental pass, and inotify does not cross a
|
||||
rootless podman bind, so folders created on the host stay invisible to it. The
|
||||
move is `POST /Items/{id}/Refresh?Recursive=true` (204) per library, the id
|
||||
from `GET /Library/VirtualFolders` — its field is `ItemId`, not `Id` — and
|
||||
that same record's `RefreshStatus` is the progress read.
|
||||
- **A file whose name carries no words is bonus material, not a folder.**
|
||||
`.Extra 1 ( 1H 27min multisub ).mkv` parses to `1`; a movie folder named `1`
|
||||
is not a movie. It becomes a review line telling a human to name the file.
|
||||
@@ -437,16 +359,15 @@ own journal, behind the music writer, and then refreshes the Plex section.
|
||||
recreates the folder, so the sweep is undone by the same command as the moves.
|
||||
Proof carries downward: inside a folder the journal proves emptied, every
|
||||
empty directory goes with it. A release's `Screens/` or `Sample/` is never a
|
||||
move source itself, so 146 of them held their emptied parents on disk after
|
||||
the first sweep.
|
||||
move source itself, so it can hold an emptied parent until the next sweep.
|
||||
A failed `rmdir` is not a removal: only a folder that is really gone leaves
|
||||
scope, so the next sweep finishes what a permission problem stopped.
|
||||
- **An episode-shaped name marks a SERIES, not a film — and the series is the
|
||||
name BEFORE the marker.** `Monk - S05E03 - Mr. Monk and the Big Game` is series
|
||||
`Monk`. Keyed on the whole title instead, every episode became a group of one
|
||||
and never reached the five-episode size that calls something a series, so a
|
||||
box set was filed as 124 movies in 124 folders. The threshold still earns its
|
||||
keep: a lone film named `Part 2` stays a film. Episodes of a series are parked
|
||||
`Monk`. Keyed on the whole title instead, every episode became a group of one,
|
||||
never reaching the five-episode size that calls something a series — a whole
|
||||
box set filed as one folder per episode. The threshold still earns its keep:
|
||||
a lone film named `Part 2` stays a film. Episodes of a series are parked
|
||||
in `_TV needs a TV library - <series>` for a human to move to a TV source —
|
||||
the movie tool never files TV.
|
||||
- **One journal is one undo.** The TV move runs on its own journal, and with
|
||||
@@ -489,6 +410,14 @@ any folder a sweep removed, restores every source path, removes only the folders
|
||||
that run created *and that are now empty*, and refuses everything if any moved
|
||||
file cannot be accounted for. A rolled-back journal refuses to roll back twice.
|
||||
|
||||
**Roll a chain back newest journal first, or the tool refuses.** Each pass
|
||||
re-files what an earlier one moved, so a journal is a clean inverse only until
|
||||
something downstream touches its files; `ls -l /mnt/entropy/media-relayout*.jsonl`
|
||||
is the order. Refusing is the tool working: on 2026-10-31 the movie journal could
|
||||
not restore files the later TV passes had re-filed, and the stray journal could
|
||||
not restore files Jellyfin's extension sweep had renamed. Roll back what still
|
||||
inverts and leave the rest as the current state — never force it.
|
||||
|
||||
## See Also
|
||||
|
||||
[[Jellyfin]], [[Plex]], [[Media Metadata View]], [[Entropy Volume]], [[Testing Policy]], [[Storage Policy]]
|
||||
|
||||
@@ -87,3 +87,45 @@ root runs as the `dank` user, and the deployer adopts the data root with
|
||||
while the HTTP layer finishes loading; the first boot of a years-old
|
||||
library took ~10 minutes (2026-09 cutover). The deployer gate now allows
|
||||
600 s before failing; later boots are much faster (warm page cache).
|
||||
|
||||
## Triggering a scan
|
||||
|
||||
A rename on disk is invisible until the section is rescanned, and a moved file is
|
||||
a `Couldn't find the file to stream` 404 until that scan passes it. Run this from
|
||||
the host shell through `podman exec`, never from a neighbour container: PMS's
|
||||
HTTPD closes without a single byte on any request that arrives from another
|
||||
container's netns (10.89.x.x reads as "empty reply", not as a 401). The token
|
||||
rides stdin, never argv, where any process on the host could read it out of
|
||||
`/proc/<pid>/cmdline`; it lives in `Preferences.xml` inside the RW data bind.
|
||||
|
||||
```bash
|
||||
umask 077
|
||||
podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \
|
||||
'/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok
|
||||
cat >/tmp/pxscan.sh <<'EOS'
|
||||
read -r T
|
||||
cfg=$(mktemp)
|
||||
mk() { { printf 'header = "X-Plex-Token: %s"\nheader = "Accept: application/json"\n' "$T"
|
||||
printf 'url = "%s"\n' "$1"; } > "$cfg"; }
|
||||
mk "http://127.0.0.1:32400/library/sections"
|
||||
for k in $(curl -s -K "$cfg" | grep -o '"key":"[0-9]*"' | cut -d\" -f4); do
|
||||
mk "http://127.0.0.1:32400/library/sections/$k/refresh"
|
||||
printf 'request = "POST"\n' >> "$cfg"
|
||||
echo "plex section $k: $(curl -s -o /dev/null -w '%{http_code}' -K "$cfg")"
|
||||
done
|
||||
shred -u "$cfg"
|
||||
EOS
|
||||
podman cp /tmp/pxscan.sh plex:/tmp/pxscan.sh
|
||||
podman exec -i plex sh /tmp/pxscan.sh </tmp/px.tok
|
||||
shred -u /tmp/px.tok
|
||||
```
|
||||
|
||||
Every section answers `204` — or `200` with a Directory body when
|
||||
`Accept: application/json` is sent. Inside the quoted `printf` format a
|
||||
backslash-newline is LITERAL, not a continuation, so `mk` writes two `printf`s:
|
||||
one glued line and one stray backslash is a config file curl rejects, and the
|
||||
failure prints as `option -K: is unknown`, never as a bad config line.
|
||||
|
||||
Proof that a rename landed is not a scan response: `scripts/media-verify.py`
|
||||
replays the run's journal against the database — the only read that
|
||||
distinguishes "not scanned yet" from "gone"; see [[Media Relayout]].
|
||||
@@ -51,7 +51,9 @@ is a locator: one row, one clause, no numbers.
|
||||
| [[Entropy Volume]] | The Dankserver entropy volume is a RAID-backed LUKS filesystem mounted at |
|
||||
| [[Jellyfin]] | Jellyfin on Dankserver: quadlet unit, overlay metadata view, provider keys, ports, access, transcode status. |
|
||||
| [[mariadb]] | The MariaDB backend for RomM: quadlet unit, network alias, env-file secret handling, and the backup path. |
|
||||
| [[Media Audiobook Relayout]] | Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals. |
|
||||
| [[Media Metadata View]] | The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback. |
|
||||
| [[Media Music Relayout]] | Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals. |
|
||||
| [[Media Relayout]] | Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback. |
|
||||
| [[Plex]] | Plex Media Server is a **containerized media server on Dankserver**: a podman |
|
||||
| [[redis]] | The Redis cache and task queue for RomM: quadlet unit, network alias topology, and the backup path. |
|
||||
|
||||
@@ -228,51 +228,6 @@ PY
|
||||
[ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 1 ] # header only
|
||||
}
|
||||
|
||||
# wal_fresh DB ROOT OUT — a row Plex matched during a scan and committed, but
|
||||
# has not checkpointed: the normal state of a library being walked. The holder
|
||||
# connection stops the WAL folding into the main file when the writer closes,
|
||||
# and the census runs as a real subprocess against those same files.
|
||||
wal_fresh() {
|
||||
local db="$1" root="$2" out="$3"
|
||||
python3 - "$db" "$root" "$out" "$CENSUS" <<'PY'
|
||||
import sqlite3, subprocess, sys
|
||||
db, root, out, census = sys.argv[1:5]
|
||||
w = sqlite3.connect(db)
|
||||
w.execute("pragma journal_mode=wal")
|
||||
w.execute("pragma wal_autocheckpoint=0")
|
||||
w.commit()
|
||||
w.execute("pragma wal_checkpoint(truncate)") # the rows so far are on disk
|
||||
holder = sqlite3.connect(db) # last close would checkpoint
|
||||
ar = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||
' year, "index", deleted_at) VALUES (NULL, 8, ?, NULL, NULL, NULL)',
|
||||
("Fresh Narrator",)).lastrowid
|
||||
al = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||
' year, "index", deleted_at) VALUES (?, 9, ?, ?, NULL, NULL)',
|
||||
(ar, "Fresh Book", 2026)).lastrowid
|
||||
mi = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||
' year, "index", deleted_at) VALUES (?, 10, ?, NULL, ?, NULL)',
|
||||
(al, "Fresh Chapter", 1)).lastrowid
|
||||
it = w.execute("insert into media_items (metadata_item_id, deleted_at)"
|
||||
" VALUES (?, NULL)", (mi,)).lastrowid
|
||||
w.execute("insert into media_parts (media_item_id, file, deleted_at)"
|
||||
" VALUES (?, ?, NULL)", (it, root + "/Fresh/01.mp3"))
|
||||
w.commit()
|
||||
sys.exit(subprocess.run([sys.executable, census, "audiobooks", "--root", root,
|
||||
"--db", db, "--out", out]).returncode)
|
||||
PY
|
||||
}
|
||||
|
||||
@test "the census exports a row the scanner committed but has not checkpointed" {
|
||||
DB="$BATS_TEST_TMPDIR/plex.db"
|
||||
db_with_part "$DB" "$BATS_TEST_TMPDIR/Audiobooks/Old/01.mp3" \
|
||||
"Old Narrator" "Old Book" 1990 1 "Old Chapter"
|
||||
run wal_fresh "$DB" "$BATS_TEST_TMPDIR/Audiobooks" "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 3 ] # header + both rows
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$BATS_TEST_TMPDIR/Audiobooks/Fresh/01\.mp3 Fresh Narrator Fresh Book 2026 1 Fresh Chapter$"
|
||||
}
|
||||
|
||||
# -------------------------------------------------------------------- the plan
|
||||
|
||||
@test "a matched chapter files under its narrator and its book, name untouched" {
|
||||
|
||||
+177
-25
@@ -8,7 +8,14 @@
|
||||
# file name. So every test here states which authority a file had — name only,
|
||||
# provider only, both, neither — and asserts where the PLAN files it, never how
|
||||
# the planner decided. A fixture Jellyfin database stands in for the live one and
|
||||
# the TMDb cache is an input file, so no test touches the network.
|
||||
# the name caches are input files, so no test touches the network.
|
||||
#
|
||||
# The target contract is tvdb-grade naming:
|
||||
# <Series> (<year>)/Season NN/<Series> (<year>) - SxxEyy - <Episode Title>.<ext>
|
||||
# tvdb is the IDENTITY authority: a provider name a server derived from a folder
|
||||
# is not a series name, so a polluted name is undone through the series' tvdb id.
|
||||
# The review pile parks OUTSIDE the library — a parked folder inside it indexes
|
||||
# as one junk show, and that show's rows are then mistaken for evidence.
|
||||
#
|
||||
# The two outcomes that cost a re-run of a 50k-file plan are a file filed as an
|
||||
# episode that is not one (it steals an episode number from its real owner) and a
|
||||
@@ -25,11 +32,11 @@ APPLY="$REPO_ROOT/scripts/media-apply.py"
|
||||
# at PATH SIZE — a file with deterministic content.
|
||||
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
|
||||
|
||||
# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] — one row of the authority
|
||||
# export scripts/media-census.py jellyfin writes.
|
||||
# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] [TMDB] [TITLE] [TVDB]
|
||||
# — one row of the authority export scripts/media-census.py jellyfin writes.
|
||||
title_row() {
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t\n' "$1" "$2" "$3" "${4:-}" "${5:-}" "${6:-}" \
|
||||
>> "$TITLES"
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "${4:-}" \
|
||||
"${5:-}" "${6:-}" "${7:-}" "${8:-}" "${9:-}" >> "$TITLES"
|
||||
}
|
||||
|
||||
# dst SRC — the destination the plan gives SRC, or nothing when it does not move.
|
||||
@@ -40,6 +47,7 @@ dst() {
|
||||
setup() {
|
||||
LIB="$BATS_TEST_TMPDIR/TVShows"
|
||||
PARK="$LIB/_unsorted"
|
||||
OUTSIDE="$BATS_TEST_TMPDIR/park"
|
||||
MOVIES="$BATS_TEST_TMPDIR/Movies"
|
||||
ENUM="$BATS_TEST_TMPDIR/enum.tsv"
|
||||
PROBED="$BATS_TEST_TMPDIR/probed.tsv"
|
||||
@@ -77,8 +85,15 @@ setup() {
|
||||
# A season folder whose rows are stale: the database numbers every file of this
|
||||
# pack season 16 episode 20, four seasons' worth of them.
|
||||
at "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" 6000
|
||||
# One episode-shaped file loose at the library root.
|
||||
at "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" 4000
|
||||
# A box-set folder whose Series item the server named AFTER THE FOLDER: the
|
||||
# row's own name is junk, and only a provider id can undo it.
|
||||
BOX="$LIB/Show Box (2016) Season 1 S01 (1080p WEB-DL)"
|
||||
at "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" 6000
|
||||
# The park inside the library: the server indexed the whole pile as ONE show
|
||||
# named after the folder, so its rows state a series that is not a show.
|
||||
at "$PARK/Park Show S01E01.mkv" 5000
|
||||
at "$PARK/Twisted.Metal.S01E01.1080p.WEB-DL.mkv" 5200
|
||||
|
||||
# The enum census lists every file; the planner reads that instead of walking
|
||||
# the tree, so the fixture writes the same three columns.
|
||||
@@ -92,17 +107,21 @@ with open(out, "w") as fh:
|
||||
fh.write(f"{os.stat(p).st_size}\t{os.path.splitext(f)[1]}\t{p}\n")
|
||||
PY
|
||||
|
||||
printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\n' > "$TITLES"
|
||||
printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\ttitle\ttvdb\n' > "$TITLES"
|
||||
printf '{}\n' > "$CACHE"
|
||||
title_row "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" \
|
||||
"The Examples" 2010 1 1
|
||||
"The Examples" 2010 1 1 "" "" "Pilot"
|
||||
title_row "$ANIME/[grp] Anime Pack - 05.mkv" "Anime Pack" "" 1 5
|
||||
title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1
|
||||
title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1
|
||||
title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2
|
||||
title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3
|
||||
title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots"
|
||||
title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots"
|
||||
title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2 "" "" "Double Trouble"
|
||||
title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3 "" "" "Loose Ends"
|
||||
title_row "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" \
|
||||
"SNL Pack" "" 16 20
|
||||
title_row "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" \
|
||||
"Show Box (2016) Season 1" 2016 1 1 "" "2334" "Pilot" "78850"
|
||||
# The row the server wrote for a parked file: the series is the PARK FOLDER.
|
||||
title_row "$PARK/Park Show S01E01.mkv" "_unsorted" "" 1 1
|
||||
}
|
||||
|
||||
plan() {
|
||||
@@ -111,6 +130,28 @@ plan() {
|
||||
--out "$OUT" --exceptions "$EXC"
|
||||
}
|
||||
|
||||
plan_park() { # the review pile outside the library
|
||||
run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --titles "$TITLES" \
|
||||
--probed "$PROBED" --tmdb-cache "$CACHE" --movies "$MOVIES" \
|
||||
--park "$OUTSIDE" --out "$OUT" --exceptions "$EXC"
|
||||
}
|
||||
|
||||
@test "--tmdb with no key is reported as an unfinished plan, not a finished one" {
|
||||
# A dead provider is invisible where it happens: every 401 is one line on
|
||||
# stderr, and a terminal that scrolled past them shows a plan that reads as
|
||||
# complete while every folder still carries the metadata provider's junk. The
|
||||
# plan is still written — the answers that DID arrive are worth their rename —
|
||||
# so the state has to ride in the summary line and in the review file.
|
||||
# env -u keeps this offline no matter what the developer's shell exports.
|
||||
run env -u TMDB_API_KEY python3 "$PLAN" --root "$LIB" --enum "$ENUM" \
|
||||
--titles "$TITLES" --probed "$PROBED" --tmdb-cache "$CACHE" \
|
||||
--movies "$MOVIES" --tmdb --out "$OUT" --exceptions "$EXC"
|
||||
[ "$status" -eq 0 ]
|
||||
grep -q 'tmdb NOT ASKED (TMDB_API_KEY unset): NOT provider-complete' <<<"$output"
|
||||
grep -q 'WARNING: tmdb NOT ASKED (TMDB_API_KEY unset) — NOT provider-complete' "$EXC"
|
||||
grep -qE '^plan: [1-9][0-9]* moves' <<<"$output"
|
||||
}
|
||||
|
||||
@test "the census exports the provider's episode numbering as a TSV" {
|
||||
DB="$BATS_TEST_TMPDIR/jellyfin.db"
|
||||
python3 - "$DB" "$SEASON" <<'PY'
|
||||
@@ -125,7 +166,7 @@ sid = str(uuid.uuid4())
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples",
|
||||
pre, None, None, 2010, None, 0,
|
||||
'{"ProviderIds":{"Tmdb":"1234"}}'))
|
||||
'{"ProviderIds":{"Tmdb":"1234","Tvdb":"78850"}}'))
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), sid, "MediaBrowser.Controller.Entities.TV.Episode",
|
||||
"Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1,
|
||||
@@ -136,34 +177,113 @@ PY
|
||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
run head -1 "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "${lines[0]}" = "file series year season episode extra tmdb" ]
|
||||
[ "${lines[0]}" = "file series year season episode extra tmdb title tvdb" ]
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234$"
|
||||
| grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234 Pilot 78850$"
|
||||
}
|
||||
|
||||
@test "an episode its own name states files as Show SxxEyy under Season NN" {
|
||||
@test "the census reads provider ids from the table newer Jellyfin keeps" {
|
||||
DB="$BATS_TEST_TMPDIR/jellyfin12.db"
|
||||
python3 - "$DB" "$SEASON" <<'PY'
|
||||
import os, sqlite3, sys, uuid
|
||||
db, season = sys.argv[1], sys.argv[2]
|
||||
c = sqlite3.connect(db)
|
||||
pre = "/data/TVShows/" + os.path.basename(season)
|
||||
c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber,
|
||||
ParentIndexNumber, ProductionYear, ExtraType,
|
||||
IsVirtualItem, Data)""")
|
||||
# Jellyfin 12 moved the ids out of the Data blob and into their own table.
|
||||
c.execute("create table BaseItemProviders (ItemId, ProviderId, ProviderValue)")
|
||||
sid = str(uuid.uuid4())
|
||||
eid = str(uuid.uuid4())
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples",
|
||||
pre, None, None, 2010, None, 0, '{}'))
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(eid, sid, "MediaBrowser.Controller.Entities.TV.Episode",
|
||||
"Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1,
|
||||
None, None, 0, '{}'))
|
||||
c.execute("insert into BaseItemProviders values (?,?,?)", (sid, "Tvdb", "9999"))
|
||||
c.commit()
|
||||
PY
|
||||
run python3 "$CENSUS" jellyfin --root "$LIB" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp12.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp12.tsv" \
|
||||
| grep -q " Pilot 9999$"
|
||||
}
|
||||
|
||||
@test "an episode its own name states files as the titled standard name" {
|
||||
plan
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv")" \
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples S01E01.mkv" ]
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.mkv" ]
|
||||
}
|
||||
|
||||
@test "the tvdb identity names the series folder, not the name a folder left behind" {
|
||||
printf '{"tvdb:series:78850": {"name": "Show Box", "year": "2016"}}\n' > "$CACHE"
|
||||
plan
|
||||
[ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" \
|
||||
= "$LIB/Show Box (2016)/Season 01/Show Box (2016) - S01E01 - Pilot.mkv" ]
|
||||
}
|
||||
|
||||
@test "a provider name that already carries the year is not given it twice" {
|
||||
# TVDb answers the reboot as `Acapulco (2021)`. The folder contract is ONE year,
|
||||
# and `Acapulco (2021) (2021)` is a folder neither matcher nor human reuses.
|
||||
printf '{"tvdb:series:78850": {"name": "Show Box (2016)", "year": "2016"}}\n' > "$CACHE"
|
||||
plan
|
||||
[ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" \
|
||||
= "$LIB/Show Box (2016)/Season 01/Show Box (2016) - S01E01 - Pilot.mkv" ]
|
||||
run grep -F '(2016) (2016)' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "a series no authority resolves keeps the provider's name and is reported" {
|
||||
plan
|
||||
[ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \
|
||||
= "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ]
|
||||
grep -A9 -i "no tvdb" "$EXC" | grep -q "SNL Pack"
|
||||
}
|
||||
|
||||
@test "a row whose series is the park folder is not evidence of a show" {
|
||||
plan_park
|
||||
run awk -F '\t' -v p="$PARK/" '$1 == "move" && index($3, p) == 1 {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
run awk -F '\t' '$1 == "move" && $2 ~ /Park Show S01E01/ {print $3}' "$OUT"
|
||||
[[ "$output" == "$OUTSIDE/"* ]]
|
||||
}
|
||||
|
||||
@test "the pile's sanitized display name is still the pile, not a series" {
|
||||
# Jellyfin drops the leading underscore when it shows the hidden folder, so the
|
||||
# row says the series is "unsorted" while the file sits in `_unsorted`. Match
|
||||
# only the underscored spelling and the plan renames real files into a visible
|
||||
# folder named after the review pile — which is how 617 of them got there.
|
||||
title_row "$PARK/Twisted.Metal.S01E01.1080p.WEB-DL.mkv" "unsorted" "" 1 1 \
|
||||
"" "" "Twisted Metal"
|
||||
plan_park
|
||||
run awk -F '\t' -v l="$LIB" \
|
||||
'$1 == "move" && index($3, l "/unsorted/") == 1 {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
run awk -F '\t' '$1 == "move" && $2 ~ /Twisted\.Metal\.S01E01/ {print $3}' "$OUT"
|
||||
[[ "$output" == "$OUTSIDE/"* ]]
|
||||
}
|
||||
|
||||
@test "a subtitle follows its episode and keeps its language flag" {
|
||||
plan
|
||||
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt")" \
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples S01E01.en.srt" ]
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.en.srt" ]
|
||||
}
|
||||
|
||||
@test "a subtitle two folders down inside Subs/<release-name>/ still follows its episode" {
|
||||
plan
|
||||
[ "$(dst "$SEASON/Subs/The.Examples.S01E01.Pilot.1080p.x265/The.Examples.S01E01.Pilot.1080p.x265.jpn.srt")" \
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples S01E01.jpn.srt" ]
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.jpn.srt" ]
|
||||
}
|
||||
|
||||
@test "a subtitle that carries the container's word in its own name still follows the episode" {
|
||||
plan
|
||||
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv.forced.srt")" \
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples S01E01.forced.srt" ]
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.forced.srt" ]
|
||||
}
|
||||
|
||||
@test "a season image keeps its own name; a series nfo keeps the series folder" {
|
||||
@@ -178,10 +298,10 @@ PY
|
||||
[ -z "$(awk -F '\t' '$3 ~ /Season .*teaser/ {print $3}' "$OUT")" ]
|
||||
}
|
||||
|
||||
@test "a file only the provider numbers files at the episode it is matched to" {
|
||||
@test "a file only the provider numbers files at the episode it is matched to, title or no title" {
|
||||
plan
|
||||
[ "$(dst "$ANIME/[grp] Anime Pack - 05.mkv")" \
|
||||
= "$LIB/Anime Pack/Season 01/Anime Pack S01E05.mkv" ]
|
||||
= "$LIB/Anime Pack/Season 01/Anime Pack - S01E05.mkv" ]
|
||||
}
|
||||
|
||||
@test "a bonus folder inside a release pack belongs to the pack's show, not to its own name" {
|
||||
@@ -193,19 +313,19 @@ PY
|
||||
@test "a Season folder outvotes the stale row its files carry" {
|
||||
plan
|
||||
[ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \
|
||||
= "$LIB/SNL Pack/Season 19/SNL Pack S19E01.avi" ]
|
||||
= "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ]
|
||||
}
|
||||
|
||||
|
||||
@test "one file holding two episodes keeps both numbers in its name" {
|
||||
plan
|
||||
[ "$(dst "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv")" \
|
||||
= "$LIB/Show Two (2016)/Season 01/Show Two S01E02-E03.mkv" ]
|
||||
= "$LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E02-E03 - Double Trouble.mkv" ]
|
||||
}
|
||||
|
||||
@test "two files claiming one episode: one keeps the name, the other parks" {
|
||||
plan
|
||||
n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two S01E01.mkv " "$OUT")
|
||||
n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E01 - Two Pilots.mkv " "$OUT")
|
||||
[ "$n" -eq 1 ]
|
||||
run dst "$LIB/Dupes/Show Two S01E01.mkv"
|
||||
[[ "$output" == "$PARK/"* ]]
|
||||
@@ -240,6 +360,38 @@ json.dump({"movie:some documentary":
|
||||
[[ "$output" == "$PARK/"* ]]
|
||||
}
|
||||
|
||||
@test "a movie answer the file does not itself name stays in the review pile" {
|
||||
# TMDb's top hit for a truncated query is a NEAR miss, not an answer: "mobile
|
||||
# suit gundam 00" comes back as Special Edition I while the file is Special
|
||||
# Edition III, and a one-word query "russia" comes back as a 1963 Bond film. A
|
||||
# file leaves the TV library only when it states the title the provider names.
|
||||
printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \
|
||||
"$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED"
|
||||
python3 -c 'import json, sys
|
||||
json.dump({"movie:some documentary":
|
||||
{"name": "Some Documentary Special Part Two", "year": "2015", "id": 55}},
|
||||
open(sys.argv[1], "w"))' "$CACHE"
|
||||
plan
|
||||
run dst "$LIB/Longform/Some Documentary Special (2015).mkv"
|
||||
[[ "$output" == "$PARK/"* ]]
|
||||
run awk -F '\t' -v m="$MOVIES/" '$1 == "move" && index($3, m) == 1 {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "a movie answer from another year is not that film" {
|
||||
# The year the file states outranks a provider's eagerness: the 2015 special is
|
||||
# not the 1999 one, however alike their names are.
|
||||
printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \
|
||||
"$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED"
|
||||
python3 -c 'import json, sys
|
||||
json.dump({"movie:some documentary":
|
||||
{"name": "Some Documentary Special", "year": "1999", "id": 55}},
|
||||
open(sys.argv[1], "w"))' "$CACHE"
|
||||
plan
|
||||
run dst "$LIB/Longform/Some Documentary Special (2015).mkv"
|
||||
[[ "$output" == "$PARK/"* ]]
|
||||
}
|
||||
|
||||
@test "the plan creates every folder before the move that needs it" {
|
||||
plan
|
||||
python3 - "$OUT" "$LIB" "$MOVIES" "$PARK" <<'PY'
|
||||
|
||||
+2
-72
@@ -195,7 +195,7 @@ setup() {
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 2" ]
|
||||
[[ "$output" == *"missing by extension: .log 1, .mkv 1"* ]]
|
||||
[ "${lines[4]}" = "missing by extension: .log 1, .mkv 1" ]
|
||||
}
|
||||
|
||||
@test "--ignore-ext keeps a file the server never indexes out of the verdict" {
|
||||
@@ -208,75 +208,5 @@ setup() {
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[1]}" = "landed: 1" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||
[[ "$output" == *"not indexed (extension ignored): 1"* ]]
|
||||
}
|
||||
|
||||
# wal_case <db> — a Jellyfin database whose NEW row lives only in the
|
||||
# write-ahead log: the server has committed the rename but not checkpointed it,
|
||||
# which is the normal state of a library a scan is writing. The holder
|
||||
# connection keeps the WAL from folding into the main file, and the verifier
|
||||
# runs as a real subprocess so it faces the same files Plex and Jellyfin leave.
|
||||
wal_case() {
|
||||
python3 - "$JFDB" "$JOURNAL" "$SHARE" "$VERIFY" <<'PY'
|
||||
import sqlite3, subprocess, sys
|
||||
db, journal, share, verify = sys.argv[1:5]
|
||||
w = sqlite3.connect(db)
|
||||
w.execute("pragma journal_mode=wal")
|
||||
w.execute("pragma wal_autocheckpoint=0")
|
||||
w.execute("insert into BaseItems (Path, Type) values ('/data/Movies/Old.mkv', 'x')")
|
||||
w.commit()
|
||||
w.execute("pragma wal_checkpoint(truncate)")
|
||||
holder = sqlite3.connect(db) # last-connection close would checkpoint
|
||||
w.execute("update BaseItems set Path='/data/Movies/Old/Old.mkv'")
|
||||
w.commit()
|
||||
r = subprocess.run([sys.executable, verify, "--journal", journal,
|
||||
"--db-kind", "jellyfin", "--db", db,
|
||||
"--share", share, "--container", "/data"],
|
||||
capture_output=True, text=True)
|
||||
print(r.stdout, end="")
|
||||
sys.exit(r.returncode)
|
||||
PY
|
||||
}
|
||||
|
||||
@test "a row the server committed but has not checkpointed is proof, not a miss" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
run wal_case
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[1]}" = "landed: 1" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||
}
|
||||
|
||||
# The DB alone cannot tell a lost file from a sidecar: both are "no row at the
|
||||
# destination". The tool therefore stats the two paths and separates the classes
|
||||
# the runbook names, so an operator never reads a `.pdf` as a lost film or,
|
||||
# worse, reads a real loss as a sidecar.
|
||||
@test "a file the database does not name that still sits at src is the pass's own damage" {
|
||||
mkdir -p "$SHARE/Movies"
|
||||
echo film > "$SHARE/Movies/Old.mkv"
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 1" ]
|
||||
[ "${lines[4]}" = " on disk at dst, no row names it: 0" ]
|
||||
[ "${lines[5]}" = " REAL — file still at src: 1" ]
|
||||
[ "${lines[6]}" = " gone — at neither path: 0" ]
|
||||
[[ "$output" == *"real: $SHARE/Movies/Old.mkv"* ]]
|
||||
}
|
||||
|
||||
@test "a file at neither path is gone, and a file at dst the server ignores is not a loss" {
|
||||
move_row "$SHARE/Movies/Gone.mkv" "$SHARE/Movies/Gone/Gone.mkv"
|
||||
mkdir -p "$SHARE/Movies/Sidecar"
|
||||
echo cover > "$SHARE/Movies/Sidecar/cover.jpg"
|
||||
move_row "$SHARE/Movies/Album/cover.jpg" "$SHARE/Movies/Sidecar/cover.jpg"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 2" ]
|
||||
[ "${lines[4]}" = " on disk at dst, no row names it: 1" ]
|
||||
[ "${lines[5]}" = " REAL — file still at src: 0" ]
|
||||
[ "${lines[6]}" = " gone — at neither path: 1" ]
|
||||
[[ "$output" == *"gone: $SHARE/Movies/Gone/Gone.mkv"* ]]
|
||||
[ "${lines[4]}" = "not indexed (extension ignored): 1" ]
|
||||
}
|
||||
Reference in new issue
Block a user