feat(media): per-episode TV duplicate census, and the Jellyfin library that indexed but fetched nothing #116

Merged
dank merged 4 commits from feat/jellyfin-provider-keys into main 2026-11-02 02:03:03 -08:00
9 changed files with 994 additions and 11 deletions

No files matched your search

+61 -1
View File
@@ -35,7 +35,8 @@
# Subcommands # Subcommands
# plan print the resolved shape and every pending change, touch nothing # plan print the resolved shape and every pending change, touch nothing
# install fetch+verify the plugin, then write the keys (backing up first) # install fetch+verify the plugin, then write the keys (backing up first)
# verify prove the plugin and both keys are actually in place # verify prove the plugin and both keys are in place, and that no library
# is a silent file index (providers off / image fetchers empty)
# #
# Nothing here restarts Jellyfin: the deployer owns that # Nothing here restarts Jellyfin: the deployer owns that
# (`scripts/deploy.sh dankserver --service jellyfin`), and the plugin is read at # (`scripts/deploy.sh dankserver --service jellyfin`), and the plugin is read at
@@ -340,6 +341,64 @@ cmd_install() {
fi fi
} }
# A library whose options.xml carries EnableInternetProviders=false, or an empty
# ImageFetchers list beside a populated MetadataFetchers list, is a SILENT FILE
# INDEX: it indexes every file, answers 204 to every refresh, stores no provider
# id, plot, rating or artwork, and logs nothing — so the item count reads like a
# healthy scan. Jellyfin 12 keeps these options in root/default/<Library>/options.xml
# (it was library.xml) and stores NONE of them in the database, so this audit is
# the only place the state can be read without a browser session.
verify_library_options() {
local root="$JF_ROOT/root/default" f lib problems
if [ ! -d "$root" ]; then
printf '= no library options under %s (nothing audited)\n' "$root"
return 0
fi
local rc=0
for f in "$root"/*/options.xml; do
[ -f "$f" ] || continue
lib="$(basename "$(dirname "$f")")"
problems="$(python3 -s - "$f" <<'PY'
import sys, xml.dom.minidom as X
root = X.parse(sys.argv[1]).documentElement
def child(el, name):
for n in el.childNodes:
if n.nodeType == n.ELEMENT_NODE and n.tagName == name:
return n
def flag(el, name):
n = child(el, name)
return None if n is None else (n.firstChild.data.strip() if n.firstChild else "")
def names(el, name):
n = child(el, name)
return None if n is None else [s.firstChild.data.strip() for s in n.childNodes
if s.nodeType == s.ELEMENT_NODE and s.firstChild]
bad = []
ip = flag(root, "EnableInternetProviders")
if ip is not None and ip.lower() != "true":
bad.append("EnableInternetProviders=%s" % (ip or "empty"))
types = child(root, "TypeOptions")
if types is not None:
for t in types.childNodes:
if t.nodeType != t.ELEMENT_NODE or t.tagName != "TypeOptions":
continue
meta = names(t, "MetadataFetchers") or []
if meta and not (names(t, "ImageFetchers") or []):
bad.append("%s: ImageFetchers empty while MetadataFetchers names %d"
% (flag(t, "Type") or "?", len(meta)))
print("; ".join(bad))
PY
)"
if [ -n "$problems" ]; then
printf 'SILENT INDEX %s: %s\n' "$lib" "$problems" >&2
rc=1
else
printf '= %s: library options fetch from providers\n' "$lib"
fi
done
return "$rc"
}
cmd_verify() { cmd_verify() {
check_root check_root
local bad=0 local bad=0
@@ -378,6 +437,7 @@ cmd_verify() {
printf '%s %s chars\n' "$elem" "${#n}" printf '%s %s chars\n' "$elem" "${#n}"
fi fi
done done
verify_library_options || bad=1
[ "$bad" -eq 0 ] || return 1 [ "$bad" -eq 0 ] || return 1
printf 'providers configured\n' printf 'providers configured\n'
} }
+473
View File
@@ -0,0 +1,473 @@
#!/usr/bin/env python3
"""media-tvdupes.py — find duplicate TV episodes and plan their staging, one
EPISODE at a time.
A TV library grows the same show into several folders: a tagged season box, an
untagged packager drop, a `Complete` folder that is not complete. The folders
look like copies of each other and are not, so the unit of comparison is the
episode and never the folder. A file is droppable only when EVERY episode it
carries is also held, somewhere else in the same show, by a file at least as
large — Frank's rule: the same episode in two places keeps the larger one. A
file that holds one episode nothing else has stays, whatever else it carries.
It writes a plan and moves nothing; `scripts/media-apply.py` is the only thing
that renames. The plan stages into a dated hold, so the pass is undoable from
the journal and frees no space until the hold is purged.
python3 scripts/media-tvdupes.py --library /mnt/entropy/TVShows \
--hold /mnt/entropy/.AGENTTRASH/20261102-tv-duplicates \
--out /mnt/entropy/tv-dupes-plan.tsv
python3 scripts/media-apply.py apply --plan /mnt/entropy/tv-dupes-plan.tsv \
--root /mnt/entropy --journal /mnt/entropy/tv-dupes.jsonl \
--metadata-view /var/lib/media-meta/view --apply
python3 scripts/media-apply.py verify --plan /mnt/entropy/tv-dupes-plan.tsv \
--root /mnt/entropy --journal /mnt/entropy/tv-dupes.jsonl
python3 scripts/media-apply.py sweep --root /mnt/entropy \
--journal /mnt/entropy/tv-dupes.jsonl --apply # empty folders only
scripts/media-view.sh reap # ghost sidecars above
Then both servers have to be told: the plan moves files on the raw array, and no
watcher crosses the metadata view — [[Media Relayout]] owns the refresh order.
Guards, and why each one is here:
* **Identity before geometry.** Two folders holding the same words are not the
same show: the 2026-11-02 census found `Avatar: The Last Airbender` (1999,
tvdb 74852) beside the 2024 remake (385925), `Star Trek: Voyager` beside
`Star Trek: The Animated Series`, and a German thriller Jellyfin had titled
`Millennium (2022) [tvdbid-472933]` beside `Millennium (1996)`. A folder's
identity is read from three places, in this order: its `[tvdbid-N]` tag, the
`tvshow.nfo` Jellyfin wrote — which lives in the metadata view, not on the
array, and is what `--meta` reads — and a `folder<TAB>id` map passed with
`--ids`. The tag outranks the derived layers: an id that contradicts a folder's
own tag is reported and ignored, never unioned — measured 2026-11-02, Jellyfin's
provider table carried a 2026 special's id on a folder tagged `[tvdbid-73838]`,
and unioning them makes one show of the 2000 series and the special, whose bigger
files then dominate the original's pilot. Two folders are one show when they share
an id, or when neither has one and name and year both agree. Two disjoint id sets
is an absolute stop: the pair is reported as a conflict and nothing under either is
staged. Same name with nothing to decide on is reported unresolved — the fix is the
tag, not a guess.
* **A folder-level verdict destroys content.** 10 of the 25 shows that looked
duplicated held COMPLEMENTARY halves: `Game of Thrones Complete` carries no
S01 and no S08, `Modern Family S05` is the only copy of season 5, `Loiter
Squad Uncensored` is the only copy of season 1. Judged per folder, 20 shows
and some 340 GB of unique episodes would have been staged; judged per episode
the true figure was a sixth of that.
* **A bare number is never an episode.** `256491.mp4` may be an episode id, a
poster, or a season, so a name that states no `SxxEyy` / `1x04` marker is
unkeyed: never staged, and counted in the report. A media server's own episode
table does know those files — pass it with `--witness` and its rows key
whatever the server has matched.
* **The audit re-reads the disk.** Sizes and the survivor's existence are
checked again when the plan is written, because the census is a snapshot: a
file that grew, vanished, or landed in the hold in between must fail the plan,
not surprise the executor.
* **A sidecar that shares an episode's stem follows it.** Subtitles and `.nfo`
left at the old coordinates are ghost bait for `media-view.sh reap`.
* A multi-episode file (`S01E01-02`) is dominated only when every episode in its
range is held elsewhere — and it competes for the survivor seat on each of
them, because the byte law compares files. A pack that states its range in
the name is a name both media servers parse the same way, so the pack winning
against two single-episode rips is the law applied, not a misread.
"""
import argparse
import collections
import importlib.util
import json
import os
import re
import sys
ENGINE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "media-titles.py")
_spec = importlib.util.spec_from_file_location("eng", ENGINE)
eng = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(eng)
# The executor's definition of a media file, plus the two the title engine omits:
# an audit that disagrees with media-apply.py is a plan that dies at its
# preflight instead of at the census.
VIDEO = tuple(sorted(set(eng.VIDEO_SUFFIX) | {".ts", ".wmv"}))
TAG_RE = re.compile(r"\[\s*tvdbid[\s._-]*(\d+)\s*\]", re.I)
YEAR_RE = re.compile(r"[(\[]\s*((?:19|20)\d\d)\s*[)\]]")
SEASON_EP = re.compile(r"(?i)\bs(?:eason)?[\s._-]*(\d{1,2})[\s._-]*e(?:p(?:isode)?)?"
r"[\s._-]*(\d{1,3})(?:[\s._-]*(?:-|to|\+)[\s._-]*e?"
r"[\s._-]*(\d{1,3}))?")
NXN = re.compile(r"(?i)(?<!\d)(\d{1,2})x(\d{1,3})(?!\d)")
def parse_args():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--library", default="/mnt/entropy/TVShows",
help="the TV library to census (never written)")
ap.add_argument("--hold", required=True,
help="dated trash the plan stages into, e.g. "
"/mnt/entropy/.AGENTTRASH/20261102-tv-duplicates")
ap.add_argument("--out", default=None,
help="plan TSV for scripts/media-apply.py (default: "
"tv-dupes-plan.tsv beside --library)")
ap.add_argument("--witness", default=None,
help="TSV of path<TAB>season<TAB>episode out of a media "
"server's episode table; keys files whose names carry"
" no marker")
ap.add_argument("--meta", default=None,
help="the merged metadata view (on the host "
"/var/lib/media-meta/view): tvshow.nfo lives in the"
" upper layer, so this is where an untagged folder's"
" tvdb id is readable")
ap.add_argument("--ids", default=None,
help="TSV of folder<TAB>tvdbid, one line per folder, for"
" identity a lookup settled and the folder name does"
" not state")
ap.add_argument("--json", default=None,
help="also dump the analysis as JSON")
return ap.parse_args()
ARGS = parse_args()
def episode_keys(name):
"""Every (season, episode) an episode name states.
`S01E03`, `Season 1 E3`, `1x04`, and a range `S01E01-02`, which expands so a
two-in-one file is only droppable when both halves are held elsewhere. A
bare number states nothing and yields nothing."""
keys = set()
for m in SEASON_EP.finditer(name):
s, first = int(m.group(1)), int(m.group(2))
keys.add((s, first))
if m.group(3):
keys.update((s, e) for e in range(first + 1, int(m.group(3)) + 1))
keys.update((int(m.group(1)), int(m.group(2))) for m in NXN.finditer(name))
return keys
def folder_identity(folder):
"""(fold_sep name, year, tvdb ids) for a top-level show folder.
Tag and year come in both orders — `Name (2008) [tvdbid-84859]` and `Name
[tvdbid-84859] (2008)` are both in the library — so neither is assumed to
sit at the end, and both are cut out of the name folders compare on."""
ids = frozenset(int(x) for x in TAG_RE.findall(folder))
year = None
for _ in range(3):
y = YEAR_RE.search(folder)
if not y:
break
year = year or y.group(1)
folder = TAG_RE.sub(" ", f"{folder[:y.start()]} {folder[y.end():]}")
return eng.fold_sep(TAG_RE.sub(" ", folder)), year, ids
def nfo_id(path):
"""The tvdb id a `tvshow.nfo` states, or None.
Three shapes are in the library: Jellyfin's `<uniqueid type="tvdb">`, Kodi's
`<tvdbid>`, and a packager's `<id>` + `<id_type>` pair; some carry the id
only in the title line as `[tvdbid-N]`."""
try:
with open(path, encoding="utf-8", errors="replace") as fh:
text = fh.read(16384)
except OSError:
return None
for rx in (r"<tvdbid\s*>\s*(\d+)\s*<",
r"<uniqueid[^>]*type\s*=\s*[\"']?tvdb[^>]*>\s*(\d+)\s*<",
r"<id\s*>\s*(\d+)\s*</id>\s*<id_type>\s*(?:tvdb|thedotdb)",
r"\[\s*tvdbid[\s._-]*(\d+)\s*\]"):
m = re.search(rx, text, re.I)
if m:
return int(m.group(1))
return None
def collect_ids(library, meta, ids_file):
"""top-level folder -> {tvdb ids} from the nfo layer and an operator map.
`meta` is the merged metadata view: Jellyfin's sidecars live in the upper
layer, so a folder's `tvshow.nfo` is not on the raw array. Reading the id
from there is what makes an untagged packager folder identifiable at all —
without it the pair stays unresolved and nothing under it is staged."""
out = collections.defaultdict(set)
if meta:
try:
tops = sorted(os.listdir(library))
except OSError:
tops = []
for folder in tops:
i = nfo_id(os.path.join(meta, folder, "tvshow.nfo"))
if i:
out[folder].add(i)
if ids_file:
with open(ids_file, encoding="utf-8", errors="replace") as fh:
for line in fh:
cols = line.rstrip("\n").split("\t")
if len(cols) < 2 or not cols[1].strip().isdigit():
continue
out[cols[0]].add(int(cols[1].strip()))
return out
def witness_rows(path):
"""absolute path -> {(season, episode)} from a media server's episode table."""
out = collections.defaultdict(set)
if not path:
return out
with open(path, encoding="utf-8", errors="replace") as fh:
for line in fh:
cols = line.rstrip("\n").split("\t")
if len(cols) < 3 or not cols[0].startswith("/"):
continue
try:
out[os.path.abspath(cols[0])].add((int(cols[1]), int(cols[2])))
except ValueError:
continue
return out
def sidecars(path):
"""The files beside PATH carrying its exact stem — subtitles, nfo, artwork."""
folder, stem = os.path.split(path)
stem = os.path.splitext(stem)[0]
out = []
try:
entries = sorted(os.listdir(folder))
except OSError:
return out
for e in entries:
p = os.path.join(folder, e)
if p == path or os.path.islink(p) or not os.path.isfile(p):
continue
if os.path.splitext(e)[0] == stem:
out.append(p)
return out
def scan(library, hold, witness):
"""Every video file under LIBRARY: relative path, size, episode keys.
A file keys from the witness when it has a row (a server matched it), and
from its own name otherwise; with neither it is unkeyed."""
files = []
for dirpath, dirnames, filenames in os.walk(library):
dirnames[:] = sorted(d for d in dirnames
if not d.startswith(".") and d != "@eaDir"
and os.path.abspath(os.path.join(dirpath, d)) != hold)
for f in sorted(filenames):
p = os.path.join(dirpath, f)
if os.path.islink(p) or not f.lower().endswith(VIDEO):
continue
try:
size = os.path.getsize(p)
except OSError:
continue
files.append({"path": os.path.abspath(p),
"rel": os.path.relpath(os.path.abspath(p), library),
"size": size,
"keys": set(witness.get(os.path.abspath(p))
or episode_keys(os.path.splitext(f)[0]))})
return files
def show_groups(folders, extra_ids):
"""(show groups, conflicts, unresolved, disagreements) over top-level folders.
A folder's id is its `[tvdbid-N]` tag; the nfo layer and `--ids` fill a folder
only when it carries no tag. The rule, in order of trust: an id in common makes
one show; two disjoint id sets are two shows, reported as a conflict and never
merged; no id on either side merges only on name AND year (the 1999 miniseries
and the 2015 series are not one show); one side ided and the other bare is
unresolved, because the fix is the tag.
The derived layers never overrule a tag. Measured 2026-11-02: Jellyfin's
provider table carried a 2026 special's tvdb id on the folder tagged
`[tvdbid-73838]`, and unioning the two made one show of the 2000 series and the
special — whose bigger files then dominate the original's pilot. A contradiction
is reported instead, because the tag is the operator's word and the provider
table is a guess that can be wrong.
"""
names = {f: folder_identity(f) for f in folders}
tags = {f: set(names[f][2]) for f in folders}
derived = {f: set(extra_ids.get(f, ())) for f in folders}
disagree = {f: (sorted(tags[f]), sorted(derived[f] - tags[f]))
for f in folders if tags[f] and derived[f] - tags[f]}
ids = {f: (tags[f] or derived[f]) for f in folders}
parent = {f: f for f in folders}
def find(f):
while parent[f] != f:
parent[f] = parent[parent[f]]
f = parent[f]
return f
conflicts, unresolved = [], []
order = sorted(folders)
for i, a in enumerate(order):
for b in order[i + 1:]:
na, ya, _ia = names[a]
nb, yb, _ib = names[b]
if na != nb:
continue
ia, ib = ids[a], ids[b]
if ia & ib:
parent[find(b)] = find(a)
elif ia and ib:
conflicts.append((a, b, f"tvdb {sorted(ia)} vs {sorted(ib)}"))
elif not ia and not ib and ya and ya == yb:
parent[find(b)] = find(a)
elif not ia and not ib:
conflicts.append((a, b,
f"same name, years {ya or '?'} vs {yb or '?'}"))
else:
unresolved.append((a, b, "one folder carries a tvdb id,"
" the other none"))
groups = collections.defaultdict(list)
for f in order:
groups[find(f)].append(f)
return sorted(groups.values()), conflicts, unresolved, disagree
def rank(rec):
"""Survivor order: biggest, then shortest path, then lexicographic.
A strict total order per key means a key's winner is never itself dominated,
so no chain of staging can cycle and exactly one copy of a byte-identical
pair is staged."""
return (-rec["size"], len(rec["path"]), rec["path"])
def main():
library = os.path.abspath(ARGS.library)
hold = os.path.abspath(ARGS.hold)
out = ARGS.out or os.path.join(os.path.dirname(library), "tv-dupes-plan.tsv")
if not os.path.isdir(library):
sys.exit(f"refused: {library} is not a directory")
if hold == library or hold.startswith(library + os.sep):
sys.exit(f"refused: --hold {hold} sits inside the library; a hold the"
" census walks is a plan that eats its own output")
files = scan(library, hold, witness_rows(ARGS.witness))
folders = sorted({r["rel"].split(os.sep)[0] for r in files})
ids = collect_ids(library, ARGS.meta, ARGS.ids)
groups, conflicts, unresolved, disagree = show_groups(folders, ids)
by_show = collections.defaultdict(list)
for members in groups:
for r in files:
if r["rel"].split(os.sep)[0] in members:
by_show[members[0]].append(r)
rows, audits, report, shows_json = [], [], [], []
for show in sorted(by_show):
recs = by_show[show]
holders = collections.defaultdict(list)
for r in recs:
for k in r["keys"]:
holders[k].append(r)
winner = {k: min(v, key=rank) for k, v in holders.items()}
dominated = [r for r in recs if r["keys"]
and all(winner[k] is not r for k in r["keys"])]
if not dominated:
continue
dup_keys = sorted({k for r in dominated for k in r["keys"]})
sole = sum(1 for v in holders.values() if len(v) == 1)
staged_folders = {os.path.dirname(r["rel"]) for r in dominated}
for r in dominated:
for k in r["keys"]:
t = winner[k]
try:
now_src = os.path.getsize(r["path"])
now_twin = os.path.getsize(t["path"])
except OSError:
audits.append(f"gone since the census: {r['rel']}"
f" or its survivor {t['rel']}")
break
if now_twin < now_src:
audits.append(f"survivor smaller on disk: {r['rel']}"
f" {now_src} > {t['rel']} {now_twin}")
for r in dominated:
dst = os.path.join(hold, r["rel"])
rows.append((r["path"], dst))
for side in sidecars(r["path"]):
rows.append((side, os.path.join(hold, os.path.relpath(side, library))))
shows_json.append({"show": show, "files": len(dominated),
"bytes": sum(r["size"] for r in dominated),
"episodes": len(dup_keys),
"sole-episode files left": sole,
"folders": sorted(staged_folders)})
n_ep, n_folder = len(dup_keys), len(staged_folders)
report.append(
f"{show}: stage {len(dominated)} files"
f" ({sum(r['size'] for r in dominated) / 2**30:.1f} GiB) — {n_ep}"
f" episode{n_ep != 1 and 's' or ''} held twice or more; {sole}"
f" episode{sole != 1 and 's' or ''} exist in one copy only, so"
f" {n_folder} folder{n_folder != 1 and 's' or ''} keep their files")
rows = sorted(set(rows))
dests = set()
for src, dst in rows:
if dst in dests:
audits.append(f"destination twice: {dst}")
dests.add(dst)
if os.path.exists(dst):
audits.append(f"hold target occupied: {dst}")
if not os.path.isfile(src):
audits.append(f"source missing: {src}")
unkeyed = collections.Counter(os.path.dirname(r["rel"]) for r in files
if not r["keys"])
if ARGS.json:
with open(ARGS.json, "w") as fh:
json.dump({"shows": shows_json,
"conflicts": [list(c) for c in conflicts],
"unresolved": [list(u) for u in unresolved],
"disagreement": {f: [t, d] for f, (t, d) in disagree.items()},
"unkeyed": dict(unkeyed)}, fh, indent=1)
for line in report:
print(line)
for a, b, why in conflicts:
print(f"CONFLICT (never merged): {a} | {b} — {why}")
for a, b, why in unresolved:
print(f"UNRESOLVED (tag it, then re-run): {a} | {b} — {why}")
for f, (tagged, extra) in sorted(disagree.items()):
print(f"DISAGREE (the tag wins): {f} — tag tvdb {tagged},"
f" derived tvdb {extra}")
if unkeyed:
print(f"unkeyed, never staged: {sum(unkeyed.values())} files in"
f" {len(unkeyed)} folders name no SxxEyy and have no --witness row;"
" most: " + ", ".join(f"{d} ({n})" for d, n in unkeyed.most_common(5)))
# The gate runs before the plan is written: a plan file that exists is a plan
# somebody will point the executor at, so a failed audit leaves no artifact.
if audits:
print(f"AUDIT FAILED ({len(audits)})", file=sys.stderr)
for a in audits[:10]:
print(f" {a}", file=sys.stderr)
print(f"refused: no plan written to {out}", file=sys.stderr)
return 1
mkdirs = sorted({os.path.dirname(dst) for _s, dst in rows},
key=lambda p: (p.count(os.sep), p))
with open(out, "w") as fh:
fh.write("action\tsrc\tdst\tkind\n")
for d in mkdirs:
fh.write(f"mkdir\t\t{d}\tdir\n")
for src, dst in rows:
fh.write(f"move\t{src}\t{dst}\ttvdup\n")
held = sum(os.path.getsize(s) for s, _d in rows if os.path.isfile(s))
print(f"plan: {len(rows)} moves ({held / 2**30:.1f} GiB) into"
f" {len(mkdirs)} hold folders -> {out}")
print("audit: every staged file has a survivor in the same show holding each of"
" its episodes at >= its size on disk, one destination each")
if rows:
root = os.path.dirname(library)
print(f"next: python3 scripts/media-apply.py apply --plan {out} --root"
f" {root} --journal {os.path.join(root, 'tv-dupes.jsonl')}"
" --metadata-view /var/lib/media-meta/view (dry run; add --apply)")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+36 -4
View File
@@ -46,10 +46,14 @@ sees is the [[Media Metadata View]] overlay, live since the 2026-10-31 cutover:
`/var/lib/media-meta/view:/data`, read-write. `/var/lib/media-meta/view:/data`, read-write.
`Local metadata` — save metadata to media folders — is enabled, and it is the `Local metadata` — save metadata to media folders — is enabled, and it is the
reason the view exists: `movie.nfo`, `tvshow.nfo`, `episode.nfo` and artwork reason the view exists: `movie.nfo`, `tvshow.nfo` and `episode.nfo` written once
written once on the NVMe upper layer are files [[Plex]] can read too — and since on the NVMe upper layer are files [[Plex]] can read too — and since the
the 2026-10-31 cutover that is what happens: Jellyfin is the fleet's only 2026-10-31 cutover that is what happens: Jellyfin is the fleet's only metadata
metadata writer, and every sidecar it saves is an SSD byte the array never sees. writer, and every sidecar it saves is an SSD byte the array never sees.
**Artwork is not one of those sidecars.** No image saver is bound for any
library, so posters and fanart go to the internal store
(`config/metadata/library/<xx>/<item-id>/poster.jpg`), which [[Plex]] never
reads — movie art has never existed beside the media on either layer.
**Library coverage is the first thing to check when a title is missing.** The **Library coverage is the first thing to check when a title is missing.** The
four libraries are `movies → /data/Movies`, `tvshows → /data/TVShows`, four libraries are `movies → /data/Movies`, `tvshows → /data/TVShows`,
@@ -66,6 +70,34 @@ instead of the view — always a fault on this host since the 2026-10-31 cutover
`scripts/media-view.sh status` says which: the quadlet drifted back to the raw `scripts/media-view.sh status` says which: the quadlet drifted back to the raw
array, or the mount died. array, or the mount died.
**`EnableInternetProviders` false turns a library into a silent file index.**
Every refresh answers `204`, stores nothing and logs nothing — no provider ids,
no plot, no rating, no art — while the item count reads like a healthy scan. The
switch lives in `config/root/default/<Library>/options.xml`, Jellyfin 12's name
for the per-library option file: grepping for the old `library.xml` concludes "no
per-library options exist", and the database holds none, so the trail dies there.
Two more traps in the same file: `TypeOptions/ImageFetchers` is the *enabled* set,
so an empty `<ImageFetchers />` switches art off while `ImageFetcherOrder` stays
fully populated — order is not enablement — and the populated `<MetadataFetchers>`
built next to it makes the library look healthy.
**`RemoteImages` is the call that separates a provider fault from a library
fault.** `GET /Items/{id}/RemoteImages?Type=Primary` returns the provider's real
poster list even with every automatic pass switched off — plugin, key and CDN all
proven by that one response — so a refresh that stores nothing beside it is a
library-switch problem, never a provider problem. It is also the delivery door:
`POST /Items/{id}/RemoteImages/Download` saves the chosen image to the internal
store on a build where the automatic image pass will not run at all — it is how
the Movies library got its art: 2,515 of 2,564 films carry a primary now, and the
rest have none in TMDb (shorts like `Astartes`, files whose names match nothing).
**API shape on 12.x:** `/Items` and `/Items/{id}` need `UserId` when
authenticated by API key — without it the answer is `400 Error processing
request`, which reads exactly like an authentication failure. `MetadataRefreshMode`
accepts `Default` (and `Realtime`) only; `Full` and `ValidationFailure` are
rejected outright, and what forces the providers to re-run is
`ReplaceAllMetadata=true`.
## Ports & access ## Ports & access
| Port | Proto | Purpose | | Port | Proto | Purpose |
@@ -1,9 +1,9 @@
--- ---
type: reference type: reference
title: Media Duplicate Prune title: Media Duplicate Prune
description: "Find and hold duplicate films on the entropy array: the hash ladder, the keep rules, the guards, the purge." description: "Find and hold duplicate films and series on the array: the screen, per-episode rule, guards, purge."
tags: [dankserver, media-server, storage, procedure] tags: [dankserver, media-server, storage, procedure]
timestamp: 2026-11-01T01:00:00Z timestamp: 2026-11-02T01:00:00Z
--- ---
# Media Duplicate Prune # Media Duplicate Prune
@@ -13,6 +13,10 @@ downloaded again, or the same film grabbed a second time at another quality —
[[Entropy Volume]] is permanently full, so both are worth finding. They need [[Entropy Volume]] is permanently full, so both are worth finding. They need
different rules, and neither is decided by name similarity alone. different rules, and neither is decided by name similarity alone.
A series duplicates differently again: one show spread over several folders, so
neither the file nor the folder is the unit — see *A series is measured per
episode*.
`/mnt/entropy/media-park` was a third kind: a parking lot that held whole shows `/mnt/entropy/media-park` was a third kind: a parking lot that held whole shows
and films nothing served. It is gone; its contents went to `NeedsProcessing`. and films nothing served. It is gone; its contents went to `NeedsProcessing`.
@@ -24,6 +28,7 @@ and films nothing served. It is gone; its contents went to `NeedsProcessing`.
| Same work, worse encode | same title, same year, every runtime within 5 % | keep the largest — Frank's rule | | Same work, worse encode | same title, same year, every runtime within 5 % | keep the largest — Frank's rule |
| Different cut | same title and year, runtimes more than 5 % apart | **never touched**: an extended and a theatrical version are different products | | Different cut | same title and year, runtimes more than 5 % apart | **never touched**: an extended and a theatrical version are different products |
| Same words, different film | title matches, year or duration does not | never grouped — this is the false-merge class that would destroy a unique file | | Same words, different film | title matches, year or duration does not | never grouped — this is the false-merge class that would destroy a unique file |
| Series split across folders | same show by tvdb id, then the episode `(S,E)` — never the folder name | keep the largest carrier of each episode (`scripts/media-tvdupes.py`) |
## The screen ## The screen
@@ -51,6 +56,84 @@ folder; then the largest. For the same-work class the largest is the answer, and
the canonical-name rule only breaks a tie — without it a `Film - 1080p.mkv` can the canonical-name rule only breaks a tie — without it a `Film - 1080p.mkv` can
outlive `Film.mkv` for no reason but a space sorting before a dot. outlive `Film.mkv` for no reason but a space sorting before a dot.
## A series is measured per episode
`scripts/media-tvdupes.py` censuses a TV library and writes a staging plan in the
executor's own format; it never moves anything itself.
```bash
python3 scripts/media-tvdupes.py --library /mnt/entropy/TVShows \
--hold /mnt/entropy/.AGENTTRASH/<date>-tv-duplicates \
--meta /var/lib/media-meta/view/TVShows --witness witness.tsv \
--ids /mnt/entropy/tv-tvdb-ids.tsv \
--out /mnt/entropy/tv-dupes-plan.tsv --json /tmp/tv-groups.json
# then media-apply.py apply / verify / sweep / rollback, exactly as for a film prune
```
- **Identity before geometry.** A folder's tvdb id is read from its `[tvdbid-N]`
name tag, from the `tvshow.nfo` the [[Media Metadata View]] exposes (`--meta`),
and from a `folder<TAB>id` map (`--ids`). Build that map from Jellyfin's own
match, not a lookup: `BaseItems` (`Type like '%TV.Series'`) joined to
`BaseItemProviders` where `ProviderId = 'Tvdb'`, the `/data/TVShows/` prefix
stripped to the bare top-level folder name — 616 folders, none carrying two
ids, written to `/mnt/entropy/tv-tvdb-ids.tsv`. Two folders with disjoint ids
are two shows however alike they read — *Avatar: The Last Airbender* (1999)
beside the 2024 remake, *Millennium* (1996) beside a German thriller Jellyfin
had also titled *Millennium* — and a pair whose identity cannot be proven is
reported `UNRESOLVED` and left alone. The fix is the tag, never a guess.
- **A tag outranks every derived id.** The `--ids` map and the `tvshow.nfo` layer
fill an untagged folder and are **dropped** when they contradict its own
`[tvdbid-N]`, which the report prints as `DISAGREE`. Measured the same day: the
provider table carries a 2026 special's id on `Malcolm in the Middle (2000)
[tvdbid-73838]`; unioned, those folders are one show and the special's bigger
files dominate the original's S01E01–E04 — 3.2 GiB of the 2000 series bound for
the hold as "worse copies". The pass that staged 54 GiB ran with no `--ids`, so no
staged file rests on a borrowed id.
- **A folder-level verdict destroys content.** Of 25 structural candidates in the
2026-11-02 census, **10 were complementary halves**: *Game of Thrones: The
Complete Series* holds no S01 and no S08, `Modern Family S05` is the only copy of
season 5, *Loiter Squad Uncensored* the only copy of season 1. Judged per folder
the pass would have staged about 340 GB of unique episodes; judged per episode it
staged 54 GiB of true copies.
- **A file is droppable only when every episode it carries is held elsewhere in the
same show at >= its size** (Frank's rule, applied per episode). A byte-identical
pair ties and the tie breaks by path so exactly one copy stages; a file holding
one unique episode stays whatever else it carries.
- **A bare number is never an episode.** `256383.mp4` may be an episode id, a
poster or a whole season, so key-less names are never staged and are counted in
the report (4,036 after the 2026-11-02 pass, almost all genuine bonus material).
Where a packager named files by id, Jellyfin's own table is the witness: a
`path<TAB>season<TAB>episode` TSV from `BaseItems` where `Type like '%TV.Episode'`
and both index numbers are set, read `mode=ro&immutable=1`, its `/data` prefix
rewritten to the raw array path. Jellyfin 12 keeps `Path`, `ParentIndexNumber` and
`IndexNumber` on `BaseItems` and stores the full CLR type name — confirm with
`pragma table_info` before trusting the query, because an empty witness silently
falls back to name parsing.
Seven clips in the library are still named only by number, and where Jellyfin
was asked its own `Name` for them is the number too, so no database holds a
better title — that residue is naming, not a task.
- **A pack states a range and competes on all of it.** `S01E01-02` is droppable only
when both halves are held elsewhere, and it wins the survivor seat for both when
it is the largest carrier: the byte law compares files, and a name both servers
parse as a range stays indexed as two episodes.
- **A sidecar that shares the doomed copy's stem goes with it** — subtitles, `.nfo`,
artwork keyed on the stem — so a staged duplicate leaves no orphan beside the
survivor. The 2026-11-02 hold is 220 media files and nothing else: the dominated
copies were bare rips, and counting a hold's non-media files is the census that
tells you which case you are in.
- **The audit gates the plan file.** Every survivor is re-statted and every
destination re-checked before `--out` is written; on failure the report prints
`AUDIT FAILED` and writes no plan, so there is nothing to hand the executor.
- **Two folders that look duplicated and share zero episode keys are complementary,
not duplicated.** That is how the two pairs the census still refuses to merge were
settled without touching them: *Star Wars - The Clone Wars* (12 files, 1.3 GiB)
beside *Star Wars The Clone Wars* (121 files, 214 GiB), and the *Batman: The
Animated Series* leftovers (3 files) beside the tagged folder (85) — `shared
episode keys: 0` in both cases, so a tag would have freed nothing and cost a
rename with library consequences. Re-running the whole census with the id map over
all 753 top-level folders (606 already tagged) staged **0 files**: the library is
duplicate-free to the limit of provable identity, and what remains is naming.
## Guards ## Guards
Each of these is a file that would have been destroyed without it: Each of these is a file that would have been destroyed without it:
@@ -97,6 +180,14 @@ keeper is missing — either one drops the whole group rather than half-applying
- The 2026-11-01 pass held 226 files (~1.1 TiB) in - The 2026-11-01 pass held 226 files (~1.1 TiB) in
`/mnt/entropy/.AGENTTRASH/20261101-duplicate-prune`, journals `/mnt/entropy/.AGENTTRASH/20261101-duplicate-prune`, journals
`dup-prune-exact.jsonl` and `dup-prune-similar.jsonl`. Purge pending Frank's word. `dup-prune-exact.jsonl` and `dup-prune-similar.jsonl`. Purge pending Frank's word.
- The 2026-11-02 TV pass holds 220 files (54 GiB) in
`/mnt/entropy/.AGENTTRASH/20261102-tv-duplicates` — plan `tv-dupes-plan.tsv`,
journal `tv-dupes.jsonl`, 220 moves / 16 hold folders / 7 husks swept — and the
same day's TVShows detritus pass holds 109 files (552 KiB) in
`/mnt/entropy/.AGENTTRASH/20261102-tvshows-detritus` with plan
`tvshows-detritus-plan.tsv` and journal `tvshows-detritus.jsonl`. Both purges
pending Frank's word, beside the 2026-11-01 hold above. All four journals and
plans live at the `entropy` root, not `/tmp`.
- **Re-verify each survivor by size, not by the recorded path, before purging.** - **Re-verify each survivor by size, not by the recorded path, before purging.**
Detection artifacts and plans key on paths, and a folder renamed by an Detection artifacts and plans key on paths, and a folder renamed by an
interactive session ([[Entropy Volume]], *Write doors*) makes its keeper look interactive session ([[Entropy Volume]], *Write doors*) makes its keeper look
@@ -117,7 +208,9 @@ stop plex && systemctl --user mask plex`) so its metadata fetches cannot stack o
live Jellyfin scan, and because a masked unit stays stopped through a reboot — live Jellyfin scan, and because a masked unit stays stopped through a reboot —
`unmask` before starting it. The 2026-11-01 morning ran it once Jellyfin reached `unmask` before starting it. The 2026-11-01 morning ran it once Jellyfin reached
`Idle`: one accepted `refresh` on the Movies section cleared every row that named a `Idle`: one accepted `refresh` on the Movies section cleared every row that named a
held path. held path. The 2026-11-02 TV pass did the same on the TV Shows section (key 3): one
accepted refresh, then `media-verify.py` against that journal reads `landed: 0`,
`still at src: 0`, `REAL: 0` on **both** servers — the correct end state for a prune.
The same evening's detritus pass is a second, separate hold: 423 files (12 MiB) in The same evening's detritus pass is a second, separate hold: 423 files (12 MiB) in
`/mnt/entropy/.AGENTTRASH/20261101-detritus-sweep`, plan `detritus-sweep-plan.tsv`, `/mnt/entropy/.AGENTTRASH/20261101-detritus-sweep`, plan `detritus-sweep-plan.tsv`,
@@ -128,6 +221,10 @@ subtitle folders (428 of them, 6 GiB) and ripped Blu-ray streams — `.sup` PGS,
`.thd`/`.eac3`/`.dts` audio, `.cci` captions, 25 GiB — are not trash, and an `.thd`/`.eac3`/`.dts` audio, `.cci` captions, 25 GiB — are not trash, and an
extension the census does not recognise is a reason to stop, not to sweep. extension the census does not recognise is a reason to stop, not to sweep.
A folder holding a `.plexmatch` is held back even when its subtree looks empty: that
file is a manual match — a human decision about what the folder is — and detritus
rules do not overrule one.
`scripts/media-verify.py` reads differently for a prune than for a relayout: `scripts/media-verify.py` reads differently for a prune than for a relayout:
`landed: 0` is the **correct** end state (no server may index the hold), so the `landed: 0` is the **correct** end state (no server may index the hold), so the
verdict is `still at src: 0` with `REAL: 0`, and the rows that simply vanished verdict is `still at src: 0` with `REAL: 0`, and the rows that simply vanished
@@ -3,7 +3,7 @@ type: reference
title: Media Metadata View title: Media Metadata View
description: "The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback." description: "The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback."
tags: [dankserver, media-server, jellyfin, plex, storage] tags: [dankserver, media-server, jellyfin, plex, storage]
timestamp: 2026-11-01T01:00:00Z timestamp: 2026-11-02T01:00:00Z
--- ---
# Media Metadata View # Media Metadata View
@@ -134,6 +134,19 @@ the defaults are the host values above.
upper layer*, not for changes on the array underneath. After any relayout or upper layer*, not for changes on the array underneath. After any relayout or
externally-written metadata, trigger a library refresh — see externally-written metadata, trigger a library refresh — see
[[Media Relayout]] for the refresh procedure. [[Media Relayout]] for the refresh procedure.
- **`stat` through the view can outlive the file.** Measured 2026-11-02 while a TV
dedupe pass moved folders on the raw path: the merged tree still answered `stat`
at the old coordinates — link count `0`, and `ls` of the same parent no longer
listed the name. Overlayfs keeps a dentry for a lower file that was moved
underneath it, so a script that tests existence with `[ -e ]` against the view can
believe a file its own `readdir` denies. Prove existence by listing the parent.
The media servers walk by `readdir`, so ghosts never reach the library — this bites
a tool, never a user, which is why no server symptom accompanies it.
- **Tools that move or measure media walk the raw array, never the view** — every
relayout, prune and detritus pass, and every journal it writes. The view is for
Jellyfin's writes and Plex's reads only. One deliberate exception: `tvshow.nfo`
sidecars live in the upper layer, so a tool reading show identity points at the
view (`media-tvdupes.py --meta /var/lib/media-meta/view/TVShows`).
## Failure signatures ## Failure signatures
@@ -148,6 +161,7 @@ the defaults are the host values above.
| A media server's scan record reads `"Status": "Aborted"` | Its container restarted (a cutover, an image bump, an OOM) and the walk was discarded; systemd's `Restart=always` restarts the *server*, never the scan. Trigger a refresh — see [[Media Relayout]]. | | A media server's scan record reads `"Status": "Aborted"` | Its container restarted (a cutover, an image bump, an OOM) and the walk was discarded; systemd's `Restart=always` restarts the *server*, never the scan. Trigger a refresh — see [[Media Relayout]]. |
| The Jellyfin API answers 401 to every token header | Almost always the header **form**, not the key: 12.4 authenticates only `Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token` for every key, valid or not — and one collapsed backslash in the curl config produces the same 401. `[[Jellyfin]]` owns the block that works. Before concluding anything, prove the key against the server's own table: `select count(*) from ApiKeys where AccessToken = ?` on `jellyfin.db` opened `mode=ro&immutable=1`. | | The Jellyfin API answers 401 to every token header | Almost always the header **form**, not the key: 12.4 authenticates only `Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token` for every key, valid or not — and one collapsed backslash in the curl config produces the same 401. `[[Jellyfin]]` owns the block that works. Before concluding anything, prove the key against the server's own table: `select count(*) from ApiKeys where AccessToken = ?` on `jellyfin.db` opened `mode=ro&immutable=1`. |
| `systemctl status` finds no such unit | Look under the escaped name, `var-lib-media\x2dmeta-view.mount`, not the repo file name. | | `systemctl status` finds no such unit | Look under the escaped name, `var-lib-media\x2dmeta-view.mount`, not the repo file name. |
| Stale dentry through the view: `stat` succeeds with link count `0` while the parent's `ls` omits the name | The lower file was moved on the raw path after the view cached its dentry. Nothing is lost; existence through the view is proved by the parent listing. |
## Rollback ## Rollback
@@ -138,7 +138,7 @@ HTTPD close without a single byte; and the same HTTPD drops requests that carry
no `Accept` and client-identifier header (see the scan block above for the form no `Accept` and client-identifier header (see the scan block above for the form
that works). that works).
A duplicate prune is the same census and the same proof, with its own keep rules: [[Media Duplicate Prune]]. A duplicate prune is the same census and the same proof, with its own keep rules — films by title and duration, a series per episode: [[Media Duplicate Prune]].
## TV in the movie library ## TV in the movie library
+1 -1
View File
@@ -52,7 +52,7 @@ is a locator: one row, one clause, no numbers.
| [[Jellyfin]] | Jellyfin on Dankserver: quadlet unit, overlay metadata view, provider keys, ports, access, transcode status. | | [[Jellyfin]] | Jellyfin on Dankserver: quadlet unit, overlay metadata view, provider keys, ports, access, transcode status. |
| [[mariadb]] | The MariaDB backend for RomM: quadlet unit, network alias, env-file secret handling, and the backup path. | | [[mariadb]] | The MariaDB backend for RomM: quadlet unit, network alias, env-file secret handling, and the backup path. |
| [[Media Audiobook Relayout]] | Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals. | | [[Media Audiobook Relayout]] | Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals. |
| [[Media Duplicate Prune]] | Find and hold duplicate films on the entropy array: the hash ladder, the keep rules, the guards, the purge. | | [[Media Duplicate Prune]] | Find and hold duplicate films and series on the array: the screen, per-episode rule, guards, purge. |
| [[Media Metadata View]] | The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback. | | [[Media Metadata View]] | The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback. |
| [[Media Music Relayout]] | Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals. | | [[Media Music Relayout]] | Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals. |
| [[Media Relayout]] | Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback. | | [[Media Relayout]] | Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback. |
+92
View File
@@ -24,6 +24,9 @@
# 9. no plugin configuration XML is tracked in git, the deployer places the # 9. no plugin configuration XML is tracked in git, the deployer places the
# keys BEFORE it restarts Jellyfin, and the planner routes a change to this # keys BEFORE it restarts Jellyfin, and the planner routes a change to this
# installer to jellyfin. # installer to jellyfin.
# 10. verify refuses a library that is a SILENT FILE INDEX — internet providers
# switched off, or a type whose ImageFetchers is empty while its
# MetadataFetchers list is populated (the 2026-11-01 missing-posters fault).
load helpers load helpers
@@ -59,6 +62,32 @@ setup() {
TRASH="$BATS_TEST_TMPDIR/trash" TRASH="$BATS_TEST_TMPDIR/trash"
SEC="$BATS_TEST_TMPDIR/secrets" SEC="$BATS_TEST_TMPDIR/secrets"
mkdir -p "$JF/plugins/configurations" "$TRASH" "$SEC" mkdir -p "$JF/plugins/configurations" "$TRASH" "$SEC"
# The per-library option file Jellyfin 12 owns (it was library.xml). Conforming
# by default so the pre-existing verify tests keep testing what they tested.
mkdir -p "$JF/root/default/Movies"
cat >"$JF/root/default/Movies/options.xml" <<XML
<?xml version="1.0" encoding="utf-8"?>
<LibraryOptions xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<EnableInternetProviders>true</EnableInternetProviders>
<MetadataSavers>
<string>Nfo</string>
</MetadataSavers>
<TypeOptions>
<TypeOptions>
<Type>Movie</Type>
<MetadataFetchers>
<string>TheMovieDb</string>
</MetadataFetchers>
<ImageFetchers>
<string>TheMovieDb</string>
</ImageFetchers>
<ImageFetcherOrder>
<string>TheMovieDb</string>
</ImageFetcherOrder>
</TypeOptions>
</TypeOptions>
</LibraryOptions>
XML
cat >"$JF/plugins/configurations/Jellyfin.Plugin.Tmdb.xml" <<XML cat >"$JF/plugins/configurations/Jellyfin.Plugin.Tmdb.xml" <<XML
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema"> <Configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
@@ -270,6 +299,69 @@ XML
printf '%s\n' "${lines[@]}" | grep -qE 'TmdbApiKey .*[0-9]+ chars' printf '%s\n' "${lines[@]}" | grep -qE 'TmdbApiKey .*[0-9]+ chars'
} }
@test "verify refuses a library whose internet providers are switched off" {
# The whole library becomes a file index: refreshes answer 204 and store
# nothing, and the item count still looks like a healthy scan.
prov install >/dev/null
sed -i 's#<EnableInternetProviders>true</EnableInternetProviders>#<EnableInternetProviders>false</EnableInternetProviders>#' \
"$JF/root/default/Movies/options.xml"
run prov verify
[ "$status" -ne 0 ]
printf '%s\n' "${lines[@]}" | grep -qiF 'SILENT INDEX Movies'
printf '%s\n' "${lines[@]}" | grep -qiF 'EnableInternetProviders'
}
@test "verify refuses a type with no image fetchers while metadata fetchers are named" {
prov install >/dev/null
cat >"$JF/root/default/Movies/options.xml" <<XML
<?xml version="1.0" encoding="utf-8"?>
<LibraryOptions>
<EnableInternetProviders>true</EnableInternetProviders>
<TypeOptions>
<TypeOptions>
<Type>Movie</Type>
<MetadataFetchers>
<string>TheMovieDb</string>
</MetadataFetchers>
<ImageFetchers />
<ImageFetcherOrder>
<string>TheMovieDb</string>
</ImageFetcherOrder>
</TypeOptions>
</TypeOptions>
</LibraryOptions>
XML
run prov verify
[ "$status" -ne 0 ]
printf '%s\n' "${lines[@]}" | grep -qiF 'ImageFetchers empty'
}
@test "an older options.xml that never carried the switch is not refused" {
# BoxSets on this host were written before the property existed; absence is
# the vendor default, not a switched-off library, so it must not fail.
prov install >/dev/null
mkdir -p "$JF/root/default/Collections"
cat >"$JF/root/default/Collections/options.xml" <<XML
<?xml version="1.0" encoding="utf-8"?>
<LibraryOptions>
<TypeOptions>
<TypeOptions>
<Type>BoxSet</Type>
<MetadataFetchers>
<string>TheMovieDb</string>
</MetadataFetchers>
<ImageFetchers>
<string>TheMovieDb</string>
</ImageFetchers>
</TypeOptions>
</TypeOptions>
</LibraryOptions>
XML
run prov verify
[ "$status" -eq 0 ]
printf '%s\n' "${lines[@]}" | grep -qiF '= Collections'
}
@test "no plugin configuration XML is tracked in this repo" { @test "no plugin configuration XML is tracked in this repo" {
# These files hold provider keys and are owned by the running container; # These files hold provider keys and are owned by the running container;
# tracking them would put credentials in git and guarantee drift. # tracking them would put credentials in git and guarantee drift.
+215
View File
@@ -0,0 +1,215 @@
#!/usr/bin/env bats
# Seam: scripts/media-tvdupes.py — the episode-level duplicate census for the TV
# library.
#
# The failure this seam exists to prevent is the one that almost cost real
# content: two folders of one show look like copies, and a folder-level "keep
# the bigger folder" deletes the half of the series only the smaller folder
# holds (the 2026-11-02 census found 10 such complementary pairs). So every
# assertion here is about WHICH files land in the plan and WHICH the tool
# refuses to touch — never about its internals. Everything runs on a synthetic
# library in the test's own tmpdir: no media server, no network, no array.
load helpers
TOOL="$REPO_ROOT/scripts/media-tvdupes.py"
# at PATH SIZE — a file of deterministic bytes.
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
# sources FILE — the sources the plan moves, one per line.
sources() { awk -F'\t' '$1=="move"{print $2}' "$1"; }
# destinations FILE — where the plan puts them.
destinations() { awk -F'\t' '$1=="move"{print $3}' "$1"; }
# plan_moves FILE — how many files the plan stages.
plan_moves() {
[ -f "$1" ] || { echo 0; return; }
awk -F'\t' '$1=="move"' "$1" | wc -l
}
setup() {
LIB="$BATS_TEST_TMPDIR/TVShows"
HOLD="$BATS_TEST_TMPDIR/.AGENTTRASH/20991231-tv-duplicates"
PLAN="$BATS_TEST_TMPDIR/plan.tsv"
mkdir -p "$LIB" "$HOLD"
}
@test "the smaller copy of an episode is staged and the larger stays put" {
at "$LIB/Doc Who (2005) [tvdbid-78804]/Season 01/Doc Who - S01E01 - Pilot.mkv" 4000
at "$LIB/Doc Who (2005) [tvdbid-78804]/Season 01/backup/Doc.Who.S01E01.720p.mkv" 1200
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$status" -eq 0 ]
[ "$(plan_moves "$PLAN")" -eq 1 ]
run sources "$PLAN"
[[ "$output" == *"backup/Doc.Who.S01E01.720p.mkv"* ]]
[[ "$output" != *"Pilot.mkv"* ]]
run destinations "$PLAN"
[[ "$output" == "$HOLD/"* ]]
}
@test "an equal-size pair stages exactly one copy" {
at "$LIB/Equal (2001) [tvdbid-11]/Season 01/Equal - S01E01.mkv" 2500
at "$LIB/Equal (2001) [tvdbid-11]/Rip/Equal - S01E01.mkv" 2500
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$status" -eq 0 ]
[ "$(plan_moves "$PLAN")" -eq 1 ]
}
@test "a file holding one episode nothing else has is never staged" {
# The shape that defeats a folder-level verdict: Season 02 exists only here,
# so the folder that looks like a leftover is the only copy of an episode.
at "$LIB/King (2010) [tvdbid-21]/Season 01/King - S01E01 - One.mkv" 3000
at "$LIB/King (2010) [tvdbid-21]/Season 01/backup/King - S01E01 - One.mkv" 900
at "$LIB/King (2010) [tvdbid-21]/Season 02/King - S02E01 - Two.mkv" 800
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$status" -eq 0 ]
run sources "$PLAN"
[[ "$output" != *"S02E01"* ]]
[[ "$output" == *"backup/King - S01E01 - One.mkv"* ]]
}
@test "a two-in-one pack competes for every episode in its range" {
at "$LIB/Range (2012) [tvdbid-31]/Season 01/Range - S01E01.mkv" 1500
at "$LIB/Range (2012) [tvdbid-31]/Season 01/Range - S01E02.mkv" 1500
at "$LIB/Range (2012) [tvdbid-31]/Packs/Range - S01E01-02.mkv" 2400
# The pack is the biggest carrier of BOTH episodes, so the two rips go.
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$(plan_moves "$PLAN")" -eq 2 ]
run sources "$PLAN"
[[ "$output" == *"Season 01/Range - S01E01.mkv"* ]]
[[ "$output" != *"Packs/Range - S01E01-02.mkv"* ]]
# Make the season file the bigger carrier of E02 and the pack becomes the
# only carrier of E01: it stays, however small it is.
rm "$LIB/Range (2012) [tvdbid-31]/Season 01/Range - S01E01.mkv"
rm "$LIB/Range (2012) [tvdbid-31]/Season 01/Range - S01E02.mkv"
at "$LIB/Range (2012) [tvdbid-31]/Season 01/Range - S01E02.mkv" 9000
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$(plan_moves "$PLAN")" -eq 0 ]
}
@test "two tvdb ids on the same words are never merged" {
at "$LIB/Avatar The Last Airbender (2024) [tvdbid-385925]/Season 01/Avatar - S01E01.mkv" 9000
at "$LIB/Avatar The Last Airbender (1999) [tvdbid-74852]/Season 01/Avatar - S01E01.mkv" 500
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$status" -eq 0 ]
[ "$(plan_moves "$PLAN")" -eq 0 ]
[[ "$output" == *CONFLICT* ]]
[[ "$output" == *"tvdb [74852] vs [385925]"* ]]
}
@test "a derived id that contradicts the folder's own tag never merges them" {
# Measured 2026-11-02: Jellyfin's provider table carried a 2026 special's tvdb
# id on the folder tagged [tvdbid-73838]. Unioning the two makes one show of a
# 2000 series and a 2026 special, and the special's bigger files dominate the
# original's pilot. The tag is the operator's word; the nfo and `--ids` layers
# fill an untagged folder and never overrule a tagged one.
at "$LIB/Middle (2000) [tvdbid-73838]/Season 01/Middle - S01E01 - Pilot.mkv" 800
at "$LIB/Middle (2026) [tvdbid-457779]/Season 01/Middle - S01E01 - Episode 1.mkv" 1100
IDS="$BATS_TEST_TMPDIR/ids.tsv"
printf '%s\t%s\n' "Middle (2000) [tvdbid-73838]" 457779 > "$IDS"
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" --ids "$IDS"
[ "$status" -eq 0 ]
[ "$(plan_moves "$PLAN")" -eq 0 ]
[[ "$output" == *"DISAGREE"* ]]
[[ "$output" == *"tag tvdb [73838]"* ]]
[[ "$output" == *"derived tvdb [457779]"* ]]
}
@test "an untagged drop is identified by the tvshow.nfo in the metadata view" {
at "$LIB/Clone Wars/Season 01/Clone Wars - S01E01.mkv" 1200
at "$LIB/Clone Wars (2008) [tvdbid-84859]/Season 01/Clone Wars - S01E01.mkv" 4000
VIEW="$BATS_TEST_TMPDIR/view"
mkdir -p "$VIEW/Clone Wars"
printf '%s\n' '<tvshow><title>Clone Wars</title>' \
'<uniqueid type="tvdb" default="true">84859</uniqueid></tvshow>' \
> "$VIEW/Clone Wars/tvshow.nfo"
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" --meta "$VIEW"
[ "$status" -eq 0 ]
[ "$(plan_moves "$PLAN")" -eq 1 ]
run sources "$PLAN"
[[ "$output" == *"Clone Wars/Season 01/Clone Wars - S01E01.mkv"* ]]
# Without the nfo layer nothing identifies the drop, so nothing moves.
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" \
--out "$BATS_TEST_TMPDIR/no-meta.tsv"
[ "$status" -eq 0 ]
[[ "$output" == *UNRESOLVED* ]]
[ "$(plan_moves "$BATS_TEST_TMPDIR/no-meta.tsv")" -eq 0 ]
}
@test "a file named by an id is keyed only when the witness names it" {
BIG="$LIB/Edd (1999) [tvdbid-75897]/Season 01/Edd - S01E01 - Down the Line.mkv"
SMALL="$LIB/Edd (1999) [tvdbid-75897]/Season 01/104348.mkv"
at "$BIG" 2100
at "$SMALL" 700
WIT="$BATS_TEST_TMPDIR/witness.tsv"
printf '%s\t1\t1\n' "$SMALL" > "$WIT"
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$(plan_moves "$PLAN")" -eq 0 ]
[[ "$output" == *"unkeyed, never staged: 1 files"* ]]
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" --witness "$WIT"
[ "$(plan_moves "$PLAN")" -eq 1 ]
run sources "$PLAN"
[[ "$output" == *104348.mkv* ]]
}
@test "the sidecars of a staged episode are staged with it" {
at "$LIB/Side (2003) [tvdbid-41]/Season 01/Side - S01E01 - First.mkv" 3000
at "$LIB/Side (2003) [tvdbid-41]/Rip/Side - S01E01 - First.mkv" 1000
at "$LIB/Side (2003) [tvdbid-41]/Rip/Side - S01E01 - First.srt" 400
at "$LIB/Side (2003) [tvdbid-41]/Season 01/Side - S01E01 - First.srt" 450
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$(plan_moves "$PLAN")" -eq 2 ]
run sources "$PLAN"
[[ "$output" == *"Rip/Side - S01E01 - First.srt"* ]]
[[ "$output" != *"Season 01/Side - S01E01 - First.srt"* ]]
}
@test "a hold inside the library is refused" {
at "$LIB/Hostage (2007) [tvdbid-51]/Season 01/Hostage - S01E01.mkv" 100
run python3 "$TOOL" --library "$LIB" --hold "$LIB/.AGENTTRASH/hold" \
--out "$PLAN"
[ "$status" -ne 0 ]
[[ "$output" == *"--hold"* ]]
}
@test "a plan whose hold target is occupied is refused, plan and all" {
at "$LIB/Gone (2009) [tvdbid-61]/Season 01/Gone - S01E01.mkv" 1000
at "$LIB/Gone (2009) [tvdbid-61]/Rip/Gone - S01E01.mkv" 400
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$(plan_moves "$PLAN")" -eq 1 ]
# The hold already holds one of the destinations: the executor would refuse
# it, so a plan that cannot run must not be handed over at all.
mkdir -p "$HOLD/Gone (2009) [tvdbid-61]/Rip"
at "$HOLD/Gone (2009) [tvdbid-61]/Rip/Gone - S01E01.mkv" 400
rm "$PLAN"
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$status" -eq 1 ]
[[ "$output" == *"hold target occupied"* ]]
[ ! -e "$PLAN" ]
}
@test "a library with nothing duplicated writes an empty plan" {
at "$LIB/Unique (2015) [tvdbid-71]/Season 01/Unique - S01E01.mkv" 1000
at "$LIB/Other Show (2016) [tvdbid-72]/Season 01/Other Show - S01E01.mkv" 1000
run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN"
[ "$status" -eq 0 ]
[ "$(plan_moves "$PLAN")" -eq 0 ]
[[ "$output" == *"plan: 0 moves"* ]]
}