From 52600b28ab9f6eb82c622c9c0c9793478be60d6b Mon Sep 17 00:00:00 2001 From: Frank Date: Sun, 1 Nov 2026 10:31:42 -0800 Subject: [PATCH 1/4] fix(jellyfin): refuse a library that indexes files but fetches nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Movies library had indexed 2,571 films and had never asked a provider for a single one: no TMDb id, no plot, no rating, no poster. Nothing was broken in the provider chain — the plugin is core in 12.4 and active, its key is set, image.tmdb.org answers the container — and nothing was broken in the write path: the overlay view has been read-write since the 2026-10-31 cutover, and movie.nfo lands on the NVMe upper layer today. The state was two settings in the per-library option file, config/root/default//options.xml (Jellyfin 12 renamed it from library.xml; the database stores none of these options, so a census that greps for the old name and then queries the DB concludes nothing is configured and goes hunting elsewhere): false (empty, beside a populated and a fully populated ) With the internet switch off every refresh answers 204, stores nothing and logs nothing, so the item count reads like a healthy scan — this passed the deploy health gate, the Playwright suite and three agent audits. Order is not enablement: the fetcher ORDER stays populated while the ENABLED set sits empty, and the populated metadata list beside it makes the library look well. verify now audits every library's options.xml and fails with `SILENT INDEX : ` on providers-off or on a type whose image fetchers are empty while its metadata fetchers are named; an options.xml that never carried the switch (BoxSets, written before the property existed) is the vendor default and stays accepted. Vault: the page claimed artwork is a sidecar like movie.nfo. It is not — no image saver is bound for any library, so art goes to the internal store and Plex never reads it. The claim is corrected, and the RemoteImages diagnostic that separates a provider fault from a library fault is written down with the 12.x API shape traps (Items needs UserId; MetadataRefreshMode has no Full/ValidationFailure, ReplaceAllMetadata=true is what re-runs providers). --- scripts/jellyfin-providers.sh | 62 ++++++++++++- serverAppConfigs-docs/Dankserver/Jellyfin.md | 38 +++++++- tests/jellyfin_providers.bats | 92 ++++++++++++++++++++ 3 files changed, 187 insertions(+), 5 deletions(-) diff --git a/scripts/jellyfin-providers.sh b/scripts/jellyfin-providers.sh index cc95ec2..c982b65 100755 --- a/scripts/jellyfin-providers.sh +++ b/scripts/jellyfin-providers.sh @@ -35,7 +35,8 @@ # Subcommands # plan print the resolved shape and every pending change, touch nothing # install fetch+verify the plugin, then write the keys (backing up first) -# verify prove the plugin and both keys are actually in place +# verify prove the plugin and both keys are in place, and that no library +# is a silent file index (providers off / image fetchers empty) # # Nothing here restarts Jellyfin: the deployer owns that # (`scripts/deploy.sh dankserver --service jellyfin`), and the plugin is read at @@ -340,6 +341,64 @@ cmd_install() { fi } + +# A library whose options.xml carries EnableInternetProviders=false, or an empty +# ImageFetchers list beside a populated MetadataFetchers list, is a SILENT FILE +# INDEX: it indexes every file, answers 204 to every refresh, stores no provider +# id, plot, rating or artwork, and logs nothing — so the item count reads like a +# healthy scan. Jellyfin 12 keeps these options in root/default//options.xml +# (it was library.xml) and stores NONE of them in the database, so this audit is +# the only place the state can be read without a browser session. +verify_library_options() { + local root="$JF_ROOT/root/default" f lib problems + if [ ! -d "$root" ]; then + printf '= no library options under %s (nothing audited)\n' "$root" + return 0 + fi + local rc=0 + for f in "$root"/*/options.xml; do + [ -f "$f" ] || continue + lib="$(basename "$(dirname "$f")")" + problems="$(python3 -s - "$f" <<'PY' +import sys, xml.dom.minidom as X +root = X.parse(sys.argv[1]).documentElement +def child(el, name): + for n in el.childNodes: + if n.nodeType == n.ELEMENT_NODE and n.tagName == name: + return n +def flag(el, name): + n = child(el, name) + return None if n is None else (n.firstChild.data.strip() if n.firstChild else "") +def names(el, name): + n = child(el, name) + return None if n is None else [s.firstChild.data.strip() for s in n.childNodes + if s.nodeType == s.ELEMENT_NODE and s.firstChild] +bad = [] +ip = flag(root, "EnableInternetProviders") +if ip is not None and ip.lower() != "true": + bad.append("EnableInternetProviders=%s" % (ip or "empty")) +types = child(root, "TypeOptions") +if types is not None: + for t in types.childNodes: + if t.nodeType != t.ELEMENT_NODE or t.tagName != "TypeOptions": + continue + meta = names(t, "MetadataFetchers") or [] + if meta and not (names(t, "ImageFetchers") or []): + bad.append("%s: ImageFetchers empty while MetadataFetchers names %d" + % (flag(t, "Type") or "?", len(meta))) +print("; ".join(bad)) +PY +)" + if [ -n "$problems" ]; then + printf 'SILENT INDEX %s: %s\n' "$lib" "$problems" >&2 + rc=1 + else + printf '= %s: library options fetch from providers\n' "$lib" + fi + done + return "$rc" +} + cmd_verify() { check_root local bad=0 @@ -378,6 +437,7 @@ cmd_verify() { printf '%s %s chars\n' "$elem" "${#n}" fi done + verify_library_options || bad=1 [ "$bad" -eq 0 ] || return 1 printf 'providers configured\n' } diff --git a/serverAppConfigs-docs/Dankserver/Jellyfin.md b/serverAppConfigs-docs/Dankserver/Jellyfin.md index 395545a..ccdc333 100644 --- a/serverAppConfigs-docs/Dankserver/Jellyfin.md +++ b/serverAppConfigs-docs/Dankserver/Jellyfin.md @@ -46,10 +46,14 @@ sees is the [[Media Metadata View]] overlay, live since the 2026-10-31 cutover: `/var/lib/media-meta/view:/data`, read-write. `Local metadata` — save metadata to media folders — is enabled, and it is the -reason the view exists: `movie.nfo`, `tvshow.nfo`, `episode.nfo` and artwork -written once on the NVMe upper layer are files [[Plex]] can read too — and since -the 2026-10-31 cutover that is what happens: Jellyfin is the fleet's only -metadata writer, and every sidecar it saves is an SSD byte the array never sees. +reason the view exists: `movie.nfo`, `tvshow.nfo` and `episode.nfo` written once +on the NVMe upper layer are files [[Plex]] can read too — and since the +2026-10-31 cutover that is what happens: Jellyfin is the fleet's only metadata +writer, and every sidecar it saves is an SSD byte the array never sees. +**Artwork is not one of those sidecars.** No image saver is bound for any +library, so posters and fanart go to the internal store +(`config/metadata/library///poster.jpg`), which [[Plex]] never +reads — movie art has never existed beside the media on either layer. **Library coverage is the first thing to check when a title is missing.** The four libraries are `movies → /data/Movies`, `tvshows → /data/TVShows`, @@ -66,6 +70,32 @@ instead of the view — always a fault on this host since the 2026-10-31 cutover `scripts/media-view.sh status` says which: the quadlet drifted back to the raw array, or the mount died. +**`EnableInternetProviders` false turns a library into a silent file index.** +Every refresh answers `204`, stores nothing and logs nothing — no provider ids, +no plot, no rating, no art — while the item count reads like a healthy scan. The +switch lives in `config/root/default//options.xml`, Jellyfin 12's name +for the per-library option file: grepping for the old `library.xml` concludes "no +per-library options exist", and the database holds none, so the trail dies there. +Two more traps in the same file: `TypeOptions/ImageFetchers` is the *enabled* set, +so an empty `` switches art off while `ImageFetcherOrder` stays +fully populated — order is not enablement — and the populated `` +built next to it makes the library look healthy. + +**`RemoteImages` is the call that separates a provider fault from a library +fault.** `GET /Items/{id}/RemoteImages?Type=Primary` returns the provider's real +poster list even with every automatic pass switched off — plugin, key and CDN all +proven by that one response — so a refresh that stores nothing beside it is a +library-switch problem, never a provider problem. It is also the delivery door: +`POST /Items/{id}/RemoteImages/Download` saves the chosen image to the internal +store on a build where the automatic image pass will not run at all. + +**API shape on 12.x:** `/Items` and `/Items/{id}` need `UserId` when +authenticated by API key — without it the answer is `400 Error processing +request`, which reads exactly like an authentication failure. `MetadataRefreshMode` +accepts `Default` (and `Realtime`) only; `Full` and `ValidationFailure` are +rejected outright, and what forces the providers to re-run is +`ReplaceAllMetadata=true`. + ## Ports & access | Port | Proto | Purpose | diff --git a/tests/jellyfin_providers.bats b/tests/jellyfin_providers.bats index 50aab63..c9e71d0 100755 --- a/tests/jellyfin_providers.bats +++ b/tests/jellyfin_providers.bats @@ -24,6 +24,9 @@ # 9. no plugin configuration XML is tracked in git, the deployer places the # keys BEFORE it restarts Jellyfin, and the planner routes a change to this # installer to jellyfin. +# 10. verify refuses a library that is a SILENT FILE INDEX — internet providers +# switched off, or a type whose ImageFetchers is empty while its +# MetadataFetchers list is populated (the 2026-11-01 missing-posters fault). load helpers @@ -59,6 +62,32 @@ setup() { TRASH="$BATS_TEST_TMPDIR/trash" SEC="$BATS_TEST_TMPDIR/secrets" mkdir -p "$JF/plugins/configurations" "$TRASH" "$SEC" + # The per-library option file Jellyfin 12 owns (it was library.xml). Conforming + # by default so the pre-existing verify tests keep testing what they tested. + mkdir -p "$JF/root/default/Movies" + cat >"$JF/root/default/Movies/options.xml" < + + true + + Nfo + + + + Movie + + TheMovieDb + + + TheMovieDb + + + TheMovieDb + + + + +XML cat >"$JF/plugins/configurations/Jellyfin.Plugin.Tmdb.xml" < @@ -270,6 +299,69 @@ XML printf '%s\n' "${lines[@]}" | grep -qE 'TmdbApiKey .*[0-9]+ chars' } +@test "verify refuses a library whose internet providers are switched off" { + # The whole library becomes a file index: refreshes answer 204 and store + # nothing, and the item count still looks like a healthy scan. + prov install >/dev/null + sed -i 's#true#false#' \ + "$JF/root/default/Movies/options.xml" + run prov verify + [ "$status" -ne 0 ] + printf '%s\n' "${lines[@]}" | grep -qiF 'SILENT INDEX Movies' + printf '%s\n' "${lines[@]}" | grep -qiF 'EnableInternetProviders' +} + +@test "verify refuses a type with no image fetchers while metadata fetchers are named" { + prov install >/dev/null + cat >"$JF/root/default/Movies/options.xml" < + + true + + + Movie + + TheMovieDb + + + + TheMovieDb + + + + +XML + run prov verify + [ "$status" -ne 0 ] + printf '%s\n' "${lines[@]}" | grep -qiF 'ImageFetchers empty' +} + +@test "an older options.xml that never carried the switch is not refused" { + # BoxSets on this host were written before the property existed; absence is + # the vendor default, not a switched-off library, so it must not fail. + prov install >/dev/null + mkdir -p "$JF/root/default/Collections" + cat >"$JF/root/default/Collections/options.xml" < + + + + BoxSet + + TheMovieDb + + + TheMovieDb + + + + +XML + run prov verify + [ "$status" -eq 0 ] + printf '%s\n' "${lines[@]}" | grep -qiF '= Collections' +} + @test "no plugin configuration XML is tracked in this repo" { # These files hold provider keys and are owned by the running container; # tracking them would put credentials in git and guarantee drift. From a25f13928134a0b13b0a978e9305ec569c54ebd6 Mon Sep 17 00:00:00 2001 From: Frank Date: Sun, 1 Nov 2026 11:39:11 -0800 Subject: [PATCH 2/4] docs(jellyfin): the art delivery result, and what stays blank on purpose --- serverAppConfigs-docs/Dankserver/Jellyfin.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/serverAppConfigs-docs/Dankserver/Jellyfin.md b/serverAppConfigs-docs/Dankserver/Jellyfin.md index ccdc333..e02fbe3 100644 --- a/serverAppConfigs-docs/Dankserver/Jellyfin.md +++ b/serverAppConfigs-docs/Dankserver/Jellyfin.md @@ -87,7 +87,9 @@ poster list even with every automatic pass switched off — plugin, key and CDN proven by that one response — so a refresh that stores nothing beside it is a library-switch problem, never a provider problem. It is also the delivery door: `POST /Items/{id}/RemoteImages/Download` saves the chosen image to the internal -store on a build where the automatic image pass will not run at all. +store on a build where the automatic image pass will not run at all — it is how +the Movies library got its art: 2,515 of 2,564 films carry a primary now, and the +rest have none in TMDb (shorts like `Astartes`, files whose names match nothing). **API shape on 12.x:** `/Items` and `/Items/{id}` need `UserId` when authenticated by API key — without it the answer is `400 Error processing From 9f9043c03763b609d7286225dd13f176f730b1e5 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 2 Nov 2026 00:44:46 -0800 Subject: [PATCH 3/4] feat(media): measure a series per episode, and let the tag outrank a borrowed id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit media-tvdupes.py censuses a TV library by episode key and never by folder. 10 of the 25 shows that looked duplicated in the 2026-11-02 census held complementary halves (Game of Thrones Complete carries no S01 or S08, Modern Family S05 is the only copy of season 5), so a folder-level verdict would have staged some 340 GB of unique episodes to recover 54 GiB of true copies. Show identity comes from the folder's [tvdbid-N] tag, from the tvshow.nfo the metadata view exposes, or from an operator --ids map. A derived id that contradicts a tag is dropped and reported, never unioned: Jellyfin's provider table carries a 2026 special's id on the folder tagged [tvdbid-73838], and unioning the two makes one show of a 2000 series and a 2026 special whose bigger files then dominate the original's pilot. Files named by an id key only when the server's own episode table (the --witness) names them, so a bare number is never mistaken for an episode. Seam: tests/media_tvdupes.bats — 12 assertions on which files land in the plan and which the tool refuses to touch, on a synthetic library. --- scripts/media-tvdupes.py | 473 ++++++++++++++++++ .../Dankserver/Media Duplicate Prune.md | 103 +++- .../Dankserver/Media Relayout.md | 2 +- serverAppConfigs-docs/index.md | 2 +- tests/media_tvdupes.bats | 215 ++++++++ 5 files changed, 790 insertions(+), 5 deletions(-) create mode 100755 scripts/media-tvdupes.py create mode 100755 tests/media_tvdupes.bats diff --git a/scripts/media-tvdupes.py b/scripts/media-tvdupes.py new file mode 100755 index 0000000..898ab80 --- /dev/null +++ b/scripts/media-tvdupes.py @@ -0,0 +1,473 @@ +#!/usr/bin/env python3 +"""media-tvdupes.py — find duplicate TV episodes and plan their staging, one +EPISODE at a time. + +A TV library grows the same show into several folders: a tagged season box, an +untagged packager drop, a `Complete` folder that is not complete. The folders +look like copies of each other and are not, so the unit of comparison is the +episode and never the folder. A file is droppable only when EVERY episode it +carries is also held, somewhere else in the same show, by a file at least as +large — Frank's rule: the same episode in two places keeps the larger one. A +file that holds one episode nothing else has stays, whatever else it carries. + +It writes a plan and moves nothing; `scripts/media-apply.py` is the only thing +that renames. The plan stages into a dated hold, so the pass is undoable from +the journal and frees no space until the hold is purged. + + python3 scripts/media-tvdupes.py --library /mnt/entropy/TVShows \ + --hold /mnt/entropy/.AGENTTRASH/20261102-tv-duplicates \ + --out /mnt/entropy/tv-dupes-plan.tsv + python3 scripts/media-apply.py apply --plan /mnt/entropy/tv-dupes-plan.tsv \ + --root /mnt/entropy --journal /mnt/entropy/tv-dupes.jsonl \ + --metadata-view /var/lib/media-meta/view --apply + python3 scripts/media-apply.py verify --plan /mnt/entropy/tv-dupes-plan.tsv \ + --root /mnt/entropy --journal /mnt/entropy/tv-dupes.jsonl + python3 scripts/media-apply.py sweep --root /mnt/entropy \ + --journal /mnt/entropy/tv-dupes.jsonl --apply # empty folders only + scripts/media-view.sh reap # ghost sidecars above + +Then both servers have to be told: the plan moves files on the raw array, and no +watcher crosses the metadata view — [[Media Relayout]] owns the refresh order. + +Guards, and why each one is here: + +* **Identity before geometry.** Two folders holding the same words are not the + same show: the 2026-11-02 census found `Avatar: The Last Airbender` (1999, + tvdb 74852) beside the 2024 remake (385925), `Star Trek: Voyager` beside + `Star Trek: The Animated Series`, and a German thriller Jellyfin had titled + `Millennium (2022) [tvdbid-472933]` beside `Millennium (1996)`. A folder's + identity is read from three places, in this order: its `[tvdbid-N]` tag, the + `tvshow.nfo` Jellyfin wrote — which lives in the metadata view, not on the + array, and is what `--meta` reads — and a `folderid` map passed with + `--ids`. The tag outranks the derived layers: an id that contradicts a folder's + own tag is reported and ignored, never unioned — measured 2026-11-02, Jellyfin's + provider table carried a 2026 special's id on a folder tagged `[tvdbid-73838]`, + and unioning them makes one show of the 2000 series and the special, whose bigger + files then dominate the original's pilot. Two folders are one show when they share + an id, or when neither has one and name and year both agree. Two disjoint id sets + is an absolute stop: the pair is reported as a conflict and nothing under either is + staged. Same name with nothing to decide on is reported unresolved — the fix is the + tag, not a guess. +* **A folder-level verdict destroys content.** 10 of the 25 shows that looked + duplicated held COMPLEMENTARY halves: `Game of Thrones Complete` carries no + S01 and no S08, `Modern Family S05` is the only copy of season 5, `Loiter + Squad Uncensored` is the only copy of season 1. Judged per folder, 20 shows + and some 340 GB of unique episodes would have been staged; judged per episode + the true figure was a sixth of that. +* **A bare number is never an episode.** `256491.mp4` may be an episode id, a + poster, or a season, so a name that states no `SxxEyy` / `1x04` marker is + unkeyed: never staged, and counted in the report. A media server's own episode + table does know those files — pass it with `--witness` and its rows key + whatever the server has matched. +* **The audit re-reads the disk.** Sizes and the survivor's existence are + checked again when the plan is written, because the census is a snapshot: a + file that grew, vanished, or landed in the hold in between must fail the plan, + not surprise the executor. +* **A sidecar that shares an episode's stem follows it.** Subtitles and `.nfo` + left at the old coordinates are ghost bait for `media-view.sh reap`. +* A multi-episode file (`S01E01-02`) is dominated only when every episode in its + range is held elsewhere — and it competes for the survivor seat on each of + them, because the byte law compares files. A pack that states its range in + the name is a name both media servers parse the same way, so the pack winning + against two single-episode rips is the law applied, not a misread. +""" +import argparse +import collections +import importlib.util +import json +import os +import re +import sys + +ENGINE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "media-titles.py") +_spec = importlib.util.spec_from_file_location("eng", ENGINE) +eng = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(eng) + +# The executor's definition of a media file, plus the two the title engine omits: +# an audit that disagrees with media-apply.py is a plan that dies at its +# preflight instead of at the census. +VIDEO = tuple(sorted(set(eng.VIDEO_SUFFIX) | {".ts", ".wmv"})) +TAG_RE = re.compile(r"\[\s*tvdbid[\s._-]*(\d+)\s*\]", re.I) +YEAR_RE = re.compile(r"[(\[]\s*((?:19|20)\d\d)\s*[)\]]") +SEASON_EP = re.compile(r"(?i)\bs(?:eason)?[\s._-]*(\d{1,2})[\s._-]*e(?:p(?:isode)?)?" + r"[\s._-]*(\d{1,3})(?:[\s._-]*(?:-|to|\+)[\s._-]*e?" + r"[\s._-]*(\d{1,3}))?") +NXN = re.compile(r"(?i)(?seasonepisode out of a media " + "server's episode table; keys files whose names carry" + " no marker") + ap.add_argument("--meta", default=None, + help="the merged metadata view (on the host " + "/var/lib/media-meta/view): tvshow.nfo lives in the" + " upper layer, so this is where an untagged folder's" + " tvdb id is readable") + ap.add_argument("--ids", default=None, + help="TSV of foldertvdbid, one line per folder, for" + " identity a lookup settled and the folder name does" + " not state") + ap.add_argument("--json", default=None, + help="also dump the analysis as JSON") + return ap.parse_args() + + +ARGS = parse_args() + + +def episode_keys(name): + """Every (season, episode) an episode name states. + + `S01E03`, `Season 1 E3`, `1x04`, and a range `S01E01-02`, which expands so a + two-in-one file is only droppable when both halves are held elsewhere. A + bare number states nothing and yields nothing.""" + keys = set() + for m in SEASON_EP.finditer(name): + s, first = int(m.group(1)), int(m.group(2)) + keys.add((s, first)) + if m.group(3): + keys.update((s, e) for e in range(first + 1, int(m.group(3)) + 1)) + keys.update((int(m.group(1)), int(m.group(2))) for m in NXN.finditer(name)) + return keys + + +def folder_identity(folder): + """(fold_sep name, year, tvdb ids) for a top-level show folder. + + Tag and year come in both orders — `Name (2008) [tvdbid-84859]` and `Name + [tvdbid-84859] (2008)` are both in the library — so neither is assumed to + sit at the end, and both are cut out of the name folders compare on.""" + ids = frozenset(int(x) for x in TAG_RE.findall(folder)) + year = None + for _ in range(3): + y = YEAR_RE.search(folder) + if not y: + break + year = year or y.group(1) + folder = TAG_RE.sub(" ", f"{folder[:y.start()]} {folder[y.end():]}") + return eng.fold_sep(TAG_RE.sub(" ", folder)), year, ids + + +def nfo_id(path): + """The tvdb id a `tvshow.nfo` states, or None. + + Three shapes are in the library: Jellyfin's ``, Kodi's + ``, and a packager's `` + `` pair; some carry the id + only in the title line as `[tvdbid-N]`.""" + try: + with open(path, encoding="utf-8", errors="replace") as fh: + text = fh.read(16384) + except OSError: + return None + for rx in (r"\s*(\d+)\s*<", + r"]*type\s*=\s*[\"']?tvdb[^>]*>\s*(\d+)\s*<", + r"\s*(\d+)\s*\s*\s*(?:tvdb|thedotdb)", + r"\[\s*tvdbid[\s._-]*(\d+)\s*\]"): + m = re.search(rx, text, re.I) + if m: + return int(m.group(1)) + return None + + +def collect_ids(library, meta, ids_file): + """top-level folder -> {tvdb ids} from the nfo layer and an operator map. + + `meta` is the merged metadata view: Jellyfin's sidecars live in the upper + layer, so a folder's `tvshow.nfo` is not on the raw array. Reading the id + from there is what makes an untagged packager folder identifiable at all — + without it the pair stays unresolved and nothing under it is staged.""" + out = collections.defaultdict(set) + if meta: + try: + tops = sorted(os.listdir(library)) + except OSError: + tops = [] + for folder in tops: + i = nfo_id(os.path.join(meta, folder, "tvshow.nfo")) + if i: + out[folder].add(i) + if ids_file: + with open(ids_file, encoding="utf-8", errors="replace") as fh: + for line in fh: + cols = line.rstrip("\n").split("\t") + if len(cols) < 2 or not cols[1].strip().isdigit(): + continue + out[cols[0]].add(int(cols[1].strip())) + return out + + +def witness_rows(path): + """absolute path -> {(season, episode)} from a media server's episode table.""" + out = collections.defaultdict(set) + if not path: + return out + with open(path, encoding="utf-8", errors="replace") as fh: + for line in fh: + cols = line.rstrip("\n").split("\t") + if len(cols) < 3 or not cols[0].startswith("/"): + continue + try: + out[os.path.abspath(cols[0])].add((int(cols[1]), int(cols[2]))) + except ValueError: + continue + return out + + +def sidecars(path): + """The files beside PATH carrying its exact stem — subtitles, nfo, artwork.""" + folder, stem = os.path.split(path) + stem = os.path.splitext(stem)[0] + out = [] + try: + entries = sorted(os.listdir(folder)) + except OSError: + return out + for e in entries: + p = os.path.join(folder, e) + if p == path or os.path.islink(p) or not os.path.isfile(p): + continue + if os.path.splitext(e)[0] == stem: + out.append(p) + return out + + +def scan(library, hold, witness): + """Every video file under LIBRARY: relative path, size, episode keys. + + A file keys from the witness when it has a row (a server matched it), and + from its own name otherwise; with neither it is unkeyed.""" + files = [] + for dirpath, dirnames, filenames in os.walk(library): + dirnames[:] = sorted(d for d in dirnames + if not d.startswith(".") and d != "@eaDir" + and os.path.abspath(os.path.join(dirpath, d)) != hold) + for f in sorted(filenames): + p = os.path.join(dirpath, f) + if os.path.islink(p) or not f.lower().endswith(VIDEO): + continue + try: + size = os.path.getsize(p) + except OSError: + continue + files.append({"path": os.path.abspath(p), + "rel": os.path.relpath(os.path.abspath(p), library), + "size": size, + "keys": set(witness.get(os.path.abspath(p)) + or episode_keys(os.path.splitext(f)[0]))}) + return files + + +def show_groups(folders, extra_ids): + """(show groups, conflicts, unresolved, disagreements) over top-level folders. + + A folder's id is its `[tvdbid-N]` tag; the nfo layer and `--ids` fill a folder + only when it carries no tag. The rule, in order of trust: an id in common makes + one show; two disjoint id sets are two shows, reported as a conflict and never + merged; no id on either side merges only on name AND year (the 1999 miniseries + and the 2015 series are not one show); one side ided and the other bare is + unresolved, because the fix is the tag. + + The derived layers never overrule a tag. Measured 2026-11-02: Jellyfin's + provider table carried a 2026 special's tvdb id on the folder tagged + `[tvdbid-73838]`, and unioning the two made one show of the 2000 series and the + special — whose bigger files then dominate the original's pilot. A contradiction + is reported instead, because the tag is the operator's word and the provider + table is a guess that can be wrong. + """ + names = {f: folder_identity(f) for f in folders} + tags = {f: set(names[f][2]) for f in folders} + derived = {f: set(extra_ids.get(f, ())) for f in folders} + disagree = {f: (sorted(tags[f]), sorted(derived[f] - tags[f])) + for f in folders if tags[f] and derived[f] - tags[f]} + ids = {f: (tags[f] or derived[f]) for f in folders} + parent = {f: f for f in folders} + + def find(f): + while parent[f] != f: + parent[f] = parent[parent[f]] + f = parent[f] + return f + + conflicts, unresolved = [], [] + order = sorted(folders) + for i, a in enumerate(order): + for b in order[i + 1:]: + na, ya, _ia = names[a] + nb, yb, _ib = names[b] + if na != nb: + continue + ia, ib = ids[a], ids[b] + if ia & ib: + parent[find(b)] = find(a) + elif ia and ib: + conflicts.append((a, b, f"tvdb {sorted(ia)} vs {sorted(ib)}")) + elif not ia and not ib and ya and ya == yb: + parent[find(b)] = find(a) + elif not ia and not ib: + conflicts.append((a, b, + f"same name, years {ya or '?'} vs {yb or '?'}")) + else: + unresolved.append((a, b, "one folder carries a tvdb id," + " the other none")) + groups = collections.defaultdict(list) + for f in order: + groups[find(f)].append(f) + return sorted(groups.values()), conflicts, unresolved, disagree + + +def rank(rec): + """Survivor order: biggest, then shortest path, then lexicographic. + + A strict total order per key means a key's winner is never itself dominated, + so no chain of staging can cycle and exactly one copy of a byte-identical + pair is staged.""" + return (-rec["size"], len(rec["path"]), rec["path"]) + + +def main(): + library = os.path.abspath(ARGS.library) + hold = os.path.abspath(ARGS.hold) + out = ARGS.out or os.path.join(os.path.dirname(library), "tv-dupes-plan.tsv") + if not os.path.isdir(library): + sys.exit(f"refused: {library} is not a directory") + if hold == library or hold.startswith(library + os.sep): + sys.exit(f"refused: --hold {hold} sits inside the library; a hold the" + " census walks is a plan that eats its own output") + + files = scan(library, hold, witness_rows(ARGS.witness)) + folders = sorted({r["rel"].split(os.sep)[0] for r in files}) + ids = collect_ids(library, ARGS.meta, ARGS.ids) + groups, conflicts, unresolved, disagree = show_groups(folders, ids) + + by_show = collections.defaultdict(list) + for members in groups: + for r in files: + if r["rel"].split(os.sep)[0] in members: + by_show[members[0]].append(r) + + rows, audits, report, shows_json = [], [], [], [] + for show in sorted(by_show): + recs = by_show[show] + holders = collections.defaultdict(list) + for r in recs: + for k in r["keys"]: + holders[k].append(r) + winner = {k: min(v, key=rank) for k, v in holders.items()} + dominated = [r for r in recs if r["keys"] + and all(winner[k] is not r for k in r["keys"])] + if not dominated: + continue + dup_keys = sorted({k for r in dominated for k in r["keys"]}) + sole = sum(1 for v in holders.values() if len(v) == 1) + staged_folders = {os.path.dirname(r["rel"]) for r in dominated} + for r in dominated: + for k in r["keys"]: + t = winner[k] + try: + now_src = os.path.getsize(r["path"]) + now_twin = os.path.getsize(t["path"]) + except OSError: + audits.append(f"gone since the census: {r['rel']}" + f" or its survivor {t['rel']}") + break + if now_twin < now_src: + audits.append(f"survivor smaller on disk: {r['rel']}" + f" {now_src} > {t['rel']} {now_twin}") + for r in dominated: + dst = os.path.join(hold, r["rel"]) + rows.append((r["path"], dst)) + for side in sidecars(r["path"]): + rows.append((side, os.path.join(hold, os.path.relpath(side, library)))) + shows_json.append({"show": show, "files": len(dominated), + "bytes": sum(r["size"] for r in dominated), + "episodes": len(dup_keys), + "sole-episode files left": sole, + "folders": sorted(staged_folders)}) + n_ep, n_folder = len(dup_keys), len(staged_folders) + report.append( + f"{show}: stage {len(dominated)} files" + f" ({sum(r['size'] for r in dominated) / 2**30:.1f} GiB) — {n_ep}" + f" episode{n_ep != 1 and 's' or ''} held twice or more; {sole}" + f" episode{sole != 1 and 's' or ''} exist in one copy only, so" + f" {n_folder} folder{n_folder != 1 and 's' or ''} keep their files") + + rows = sorted(set(rows)) + dests = set() + for src, dst in rows: + if dst in dests: + audits.append(f"destination twice: {dst}") + dests.add(dst) + if os.path.exists(dst): + audits.append(f"hold target occupied: {dst}") + if not os.path.isfile(src): + audits.append(f"source missing: {src}") + + unkeyed = collections.Counter(os.path.dirname(r["rel"]) for r in files + if not r["keys"]) + if ARGS.json: + with open(ARGS.json, "w") as fh: + json.dump({"shows": shows_json, + "conflicts": [list(c) for c in conflicts], + "unresolved": [list(u) for u in unresolved], + "disagreement": {f: [t, d] for f, (t, d) in disagree.items()}, + "unkeyed": dict(unkeyed)}, fh, indent=1) + + for line in report: + print(line) + for a, b, why in conflicts: + print(f"CONFLICT (never merged): {a} | {b} — {why}") + for a, b, why in unresolved: + print(f"UNRESOLVED (tag it, then re-run): {a} | {b} — {why}") + for f, (tagged, extra) in sorted(disagree.items()): + print(f"DISAGREE (the tag wins): {f} — tag tvdb {tagged}," + f" derived tvdb {extra}") + if unkeyed: + print(f"unkeyed, never staged: {sum(unkeyed.values())} files in" + f" {len(unkeyed)} folders name no SxxEyy and have no --witness row;" + " most: " + ", ".join(f"{d} ({n})" for d, n in unkeyed.most_common(5))) + + # The gate runs before the plan is written: a plan file that exists is a plan + # somebody will point the executor at, so a failed audit leaves no artifact. + if audits: + print(f"AUDIT FAILED ({len(audits)})", file=sys.stderr) + for a in audits[:10]: + print(f" {a}", file=sys.stderr) + print(f"refused: no plan written to {out}", file=sys.stderr) + return 1 + + mkdirs = sorted({os.path.dirname(dst) for _s, dst in rows}, + key=lambda p: (p.count(os.sep), p)) + with open(out, "w") as fh: + fh.write("action\tsrc\tdst\tkind\n") + for d in mkdirs: + fh.write(f"mkdir\t\t{d}\tdir\n") + for src, dst in rows: + fh.write(f"move\t{src}\t{dst}\ttvdup\n") + + held = sum(os.path.getsize(s) for s, _d in rows if os.path.isfile(s)) + print(f"plan: {len(rows)} moves ({held / 2**30:.1f} GiB) into" + f" {len(mkdirs)} hold folders -> {out}") + print("audit: every staged file has a survivor in the same show holding each of" + " its episodes at >= its size on disk, one destination each") + if rows: + root = os.path.dirname(library) + print(f"next: python3 scripts/media-apply.py apply --plan {out} --root" + f" {root} --journal {os.path.join(root, 'tv-dupes.jsonl')}" + " --metadata-view /var/lib/media-meta/view (dry run; add --apply)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/serverAppConfigs-docs/Dankserver/Media Duplicate Prune.md b/serverAppConfigs-docs/Dankserver/Media Duplicate Prune.md index f11c361..943c359 100644 --- a/serverAppConfigs-docs/Dankserver/Media Duplicate Prune.md +++ b/serverAppConfigs-docs/Dankserver/Media Duplicate Prune.md @@ -1,9 +1,9 @@ --- type: reference title: Media Duplicate Prune -description: "Find and hold duplicate films on the entropy array: the hash ladder, the keep rules, the guards, the purge." +description: "Find and hold duplicate films and series on the array: the screen, per-episode rule, guards, purge." tags: [dankserver, media-server, storage, procedure] -timestamp: 2026-11-01T01:00:00Z +timestamp: 2026-11-02T01:00:00Z --- # Media Duplicate Prune @@ -13,6 +13,10 @@ downloaded again, or the same film grabbed a second time at another quality — [[Entropy Volume]] is permanently full, so both are worth finding. They need different rules, and neither is decided by name similarity alone. +A series duplicates differently again: one show spread over several folders, so +neither the file nor the folder is the unit — see *A series is measured per +episode*. + `/mnt/entropy/media-park` was a third kind: a parking lot that held whole shows and films nothing served. It is gone; its contents went to `NeedsProcessing`. @@ -24,6 +28,7 @@ and films nothing served. It is gone; its contents went to `NeedsProcessing`. | Same work, worse encode | same title, same year, every runtime within 5 % | keep the largest — Frank's rule | | Different cut | same title and year, runtimes more than 5 % apart | **never touched**: an extended and a theatrical version are different products | | Same words, different film | title matches, year or duration does not | never grouped — this is the false-merge class that would destroy a unique file | +| Series split across folders | same show by tvdb id, then the episode `(S,E)` — never the folder name | keep the largest carrier of each episode (`scripts/media-tvdupes.py`) | ## The screen @@ -51,6 +56,84 @@ folder; then the largest. For the same-work class the largest is the answer, and the canonical-name rule only breaks a tie — without it a `Film - 1080p.mkv` can outlive `Film.mkv` for no reason but a space sorting before a dot. +## A series is measured per episode + +`scripts/media-tvdupes.py` censuses a TV library and writes a staging plan in the +executor's own format; it never moves anything itself. + +```bash +python3 scripts/media-tvdupes.py --library /mnt/entropy/TVShows \ + --hold /mnt/entropy/.AGENTTRASH/-tv-duplicates \ + --meta /var/lib/media-meta/view/TVShows --witness witness.tsv \ + --ids /mnt/entropy/tv-tvdb-ids.tsv \ + --out /mnt/entropy/tv-dupes-plan.tsv --json /tmp/tv-groups.json +# then media-apply.py apply / verify / sweep / rollback, exactly as for a film prune +``` + +- **Identity before geometry.** A folder's tvdb id is read from its `[tvdbid-N]` + name tag, from the `tvshow.nfo` the [[Media Metadata View]] exposes (`--meta`), + and from a `folderid` map (`--ids`). Build that map from Jellyfin's own + match, not a lookup: `BaseItems` (`Type like '%TV.Series'`) joined to + `BaseItemProviders` where `ProviderId = 'Tvdb'`, the `/data/TVShows/` prefix + stripped to the bare top-level folder name — 616 folders, none carrying two + ids, written to `/mnt/entropy/tv-tvdb-ids.tsv`. Two folders with disjoint ids + are two shows however alike they read — *Avatar: The Last Airbender* (1999) + beside the 2024 remake, *Millennium* (1996) beside a German thriller Jellyfin + had also titled *Millennium* — and a pair whose identity cannot be proven is + reported `UNRESOLVED` and left alone. The fix is the tag, never a guess. +- **A tag outranks every derived id.** The `--ids` map and the `tvshow.nfo` layer + fill an untagged folder and are **dropped** when they contradict its own + `[tvdbid-N]`, which the report prints as `DISAGREE`. Measured the same day: the + provider table carries a 2026 special's id on `Malcolm in the Middle (2000) + [tvdbid-73838]`; unioned, those folders are one show and the special's bigger + files dominate the original's S01E01–E04 — 3.2 GiB of the 2000 series bound for + the hold as "worse copies". The pass that staged 54 GiB ran with no `--ids`, so no + staged file rests on a borrowed id. +- **A folder-level verdict destroys content.** Of 25 structural candidates in the + 2026-11-02 census, **10 were complementary halves**: *Game of Thrones: The + Complete Series* holds no S01 and no S08, `Modern Family S05` is the only copy of + season 5, *Loiter Squad Uncensored* the only copy of season 1. Judged per folder + the pass would have staged about 340 GB of unique episodes; judged per episode it + staged 54 GiB of true copies. +- **A file is droppable only when every episode it carries is held elsewhere in the + same show at >= its size** (Frank's rule, applied per episode). A byte-identical + pair ties and the tie breaks by path so exactly one copy stages; a file holding + one unique episode stays whatever else it carries. +- **A bare number is never an episode.** `256383.mp4` may be an episode id, a + poster or a whole season, so key-less names are never staged and are counted in + the report (4,036 after the 2026-11-02 pass, almost all genuine bonus material). + Where a packager named files by id, Jellyfin's own table is the witness: a + `pathseasonepisode` TSV from `BaseItems` where `Type like '%TV.Episode'` + and both index numbers are set, read `mode=ro&immutable=1`, its `/data` prefix + rewritten to the raw array path. Jellyfin 12 keeps `Path`, `ParentIndexNumber` and +`IndexNumber` on `BaseItems` and stores the full CLR type name — confirm with +`pragma table_info` before trusting the query, because an empty witness silently +falls back to name parsing. + Seven clips in the library are still named only by number, and where Jellyfin + was asked its own `Name` for them is the number too, so no database holds a + better title — that residue is naming, not a task. +- **A pack states a range and competes on all of it.** `S01E01-02` is droppable only + when both halves are held elsewhere, and it wins the survivor seat for both when + it is the largest carrier: the byte law compares files, and a name both servers + parse as a range stays indexed as two episodes. +- **A sidecar that shares the doomed copy's stem goes with it** — subtitles, `.nfo`, + artwork keyed on the stem — so a staged duplicate leaves no orphan beside the + survivor. The 2026-11-02 hold is 220 media files and nothing else: the dominated + copies were bare rips, and counting a hold's non-media files is the census that + tells you which case you are in. +- **The audit gates the plan file.** Every survivor is re-statted and every + destination re-checked before `--out` is written; on failure the report prints + `AUDIT FAILED` and writes no plan, so there is nothing to hand the executor. +- **Two folders that look duplicated and share zero episode keys are complementary, + not duplicated.** That is how the two pairs the census still refuses to merge were + settled without touching them: *Star Wars - The Clone Wars* (12 files, 1.3 GiB) + beside *Star Wars The Clone Wars* (121 files, 214 GiB), and the *Batman: The + Animated Series* leftovers (3 files) beside the tagged folder (85) — `shared + episode keys: 0` in both cases, so a tag would have freed nothing and cost a + rename with library consequences. Re-running the whole census with the id map over + all 753 top-level folders (606 already tagged) staged **0 files**: the library is + duplicate-free to the limit of provable identity, and what remains is naming. + ## Guards Each of these is a file that would have been destroyed without it: @@ -97,6 +180,14 @@ keeper is missing — either one drops the whole group rather than half-applying - The 2026-11-01 pass held 226 files (~1.1 TiB) in `/mnt/entropy/.AGENTTRASH/20261101-duplicate-prune`, journals `dup-prune-exact.jsonl` and `dup-prune-similar.jsonl`. Purge pending Frank's word. +- The 2026-11-02 TV pass holds 220 files (54 GiB) in + `/mnt/entropy/.AGENTTRASH/20261102-tv-duplicates` — plan `tv-dupes-plan.tsv`, + journal `tv-dupes.jsonl`, 220 moves / 16 hold folders / 7 husks swept — and the + same day's TVShows detritus pass holds 109 files (552 KiB) in + `/mnt/entropy/.AGENTTRASH/20261102-tvshows-detritus` with plan + `tvshows-detritus-plan.tsv` and journal `tvshows-detritus.jsonl`. Both purges + pending Frank's word, beside the 2026-11-01 hold above. All four journals and + plans live at the `entropy` root, not `/tmp`. - **Re-verify each survivor by size, not by the recorded path, before purging.** Detection artifacts and plans key on paths, and a folder renamed by an interactive session ([[Entropy Volume]], *Write doors*) makes its keeper look @@ -117,7 +208,9 @@ stop plex && systemctl --user mask plex`) so its metadata fetches cannot stack o live Jellyfin scan, and because a masked unit stays stopped through a reboot — `unmask` before starting it. The 2026-11-01 morning ran it once Jellyfin reached `Idle`: one accepted `refresh` on the Movies section cleared every row that named a -held path. +held path. The 2026-11-02 TV pass did the same on the TV Shows section (key 3): one +accepted refresh, then `media-verify.py` against that journal reads `landed: 0`, +`still at src: 0`, `REAL: 0` on **both** servers — the correct end state for a prune. The same evening's detritus pass is a second, separate hold: 423 files (12 MiB) in `/mnt/entropy/.AGENTTRASH/20261101-detritus-sweep`, plan `detritus-sweep-plan.tsv`, @@ -128,6 +221,10 @@ subtitle folders (428 of them, 6 GiB) and ripped Blu-ray streams — `.sup` PGS, `.thd`/`.eac3`/`.dts` audio, `.cci` captions, 25 GiB — are not trash, and an extension the census does not recognise is a reason to stop, not to sweep. +A folder holding a `.plexmatch` is held back even when its subtree looks empty: that +file is a manual match — a human decision about what the folder is — and detritus +rules do not overrule one. + `scripts/media-verify.py` reads differently for a prune than for a relayout: `landed: 0` is the **correct** end state (no server may index the hold), so the verdict is `still at src: 0` with `REAL: 0`, and the rows that simply vanished diff --git a/serverAppConfigs-docs/Dankserver/Media Relayout.md b/serverAppConfigs-docs/Dankserver/Media Relayout.md index c8a357c..3ed15db 100644 --- a/serverAppConfigs-docs/Dankserver/Media Relayout.md +++ b/serverAppConfigs-docs/Dankserver/Media Relayout.md @@ -138,7 +138,7 @@ HTTPD close without a single byte; and the same HTTPD drops requests that carry no `Accept` and client-identifier header (see the scan block above for the form that works). -A duplicate prune is the same census and the same proof, with its own keep rules: [[Media Duplicate Prune]]. +A duplicate prune is the same census and the same proof, with its own keep rules — films by title and duration, a series per episode: [[Media Duplicate Prune]]. ## TV in the movie library diff --git a/serverAppConfigs-docs/index.md b/serverAppConfigs-docs/index.md index 59d38c6..27a3bc7 100644 --- a/serverAppConfigs-docs/index.md +++ b/serverAppConfigs-docs/index.md @@ -52,7 +52,7 @@ is a locator: one row, one clause, no numbers. | [[Jellyfin]] | Jellyfin on Dankserver: quadlet unit, overlay metadata view, provider keys, ports, access, transcode status. | | [[mariadb]] | The MariaDB backend for RomM: quadlet unit, network alias, env-file secret handling, and the backup path. | | [[Media Audiobook Relayout]] | Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals. | -| [[Media Duplicate Prune]] | Find and hold duplicate films on the entropy array: the hash ladder, the keep rules, the guards, the purge. | +| [[Media Duplicate Prune]] | Find and hold duplicate films and series on the array: the screen, per-episode rule, guards, purge. | | [[Media Metadata View]] | The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback. | | [[Media Music Relayout]] | Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals. | | [[Media Relayout]] | Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback. | diff --git a/tests/media_tvdupes.bats b/tests/media_tvdupes.bats new file mode 100755 index 0000000..a074fd6 --- /dev/null +++ b/tests/media_tvdupes.bats @@ -0,0 +1,215 @@ +#!/usr/bin/env bats +# Seam: scripts/media-tvdupes.py — the episode-level duplicate census for the TV +# library. +# +# The failure this seam exists to prevent is the one that almost cost real +# content: two folders of one show look like copies, and a folder-level "keep +# the bigger folder" deletes the half of the series only the smaller folder +# holds (the 2026-11-02 census found 10 such complementary pairs). So every +# assertion here is about WHICH files land in the plan and WHICH the tool +# refuses to touch — never about its internals. Everything runs on a synthetic +# library in the test's own tmpdir: no media server, no network, no array. + +load helpers + +TOOL="$REPO_ROOT/scripts/media-tvdupes.py" + +# at PATH SIZE — a file of deterministic bytes. +at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; } + +# sources FILE — the sources the plan moves, one per line. +sources() { awk -F'\t' '$1=="move"{print $2}' "$1"; } + +# destinations FILE — where the plan puts them. +destinations() { awk -F'\t' '$1=="move"{print $3}' "$1"; } + +# plan_moves FILE — how many files the plan stages. +plan_moves() { + [ -f "$1" ] || { echo 0; return; } + awk -F'\t' '$1=="move"' "$1" | wc -l +} + +setup() { + LIB="$BATS_TEST_TMPDIR/TVShows" + HOLD="$BATS_TEST_TMPDIR/.AGENTTRASH/20991231-tv-duplicates" + PLAN="$BATS_TEST_TMPDIR/plan.tsv" + mkdir -p "$LIB" "$HOLD" +} + +@test "the smaller copy of an episode is staged and the larger stays put" { + at "$LIB/Doc Who (2005) [tvdbid-78804]/Season 01/Doc Who - S01E01 - Pilot.mkv" 4000 + at "$LIB/Doc Who (2005) [tvdbid-78804]/Season 01/backup/Doc.Who.S01E01.720p.mkv" 1200 + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$status" -eq 0 ] + [ "$(plan_moves "$PLAN")" -eq 1 ] + run sources "$PLAN" + [[ "$output" == *"backup/Doc.Who.S01E01.720p.mkv"* ]] + [[ "$output" != *"Pilot.mkv"* ]] + run destinations "$PLAN" + [[ "$output" == "$HOLD/"* ]] +} + +@test "an equal-size pair stages exactly one copy" { + at "$LIB/Equal (2001) [tvdbid-11]/Season 01/Equal - S01E01.mkv" 2500 + at "$LIB/Equal (2001) [tvdbid-11]/Rip/Equal - S01E01.mkv" 2500 + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$status" -eq 0 ] + [ "$(plan_moves "$PLAN")" -eq 1 ] +} + +@test "a file holding one episode nothing else has is never staged" { + # The shape that defeats a folder-level verdict: Season 02 exists only here, + # so the folder that looks like a leftover is the only copy of an episode. + at "$LIB/King (2010) [tvdbid-21]/Season 01/King - S01E01 - One.mkv" 3000 + at "$LIB/King (2010) [tvdbid-21]/Season 01/backup/King - S01E01 - One.mkv" 900 + at "$LIB/King (2010) [tvdbid-21]/Season 02/King - S02E01 - Two.mkv" 800 + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$status" -eq 0 ] + run sources "$PLAN" + [[ "$output" != *"S02E01"* ]] + [[ "$output" == *"backup/King - S01E01 - One.mkv"* ]] +} + +@test "a two-in-one pack competes for every episode in its range" { + at "$LIB/Range (2012) [tvdbid-31]/Season 01/Range - S01E01.mkv" 1500 + at "$LIB/Range (2012) [tvdbid-31]/Season 01/Range - S01E02.mkv" 1500 + at "$LIB/Range (2012) [tvdbid-31]/Packs/Range - S01E01-02.mkv" 2400 + + # The pack is the biggest carrier of BOTH episodes, so the two rips go. + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$(plan_moves "$PLAN")" -eq 2 ] + run sources "$PLAN" + [[ "$output" == *"Season 01/Range - S01E01.mkv"* ]] + [[ "$output" != *"Packs/Range - S01E01-02.mkv"* ]] + + # Make the season file the bigger carrier of E02 and the pack becomes the + # only carrier of E01: it stays, however small it is. + rm "$LIB/Range (2012) [tvdbid-31]/Season 01/Range - S01E01.mkv" + rm "$LIB/Range (2012) [tvdbid-31]/Season 01/Range - S01E02.mkv" + at "$LIB/Range (2012) [tvdbid-31]/Season 01/Range - S01E02.mkv" 9000 + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$(plan_moves "$PLAN")" -eq 0 ] +} + +@test "two tvdb ids on the same words are never merged" { + at "$LIB/Avatar The Last Airbender (2024) [tvdbid-385925]/Season 01/Avatar - S01E01.mkv" 9000 + at "$LIB/Avatar The Last Airbender (1999) [tvdbid-74852]/Season 01/Avatar - S01E01.mkv" 500 + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$status" -eq 0 ] + [ "$(plan_moves "$PLAN")" -eq 0 ] + [[ "$output" == *CONFLICT* ]] + [[ "$output" == *"tvdb [74852] vs [385925]"* ]] +} + +@test "a derived id that contradicts the folder's own tag never merges them" { + # Measured 2026-11-02: Jellyfin's provider table carried a 2026 special's tvdb + # id on the folder tagged [tvdbid-73838]. Unioning the two makes one show of a + # 2000 series and a 2026 special, and the special's bigger files dominate the + # original's pilot. The tag is the operator's word; the nfo and `--ids` layers + # fill an untagged folder and never overrule a tagged one. + at "$LIB/Middle (2000) [tvdbid-73838]/Season 01/Middle - S01E01 - Pilot.mkv" 800 + at "$LIB/Middle (2026) [tvdbid-457779]/Season 01/Middle - S01E01 - Episode 1.mkv" 1100 + IDS="$BATS_TEST_TMPDIR/ids.tsv" + printf '%s\t%s\n' "Middle (2000) [tvdbid-73838]" 457779 > "$IDS" + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" --ids "$IDS" + [ "$status" -eq 0 ] + [ "$(plan_moves "$PLAN")" -eq 0 ] + [[ "$output" == *"DISAGREE"* ]] + [[ "$output" == *"tag tvdb [73838]"* ]] + [[ "$output" == *"derived tvdb [457779]"* ]] +} + +@test "an untagged drop is identified by the tvshow.nfo in the metadata view" { + at "$LIB/Clone Wars/Season 01/Clone Wars - S01E01.mkv" 1200 + at "$LIB/Clone Wars (2008) [tvdbid-84859]/Season 01/Clone Wars - S01E01.mkv" 4000 + VIEW="$BATS_TEST_TMPDIR/view" + mkdir -p "$VIEW/Clone Wars" + printf '%s\n' 'Clone Wars' \ + '84859' \ + > "$VIEW/Clone Wars/tvshow.nfo" + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" --meta "$VIEW" + [ "$status" -eq 0 ] + [ "$(plan_moves "$PLAN")" -eq 1 ] + run sources "$PLAN" + [[ "$output" == *"Clone Wars/Season 01/Clone Wars - S01E01.mkv"* ]] + + # Without the nfo layer nothing identifies the drop, so nothing moves. + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" \ + --out "$BATS_TEST_TMPDIR/no-meta.tsv" + [ "$status" -eq 0 ] + [[ "$output" == *UNRESOLVED* ]] + [ "$(plan_moves "$BATS_TEST_TMPDIR/no-meta.tsv")" -eq 0 ] +} + +@test "a file named by an id is keyed only when the witness names it" { + BIG="$LIB/Edd (1999) [tvdbid-75897]/Season 01/Edd - S01E01 - Down the Line.mkv" + SMALL="$LIB/Edd (1999) [tvdbid-75897]/Season 01/104348.mkv" + at "$BIG" 2100 + at "$SMALL" 700 + WIT="$BATS_TEST_TMPDIR/witness.tsv" + printf '%s\t1\t1\n' "$SMALL" > "$WIT" + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$(plan_moves "$PLAN")" -eq 0 ] + [[ "$output" == *"unkeyed, never staged: 1 files"* ]] + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" --witness "$WIT" + [ "$(plan_moves "$PLAN")" -eq 1 ] + run sources "$PLAN" + [[ "$output" == *104348.mkv* ]] +} + +@test "the sidecars of a staged episode are staged with it" { + at "$LIB/Side (2003) [tvdbid-41]/Season 01/Side - S01E01 - First.mkv" 3000 + at "$LIB/Side (2003) [tvdbid-41]/Rip/Side - S01E01 - First.mkv" 1000 + at "$LIB/Side (2003) [tvdbid-41]/Rip/Side - S01E01 - First.srt" 400 + at "$LIB/Side (2003) [tvdbid-41]/Season 01/Side - S01E01 - First.srt" 450 + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$(plan_moves "$PLAN")" -eq 2 ] + run sources "$PLAN" + [[ "$output" == *"Rip/Side - S01E01 - First.srt"* ]] + [[ "$output" != *"Season 01/Side - S01E01 - First.srt"* ]] +} + +@test "a hold inside the library is refused" { + at "$LIB/Hostage (2007) [tvdbid-51]/Season 01/Hostage - S01E01.mkv" 100 + run python3 "$TOOL" --library "$LIB" --hold "$LIB/.AGENTTRASH/hold" \ + --out "$PLAN" + [ "$status" -ne 0 ] + [[ "$output" == *"--hold"* ]] +} + +@test "a plan whose hold target is occupied is refused, plan and all" { + at "$LIB/Gone (2009) [tvdbid-61]/Season 01/Gone - S01E01.mkv" 1000 + at "$LIB/Gone (2009) [tvdbid-61]/Rip/Gone - S01E01.mkv" 400 + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$(plan_moves "$PLAN")" -eq 1 ] + + # The hold already holds one of the destinations: the executor would refuse + # it, so a plan that cannot run must not be handed over at all. + mkdir -p "$HOLD/Gone (2009) [tvdbid-61]/Rip" + at "$HOLD/Gone (2009) [tvdbid-61]/Rip/Gone - S01E01.mkv" 400 + rm "$PLAN" + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$status" -eq 1 ] + [[ "$output" == *"hold target occupied"* ]] + [ ! -e "$PLAN" ] +} + +@test "a library with nothing duplicated writes an empty plan" { + at "$LIB/Unique (2015) [tvdbid-71]/Season 01/Unique - S01E01.mkv" 1000 + at "$LIB/Other Show (2016) [tvdbid-72]/Season 01/Other Show - S01E01.mkv" 1000 + + run python3 "$TOOL" --library "$LIB" --hold "$HOLD" --out "$PLAN" + [ "$status" -eq 0 ] + [ "$(plan_moves "$PLAN")" -eq 0 ] + [[ "$output" == *"plan: 0 moves"* ]] +} From 51ed5c0b2ae1b16395e75c773a5281f1cf74f6d7 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 2 Nov 2026 00:44:51 -0800 Subject: [PATCH 4/4] docs(media): stat through the overlay can outlive the file it points at MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured 2026-11-02 while a TV dedupe pass moved folders on the raw path: the merged tree still answered stat at the old coordinates, link count 0, while ls of the same parent no longer listed the name. Overlayfs keeps a dentry for a lower file moved underneath it, so a tool that tests existence with [ -e ] against the view can believe a file its own readdir denies — the media servers walk by readdir, so this bites a script and never a user. Recorded with it: every pass that moves or measures media walks /mnt/entropy, and the one reader allowed to point at the view is the tool that needs show identity. --- .../Dankserver/Media Metadata View.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/serverAppConfigs-docs/Dankserver/Media Metadata View.md b/serverAppConfigs-docs/Dankserver/Media Metadata View.md index 134ad6d..57691f9 100644 --- a/serverAppConfigs-docs/Dankserver/Media Metadata View.md +++ b/serverAppConfigs-docs/Dankserver/Media Metadata View.md @@ -3,7 +3,7 @@ type: reference title: Media Metadata View description: "The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback." tags: [dankserver, media-server, jellyfin, plex, storage] -timestamp: 2026-11-01T01:00:00Z +timestamp: 2026-11-02T01:00:00Z --- # Media Metadata View @@ -134,6 +134,19 @@ the defaults are the host values above. upper layer*, not for changes on the array underneath. After any relayout or externally-written metadata, trigger a library refresh — see [[Media Relayout]] for the refresh procedure. +- **`stat` through the view can outlive the file.** Measured 2026-11-02 while a TV + dedupe pass moved folders on the raw path: the merged tree still answered `stat` + at the old coordinates — link count `0`, and `ls` of the same parent no longer + listed the name. Overlayfs keeps a dentry for a lower file that was moved + underneath it, so a script that tests existence with `[ -e ]` against the view can + believe a file its own `readdir` denies. Prove existence by listing the parent. + The media servers walk by `readdir`, so ghosts never reach the library — this bites + a tool, never a user, which is why no server symptom accompanies it. +- **Tools that move or measure media walk the raw array, never the view** — every + relayout, prune and detritus pass, and every journal it writes. The view is for + Jellyfin's writes and Plex's reads only. One deliberate exception: `tvshow.nfo` + sidecars live in the upper layer, so a tool reading show identity points at the + view (`media-tvdupes.py --meta /var/lib/media-meta/view/TVShows`). ## Failure signatures @@ -148,6 +161,7 @@ the defaults are the host values above. | A media server's scan record reads `"Status": "Aborted"` | Its container restarted (a cutover, an image bump, an OOM) and the walk was discarded; systemd's `Restart=always` restarts the *server*, never the scan. Trigger a refresh — see [[Media Relayout]]. | | The Jellyfin API answers 401 to every token header | Almost always the header **form**, not the key: 12.4 authenticates only `Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token` for every key, valid or not — and one collapsed backslash in the curl config produces the same 401. `[[Jellyfin]]` owns the block that works. Before concluding anything, prove the key against the server's own table: `select count(*) from ApiKeys where AccessToken = ?` on `jellyfin.db` opened `mode=ro&immutable=1`. | | `systemctl status` finds no such unit | Look under the escaped name, `var-lib-media\x2dmeta-view.mount`, not the repo file name. | +| Stale dentry through the view: `stat` succeeds with link count `0` while the parent's `ls` omits the name | The lower file was moved on the raw path after the view cached its dentry. Nothing is lost; existence through the view is proved by the parent listing. | ## Rollback