feat(media): the movie-library relayout toolchain — census, plan, executor
The movie library on the entropy volume is release-folded: the video inside
`Some.Movie.2019.1080p…-FOO/` is named after the release, and Jellyfin matches
a movie by folder and file name, so the noise cost posters, extras and versions.
Three scripts own the fix, and nothing else in the repo moves a media file.
* media-census.py the read-only inputs: a stat pass, an ffprobe header pass
(nice'd, so it is safe beside a live endpoint), and the
Plex metadata cache read that outranks any filename guess
* media-plan.py the Jellyfin-format planner: one folder per movie, the
video named exactly like the folder, extras in the
canonical subfolders, copies of one film as versions. It
opens no media file, so a whole-library plan costs seconds,
and it ends in gates — bad-names, occupied, double-claim,
left-unmoved, unplanned — that must all read zero before a
human points the executor at 24 TB
* media-apply.py the executor: rename(2) only, the intent fsynced to a
write-ahead journal BEFORE the move it authorises, so an
interrupted run is finishable and every run is undoable.
Refusals are all-or-nothing and it never overwrites a file
Laws the fixture pins (tests/media_relayout.bats, 16 tests, each proven red
against a reverted copy of the rule it guards):
* a Plex year of 0 means "indexed, never matched" — trusting it produced
folders "Title (0)": items with no year, no poster, no identity
* a Plex title carrying "/" is sanitised, or "V/H/S/2" became a five-level
tree neither matcher reads as a movie
* a folder inside a movie's folder is bonus territory even when its name is
misspelled ("Exrtas"), unless the file's own name carries another year
* matching words split on everything but a letter or digit — the earlier
tokenizer left "(1960)/extras" glued, so a film could not be found by the
folder holding it and one box set's featurettes drifted into a neighbour
* a name with no words in it is bonus material, never a movie folder named 1
Docs: the runbook page owns the procedure, the gates and the rollback; AGENTS.md
links it from the runbook index and the summary table.
This commit is contained in:
dank committed
1 parent
4db985b90d
commit
272e06daf6
8 files changed
+2435
No files matched your search
Executable
+321
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
"""media-apply.py — the only thing that moves movie-library files.
|
||||
|
||||
Reads the plan written by scripts/media-plan.py (rows: `action src dst kind`,
|
||||
plus `mkdir` rows) and performs it with rename(2) — bytes are never copied, so
|
||||
an interrupted run has destroyed nothing. Every run is resumable and every run
|
||||
is undoable: the journal is a write-ahead log, one JSON line per intent, fsynced
|
||||
BEFORE the rename it authorises. That ordering is the whole design: a crash
|
||||
leaves either an intent whose move happened or an intent whose move never ran,
|
||||
and both are recognised on the next run. A file that moved with no journal line
|
||||
cannot have been moved by this tool, and saying so is better than guessing.
|
||||
|
||||
Three subcommands:
|
||||
|
||||
apply --plan PLAN [--root ROOT] [--journal J] [--apply]
|
||||
no --apply => report only and write NOTHING, not even folders
|
||||
rollback --journal J [--root ROOT] [--apply]
|
||||
back to the paths the census saw; folders this run created are
|
||||
removed only when empty, never anything else
|
||||
verify --plan PLAN --journal J [--root ROOT]
|
||||
every planned file is at its target with the journalled byte count
|
||||
|
||||
Refusals (exit 2, before the first rename, with nothing touched): a target
|
||||
already occupied by a foreign file, a source that has gone, a duplicate target,
|
||||
a path outside --root, a journal that was already rolled back, a move that would
|
||||
cross a filesystem (rename cannot do it and a silent copy at 24 TB would not).
|
||||
|
||||
--force relaxes only the guards that are opinion (journal already rolled back,
|
||||
paths outside --root); it never allows overwriting a file.
|
||||
|
||||
The journal defaults to `media-relayout.jsonl` BESIDE the library root, never
|
||||
inside it: a log inside the library is a file Jellyfin will index.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
DEFAULT_ROOT = os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies")
|
||||
|
||||
|
||||
class Refuse(Exception):
|
||||
"""A gate the operator has to see; every refusal leaves the library alone."""
|
||||
|
||||
|
||||
def journal_default(root):
|
||||
return os.path.join(os.path.dirname(os.path.abspath(root)), "media-relayout.jsonl")
|
||||
|
||||
|
||||
class Journal:
|
||||
"""Append-only JSONL of intents. Writes are fsynced before the tool acts."""
|
||||
|
||||
def __init__(self, path, writing):
|
||||
self.path, self.writing = path, writing
|
||||
self.records = []
|
||||
if os.path.exists(path):
|
||||
for line in open(path):
|
||||
line = line.strip()
|
||||
if line:
|
||||
self.records.append(json.loads(line))
|
||||
self._fh = None
|
||||
|
||||
def already(self, dst, src):
|
||||
"""An intent for this exact (src, dst) was written by an earlier run."""
|
||||
return any(r.get("op") == "move" and r.get("dst") == dst
|
||||
and r.get("src") == src for r in self.records)
|
||||
|
||||
@property
|
||||
def rolled_back(self):
|
||||
return any(r.get("op") == "rollback" for r in self.records)
|
||||
|
||||
def record(self, **fields):
|
||||
self.records.append(fields)
|
||||
if not self.writing:
|
||||
return
|
||||
if self._fh is None:
|
||||
self._fh = open(self.path, "a")
|
||||
self._fh.write(json.dumps(fields) + "\n")
|
||||
self._fh.flush()
|
||||
os.fsync(self._fh.fileno())
|
||||
|
||||
def close(self):
|
||||
if self._fh:
|
||||
self._fh.close()
|
||||
|
||||
|
||||
def read_plan(path, root, force):
|
||||
"""Plan rows: (mkdirs, keeps, moves), header dropped, all paths under root."""
|
||||
mkdirs, keeps, moves = [], [], []
|
||||
with open(path) as fh:
|
||||
header = fh.readline()
|
||||
if not header.startswith("action\t"):
|
||||
raise Refuse(f"{path} is not a plan (first column is "
|
||||
f"{header.split(chr(9))[0]!r}, expected 'action') —"
|
||||
" regenerate it with scripts/media-plan.py")
|
||||
for line in fh:
|
||||
f = line.rstrip("\n").split("\t")
|
||||
if f[0] == "mkdir":
|
||||
mkdirs.append(f[2])
|
||||
elif f[0] == "keep":
|
||||
keeps.append(f[1])
|
||||
elif f[0] == "move":
|
||||
moves.append((f[1], f[2], f[3] if len(f) > 3 else ""))
|
||||
if not force:
|
||||
for p in mkdirs + keeps + [p for m in moves for p in m[:2]]:
|
||||
if p and not (p == root or p.startswith(root + os.sep)):
|
||||
raise Refuse(f"{p} is outside --root {root}"
|
||||
" (a plan built for another library?)")
|
||||
return mkdirs, keeps, moves
|
||||
|
||||
|
||||
def do_apply(args, root, journal, mkdirs, keeps, moves):
|
||||
"""Preflight everything, then folders, then renames. Never a partial start."""
|
||||
problems = []
|
||||
for p in keeps:
|
||||
if not os.path.exists(p):
|
||||
problems.append(f"missing: keep row source is gone: {p}")
|
||||
seen = {}
|
||||
for src, dst, _kind in moves:
|
||||
if src == dst:
|
||||
problems.append(f"no-op: move row whose target is its source: {src}")
|
||||
if dst in seen:
|
||||
problems.append(f"double-claim: {dst} claimed by {seen[dst]} and {src}")
|
||||
seen[dst] = src
|
||||
if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
|
||||
continue # this intent already ran
|
||||
if not os.path.exists(src):
|
||||
if os.path.exists(dst):
|
||||
problems.append(f"missing: {src} is gone and {dst} carries no journal"
|
||||
" line — it moved outside this tool, refusing to guess")
|
||||
else:
|
||||
problems.append(f"missing: {src} is gone and {dst} is not there either")
|
||||
continue
|
||||
if os.path.exists(dst):
|
||||
problems.append(f"occupied: target already holds another file: {dst}")
|
||||
if problems:
|
||||
print(f"refusing: {len(problems)} problem(s), nothing moved")
|
||||
for p in sorted(problems)[:20]:
|
||||
print(" " + p)
|
||||
if len(problems) > 20:
|
||||
print(f" ... and {len(problems) - 20} more")
|
||||
return 2
|
||||
|
||||
if not args.apply:
|
||||
print(f"dry run: {len(moves)} moves, {len(mkdirs)} folders, {len(keeps)} files stay"
|
||||
" — nothing written (pass --apply to run)")
|
||||
for src, dst, _k in moves[:5]:
|
||||
print(f" would move {src} -> {dst}")
|
||||
return 0
|
||||
|
||||
made = {r.get("dst") for r in journal.records if r.get("op") == "mkdir"}
|
||||
dirs_new = 0
|
||||
for d in sorted(mkdirs, key=lambda q: q.count(os.sep)):
|
||||
if d in made or os.path.isdir(d):
|
||||
continue
|
||||
journal.record(op="mkdir", dst=d)
|
||||
os.makedirs(d, exist_ok=True)
|
||||
dirs_new += 1
|
||||
|
||||
# same-filesystem gate: rename cannot cross a device, and a silent copy of a
|
||||
# library this size is not an option anyone would choose.
|
||||
cross = []
|
||||
for src, dst, _k in moves:
|
||||
if journal.already(dst, src) and os.path.exists(dst) and not os.path.exists(src):
|
||||
continue
|
||||
parent = os.path.dirname(dst)
|
||||
if os.path.exists(parent) and os.stat(parent).st_dev != os.stat(src).st_dev:
|
||||
cross.append(dst)
|
||||
if cross:
|
||||
print(f"refusing: {len(cross)} moves would cross a filesystem"
|
||||
f" (first: {cross[0]}) — this tool only renames")
|
||||
return 2
|
||||
|
||||
moved = done = 0
|
||||
for src, dst, _kind in moves:
|
||||
if journal.already(dst, src):
|
||||
if os.path.exists(dst) and not os.path.exists(src):
|
||||
done += 1
|
||||
continue
|
||||
if os.path.exists(src) and not os.path.exists(dst):
|
||||
os.rename(src, dst) # the intent ran, the move did not
|
||||
moved += 1
|
||||
continue
|
||||
st = os.stat(src)
|
||||
journal.record(op="move", src=src, dst=dst, bytes=st.st_size, mode=st.st_mode,
|
||||
mtime=st.st_mtime, ino=st.st_ino, dev=st.st_dev)
|
||||
os.rename(src, dst)
|
||||
moved += 1
|
||||
print(f"applied: {moved} moved, {done} already done in an earlier run,"
|
||||
f" {dirs_new} folders created, {len(keeps)} files stayed put")
|
||||
print(f"journal: {journal.path} — undo with: media-apply.py rollback"
|
||||
f" --journal {journal.path} --root {root} --apply")
|
||||
return 0
|
||||
|
||||
|
||||
def do_verify(root, journal, moves, keeps):
|
||||
"""Every planned file sits at its target carrying the journalled bytes."""
|
||||
journalled = {(r["src"], r["dst"]): r for r in journal.records
|
||||
if r.get("op") == "move"}
|
||||
todo = moves or [(r["src"], r["dst"], "") for r in journal.records
|
||||
if r.get("op") == "move"]
|
||||
bad, missing, bytes_total = [], [], 0
|
||||
for src, dst, _kind in todo:
|
||||
rec = journalled.get((src, dst))
|
||||
if not os.path.exists(dst):
|
||||
missing.append(dst)
|
||||
continue
|
||||
size = os.path.getsize(dst)
|
||||
bytes_total += size
|
||||
if rec and rec.get("bytes") not in (None, size):
|
||||
bad.append(f"{dst}: {size} bytes on disk, {rec['bytes']} at move time")
|
||||
if os.path.exists(src) and src != dst:
|
||||
bad.append(f"{src}: still in place as well as at {dst}")
|
||||
missing += [p for p in keeps if not os.path.exists(p)]
|
||||
if bad or missing:
|
||||
print(f"verify FAILED: {len(missing)} absent, {len(bad)} wrong")
|
||||
for line in (missing + bad)[:20]:
|
||||
print(" " + line)
|
||||
return 2
|
||||
empty = [r["dst"] for r in journal.records
|
||||
if r.get("op") == "mkdir" and os.path.isdir(r["dst"])
|
||||
and not os.listdir(r["dst"])]
|
||||
print(f"verified: {len(todo)} files at their targets"
|
||||
f" ({bytes_total / 2**40:.2f} TiB), {len(keeps)} kept")
|
||||
if empty:
|
||||
print(f"note: {len(empty)} folders are empty (left in place — see the trash law)")
|
||||
return 0
|
||||
|
||||
|
||||
def do_rollback(args, root, journal):
|
||||
"""Undo every journalled move, newest first, after a full preflight."""
|
||||
moves = [r for r in journal.records if r.get("op") == "move"]
|
||||
made = [r["dst"] for r in journal.records if r.get("op") == "mkdir"]
|
||||
problems = []
|
||||
for r in reversed(moves):
|
||||
src, dst = r["src"], r["dst"]
|
||||
if not args.force and not (src.startswith(root + os.sep)
|
||||
or dst.startswith(root + os.sep)):
|
||||
problems.append(f"outside --root: {dst}")
|
||||
continue
|
||||
if os.path.exists(dst) and os.path.exists(src):
|
||||
problems.append(f"occupied: both paths exist, refusing to choose:"
|
||||
f" {src} | {dst}")
|
||||
elif os.path.exists(dst):
|
||||
pass
|
||||
elif os.path.exists(src):
|
||||
continue # already undone
|
||||
else:
|
||||
problems.append(f"missing: {dst} is gone and {src} never came back")
|
||||
if problems:
|
||||
print(f"refusing: {len(problems)} problem(s), nothing moved back")
|
||||
for p in problems[:20]:
|
||||
print(" " + p)
|
||||
return 2
|
||||
if not args.apply:
|
||||
print(f"dry run: would move {len(moves)} files back to their census paths"
|
||||
" (pass --apply)")
|
||||
return 0
|
||||
undone = 0
|
||||
for r in reversed(moves):
|
||||
if os.path.exists(r["dst"]) and not os.path.exists(r["src"]):
|
||||
os.makedirs(os.path.dirname(r["src"]), exist_ok=True)
|
||||
os.rename(r["dst"], r["src"])
|
||||
undone += 1
|
||||
dirs_gone = 0
|
||||
for d in sorted(set(made), key=lambda q: -q.count(os.sep)):
|
||||
if d == root or not os.path.isdir(d) or os.listdir(d):
|
||||
continue
|
||||
try:
|
||||
os.rmdir(d)
|
||||
dirs_gone += 1
|
||||
except OSError as e:
|
||||
print(f"left in place: {d} ({e})")
|
||||
journal.record(op="rollback", undone=undone, dirs_removed=dirs_gone,
|
||||
t=datetime.datetime.now(datetime.timezone.utc).isoformat())
|
||||
print(f"rolled back: {undone} files back where the census found them,"
|
||||
f" {dirs_gone} empty folders removed")
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="apply or undo a media relayout plan")
|
||||
ap.add_argument("cmd", choices=["apply", "rollback", "verify"])
|
||||
ap.add_argument("--plan")
|
||||
ap.add_argument("--root", default=DEFAULT_ROOT)
|
||||
ap.add_argument("--journal")
|
||||
ap.add_argument("--apply", action="store_true", help="actually change the library")
|
||||
ap.add_argument("--force", action="store_true", help="relax the opinion guards")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = os.path.abspath(args.root)
|
||||
if not os.path.isdir(root):
|
||||
raise Refuse(f"no library at {root}")
|
||||
journal_path = args.journal or journal_default(root)
|
||||
journal = Journal(journal_path, args.apply)
|
||||
try:
|
||||
if args.cmd in ("rollback", "verify") and not os.path.exists(journal_path):
|
||||
raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
|
||||
if args.cmd == "rollback" and not args.force and journal.rolled_back:
|
||||
raise Refuse("this journal has already been rolled back"
|
||||
" (--force to replay it)")
|
||||
if args.cmd in ("apply", "verify") and not args.plan:
|
||||
raise Refuse(f"{args.cmd} needs --plan")
|
||||
mkdirs, keeps, moves = (read_plan(args.plan, root, args.force)
|
||||
if args.plan else ([], [], []))
|
||||
if args.cmd == "apply":
|
||||
return do_apply(args, root, journal, mkdirs, keeps, moves)
|
||||
if args.cmd == "verify":
|
||||
return do_verify(root, journal, moves, keeps)
|
||||
return do_rollback(args, root, journal)
|
||||
finally:
|
||||
journal.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except Refuse as r:
|
||||
print(f"refusing: {r}")
|
||||
sys.exit(2)
|
||||
Executable
+203
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""Read-only census of a movie library — the inputs a relayout plan is built from.
|
||||
|
||||
media-census.py enumerate [--root DIR] [--out TSV]
|
||||
media-census.py probe [--enum TSV] [--out TSV] [--jobs N]
|
||||
media-census.py plex [--root DIR] [--db PATH] [--out TSV]
|
||||
|
||||
Run it on the machine that mounts the library. Nothing here writes inside the
|
||||
library and nothing is deleted; the only outputs are the three TSVs below, and
|
||||
scripts/media-plan.py reads them instead of touching media.
|
||||
|
||||
enum bytes <TAB> ext <TAB> path every file under the root
|
||||
probed bytes <TAB> path <TAB> dur <TAB> vcodec <TAB> WxH <TAB> fps <TAB> vbit
|
||||
<TAB> pix <TAB> audio <TAB> subs <TAB> spare <TAB> format
|
||||
one row per video-container file; `audio` is "codec:lang+codec:lang",
|
||||
`subs` the subtitle-stream count, `spare` the audio streams carrying
|
||||
neither a default nor a forced tag
|
||||
plex file <TAB> leaf_title <TAB> … (see PLEX_COLS)
|
||||
the titles Plex already matched — the authority over any filename guess
|
||||
|
||||
`probe` is the slow step (every container is opened for its header, ~10 min for
|
||||
4,800 files on a spindle array at 6 workers) so it is `nice`d: a census beside a
|
||||
live service must not starve it. `enumerate` is a stat pass, so re-censusing
|
||||
before a plan is cheap — and a plan is only ever as good as these three files.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
# Containers ffprobe can read. .iso/.img are deliberately NOT here: a disc image is
|
||||
# a mounted volume, not a stream, and the planner treats it as a movie file instead.
|
||||
PROBE_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg",
|
||||
".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".bdmv", ".vob"}
|
||||
# Trash and thumbnail caches are not library content, and this repo's trash policy
|
||||
# means .AGENTTRASH can sit inside a movie folder at any time.
|
||||
SKIP_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir|lost\+found)(/|$)")
|
||||
PLEX_DB = ("/var/lib/plexmediaserver/Library/Application Support/Plex Media Server"
|
||||
"/Plug-in Support/Databases/com.plexapp.plugins.library.db")
|
||||
PLEX_COLS = ["file", "leaf_title", "leaf_orig", "leaf_edition", "leaf_year",
|
||||
"parent_title", "parent_orig", "parent_year", "parent_edition",
|
||||
"container", "vcodec", "acodec", "width", "height"]
|
||||
PLEX_SQL = """
|
||||
select p.file, coalesce(mi.title,''), coalesce(mi.original_title,''),
|
||||
coalesce(mi.edition_title,''), coalesce(mi.year,0),
|
||||
coalesce(par.title,''), coalesce(par.original_title,''),
|
||||
coalesce(par.year,0), coalesce(par.edition_title,''),
|
||||
m.container, m.video_codec, m.audio_codec, m.width, m.height
|
||||
from media_parts p
|
||||
join media_items m on m.id = p.media_item_id
|
||||
join metadata_items mi on mi.id = m.metadata_item_id
|
||||
left join metadata_items par on par.id = mi.parent_id
|
||||
where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
||||
"""
|
||||
|
||||
|
||||
def enumerate_lib(root, out):
|
||||
"""One row per real file under root: size, extension, path."""
|
||||
rows = []
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
dirnames[:] = [d for d in dirnames
|
||||
if not SKIP_DIR.search(os.path.join(dirpath, d))]
|
||||
for fn in filenames:
|
||||
p = os.path.join(dirpath, fn)
|
||||
if SKIP_DIR.search(p):
|
||||
continue
|
||||
try:
|
||||
st = os.lstat(p)
|
||||
except OSError:
|
||||
continue
|
||||
if not os.path.isfile(p) or os.path.islink(p):
|
||||
continue # links and sockets are not library content
|
||||
rows.append((st.st_size, os.path.splitext(fn)[1].lower(), p))
|
||||
with open(out, "w") as fh:
|
||||
for size, ext, p in rows:
|
||||
fh.write(f"{size}\t{ext}\t{p}\n")
|
||||
vid = [r for r in rows if r[1] in PROBE_EXT]
|
||||
print(f"files={len(rows)} bytes={sum(r[0] for r in rows)} "
|
||||
f"probes={len(vid)} probe_bytes={sum(r[0] for r in vid)} -> {out}")
|
||||
|
||||
|
||||
def probe_one(path):
|
||||
"""ffprobe header read -> the probed TSV row (never the whole file)."""
|
||||
try:
|
||||
os.nice(19) # a census must never outrank the media server
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
size = os.stat(path).st_size
|
||||
except OSError:
|
||||
return None
|
||||
cmd = ["ffprobe", "-v", "error", "-print_format", "json",
|
||||
"-show_format", "-show_streams", path]
|
||||
try:
|
||||
done = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||
data = json.loads(done.stdout or "{}")
|
||||
except (subprocess.TimeoutExpired, subprocess.SubprocessError,
|
||||
json.JSONDecodeError, OSError):
|
||||
data = {}
|
||||
st = data.get("streams", [])
|
||||
fmt = data.get("format", {})
|
||||
v = next((s for s in st if s.get("codec_type") == "video"), None)
|
||||
aud = [s for s in st if s.get("codec_type") == "audio"]
|
||||
subs = [s for s in st if s.get("codec_type") == "subtitle"]
|
||||
fps = "-"
|
||||
if v and v.get("r_frame_rate"):
|
||||
try:
|
||||
n, d = (int(x) for x in v["r_frame_rate"].split("/", 1))
|
||||
fps = f"{round(n / d * 100) / 100:g}" if d else "0"
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def num(x):
|
||||
try:
|
||||
return round(float(x))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
return [size, path, num(fmt.get("duration")),
|
||||
(v or {}).get("codec_name", "-"),
|
||||
f"{v['width']}x{v['height']}" if v else "-",
|
||||
fps, num(fmt.get("bit_rate")), (v or {}).get("pix_fmt", "-"),
|
||||
"+".join(f"{a.get('codec_name', '?')}:{a.get('language', '?')}" for a in aud),
|
||||
len(subs),
|
||||
sum(1 for a in aud if not re.search(
|
||||
r"default|forced", json.dumps(a.get("tags", {})), re.I)),
|
||||
fmt.get("format_name", "-") or "-"]
|
||||
|
||||
|
||||
def probe(enum, out, jobs):
|
||||
"""Probe every video-container row of the enum TSV, in parallel."""
|
||||
paths = []
|
||||
with open(enum) as fh:
|
||||
for line in fh:
|
||||
f = line.rstrip("\n").split("\t", 2)
|
||||
if len(f) == 3 and f[1] in PROBE_EXT:
|
||||
paths.append(f[2])
|
||||
print(f"probing {len(paths)} files with {jobs} workers", flush=True)
|
||||
done = unreadable = 0
|
||||
with open(out, "w") as fh, ThreadPoolExecutor(max_workers=jobs) as pool:
|
||||
for row in pool.map(probe_one, paths):
|
||||
if row is None:
|
||||
continue
|
||||
if row[3] == "-":
|
||||
unreadable += 1 # header unreadable: the row still gets written
|
||||
done += 1
|
||||
fh.write("\t".join(str(x) for x in row) + "\n")
|
||||
if done % 250 == 0:
|
||||
print(f" {done}/{len(paths)}", flush=True)
|
||||
print(f"probed={done} unreadable={unreadable} -> {out}")
|
||||
|
||||
|
||||
def plex(root, db, out):
|
||||
"""Plex's matched titles for files under root, from its DB, read-only."""
|
||||
uri = f"file:{db}?mode=ro&immutable=1"
|
||||
try:
|
||||
con = sqlite3.connect(uri, uri=True)
|
||||
except sqlite3.Error as e:
|
||||
sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n"
|
||||
"Plex runs rootless here: run this as the container's owner, never "
|
||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||
with open(out, "w", newline="") as fh:
|
||||
w = csv.writer(fh, delimiter="\t")
|
||||
w.writerow(PLEX_COLS)
|
||||
n = 0
|
||||
for r in con.execute(PLEX_SQL, (root + "/%",)):
|
||||
w.writerow([str(x).replace("\t", " ").replace("\n", " ") for x in r])
|
||||
n += 1
|
||||
print(f"plex-matched={n} -> {out}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
root = {"default": os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies"),
|
||||
"help": "library root"}
|
||||
e = sub.add_parser("enumerate", help="stat every file under the root")
|
||||
e.add_argument("--root", **root)
|
||||
e.add_argument("--out", default="/tmp/media-enum.tsv")
|
||||
p = sub.add_parser("probe", help="ffprobe every video container")
|
||||
p.add_argument("--enum", default="/tmp/media-enum.tsv")
|
||||
p.add_argument("--out", default="/tmp/media-probed.tsv")
|
||||
p.add_argument("--jobs", type=int, default=6)
|
||||
x = sub.add_parser("plex", help="export Plex's matched titles for the root")
|
||||
x.add_argument("--root", **root)
|
||||
x.add_argument("--db", default=PLEX_DB)
|
||||
x.add_argument("--out", default="/tmp/plex-movies.tsv")
|
||||
a = ap.parse_args()
|
||||
if a.cmd == "enumerate":
|
||||
enumerate_lib(a.root, a.out)
|
||||
elif a.cmd == "probe":
|
||||
probe(a.enum, a.out, a.jobs)
|
||||
else:
|
||||
plex(a.root, a.db, a.out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+882
@@ -0,0 +1,882 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""Plan a Jellyfin-format relayout of a movie library. DRY RUN ONLY: nothing moves.
|
||||
|
||||
media-plan.py [--root DIR] [--enum TSV] [--probed TSV] [--plex TSV]
|
||||
[--out TSV] [--exceptions TXT]
|
||||
|
||||
Inputs are the TSVs scripts/media-census.py exports, plus the optional Plex
|
||||
metadata-cache export whose clean titles outrank the filename parser; the title
|
||||
authority for everything Plex never matched is scripts/media-titles.py beside this
|
||||
file. No media file is opened, so planning the whole library costs seconds.
|
||||
|
||||
PLAN is a TSV the executor (scripts/media-apply.py) replays verbatim —
|
||||
`action<TAB>src<TAB>dst<TAB>kind` with action in mkdir | move | keep, ordered so
|
||||
parents exist before children. EXCEPTIONS is the human review list: every file the
|
||||
plan could not name a film for, every collision it refused, the READMEs the
|
||||
executor must write into the piles it created.
|
||||
|
||||
Target contract (jellyfin.org/docs/general/server/media/movies):
|
||||
* one folder per movie, named Title (year)
|
||||
* the video inside is named exactly like the folder
|
||||
* extra copies of one film are versions: "<base> - <label>.ext"; the base must
|
||||
match the folder name character-for-character or Jellyfin splits them apart
|
||||
* extras live in canonical subfolders (trailers, featurettes, deleted scenes,
|
||||
interviews, samples, shorts, clips, other, extras, backdrops, theme-music)
|
||||
* sidecars <base>.<lang>.<flag>.<ext>; images <base>-<poster|backdrop|...>
|
||||
* <>:"/\|?* break the matcher, so they never survive here
|
||||
* VIDEO_TS/BDMV folders support neither multiple versions nor parts
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import csv
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
|
||||
# Inputs are the census TSVs scripts/media-census.py writes; the engine beside
|
||||
# this file is the title authority. Nothing here opens a media file.
|
||||
def parse_args():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--root", default=os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies"),
|
||||
help="library root the plan moves files inside")
|
||||
ap.add_argument("--enum", default="/tmp/media-enum.tsv", help="file census TSV")
|
||||
ap.add_argument("--probed", default="/tmp/media-probed.tsv", help="ffprobe census TSV")
|
||||
ap.add_argument("--plex", default="/tmp/plex-movies.tsv",
|
||||
help="Plex metadata export (clean titles); absent = parser only")
|
||||
ap.add_argument("--out", default="/tmp/jf-plan.tsv")
|
||||
ap.add_argument("--exceptions", default="/tmp/jf-exceptions.txt")
|
||||
return ap.parse_args()
|
||||
|
||||
|
||||
ARGS = parse_args()
|
||||
ROOT = ARGS.root
|
||||
ENUM, PROBE, PLEX = ARGS.enum, ARGS.probed, ARGS.plex
|
||||
PLAN, EXCEPTS = ARGS.out, ARGS.exceptions
|
||||
ENGINE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "media-titles.py")
|
||||
|
||||
spec = importlib.util.spec_from_file_location("eng", ENGINE)
|
||||
eng = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(eng)
|
||||
|
||||
VIDEO_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg",
|
||||
".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".flv", ".vob", ".iso",
|
||||
".img", ".mpv", ".3gp", ".rm", ".rmvb", ".m2v", ".mpv"}
|
||||
DISC_DIRS = {"BDMV", "VIDEO_TS", "CERTIFICATE", "AACS", "MLPI", "CLIPINF"}
|
||||
SUB_EXT = {".srt", ".ass", ".ssa", ".sub", ".vtt"}
|
||||
IMG_EXT = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif", ".tbn"}
|
||||
AUDIO_DIR_RE = re.compile(r"soundtrack|score|\bost\b|music|song|album", re.I)
|
||||
SUB_KEEP_RE = re.compile(r"^(?!.*\d)(?!.*[._])(?!.*\[)(?!.*\()([A-Za-z][A-Za-z -]{1,24})$")
|
||||
MUSIC_TOP_RE = re.compile(r"(?i)soundtrack|score|\bost\b|music|album|song")
|
||||
SUB_DIR_OK = re.compile(r"(?i)^$|subs?|subtitles?|srt|custom ?subs?(\[.*\])?"
|
||||
r"|(additional|extra) languages|languages|resources")
|
||||
AUDIO_DIR_OK = re.compile(r"(?i)^$|audio(s)?|extra audio|additional audio"
|
||||
r"|(additional|extra) languages|commentar(y|ies)"
|
||||
r"|audio commentary|audio tracks?")
|
||||
|
||||
|
||||
AUDIO_EXT = {".mp3", ".flac", ".m4a", ".aac", ".dts", ".wav", ".ac3", ".mka",
|
||||
".opus", ".ogg", ".wma"}
|
||||
NFO_EXT = {".nfo"}
|
||||
IMG_PRIMARY = {"poster", "folder", "cover", "default", "movie", "show"}
|
||||
IMG_OTHER = {"backdrop": "backdrop", "fanart": "backdrop", "background": "backdrop",
|
||||
"banner": "banner", "logo": "logo", "clearlogo": "logo",
|
||||
"thumb": "thumb", "landscape": "thumb", "art": "backdrop",
|
||||
"clearart": "backdrop", "jacket": "poster", "cdart": "poster",
|
||||
"discart": "poster", "disc": "poster"}
|
||||
RES_RE = re.compile(r"\b(2160|4320|4k|1080|720|576|480|360|240)(i|p)\b", re.I)
|
||||
# The characters Jellyfin's matcher cannot carry in a file or folder name. "/" is
|
||||
# one of them and it is the dangerous one: Plex titles KEEP it ("V/H/S/2", and this
|
||||
# library holds six of those films), so leaving it in a name turns ONE movie folder
|
||||
# into a nested tree — Movies/V/H/S/2 (2013)/… — and Jellyfin then sees four
|
||||
# folders, none of them a movie.
|
||||
BAD_CHARS = re.compile(r'[<>:"/|?*]')
|
||||
PART_RE = re.compile(r"(?:^|[\s._-])(cd|dvd|part|pt|disc|disk|disco)"
|
||||
r"[\s._-]?([1-9abcd])(?:$|[\s._-])", re.I)
|
||||
LANGS = {"en", "eng", "it", "ita", "ja", "jpn", "de", "deu", "fr", "fra", "es",
|
||||
"spa", "pt", "por", "ru", "rus", "zh", "chi", "ko", "kor", "sv", "no",
|
||||
"da", "nl", "pl", "cs", "hu", "fi", "tr", "el", "he", "ar", "uk", "ro",
|
||||
"bg", "hr", "sr", "sk", "et", "lv", "lt", "id", "th", "vi", "fa", "la"}
|
||||
FLAGS = {"default", "forced", "foreign", "sdh", "cc"}
|
||||
NOISE = {"sub", "subs", "subtitle", "subtitles", "srt", "ass", "ext", "external",
|
||||
"the", "a", "an", "of", "and", "in", "on", "to", "for", "with", "ita",
|
||||
"eng", "english", "italian", "dual", "audio", "forced", "default"}
|
||||
# Canonical title where the library only ever carries a localised name.
|
||||
OVERRIDES = {"28 giorni dopo": "28 Days Later",
|
||||
"28 settimane dopo": "28 Weeks Later"}
|
||||
|
||||
rows = []
|
||||
for line in open(ENUM):
|
||||
size, ext, path = line.rstrip("\n").split("\t")
|
||||
rows.append({"bytes": int(size), "ext": ext.lower(), "path": path})
|
||||
ext_by = {r["path"]: r["ext"] for r in rows}
|
||||
size_by = {r["path"]: r["bytes"] for r in rows}
|
||||
|
||||
|
||||
def rel(p):
|
||||
return p[len(ROOT) + 1:] if p.startswith(ROOT + "/") else p
|
||||
|
||||
|
||||
def parts_of(p):
|
||||
return [q for q in rel(p).split("/") if q]
|
||||
|
||||
|
||||
def is_qb(p):
|
||||
return ".!q" in p.lower()
|
||||
|
||||
|
||||
def disc_root(p, fold=True):
|
||||
"""Absolute root of the disc tree containing p (the MOVIE folder that holds
|
||||
BDMV/ or VIDEO_TS/), else None. `fold` also folds bookkeeping dirs that sit
|
||||
beside the tree (CERTIFICATE/, AACS/) into the same movie folder."""
|
||||
ps = parts_of(p)
|
||||
kinds = DISC_DIRS if fold else {"BDMV", "VIDEO_TS"}
|
||||
for i, q in enumerate(ps):
|
||||
if q.upper() in kinds:
|
||||
return ROOT + "/" + "/".join(ps[:i])
|
||||
return None
|
||||
|
||||
|
||||
def row_name(r):
|
||||
"""The name a disc-tree entry should be identified by: the engine stores the
|
||||
disc's own folder for it, because "BDMV/index.bdmv" is not a title."""
|
||||
return r.get("name_src") or r["path"]
|
||||
|
||||
|
||||
def sanitize(name):
|
||||
name = BAD_CHARS.sub("-", name).replace("\\", "-")
|
||||
name = re.sub(r"\s+", " ", name).strip().strip(". ").strip()
|
||||
return (name[:180] or "untitled")
|
||||
|
||||
|
||||
def ok_year(y):
|
||||
"""True for a year a real film could carry. Plex stores 0 for a file it indexed
|
||||
but never MATCHED, and "0" is a digit string: trusting it named 67 movie
|
||||
folders "Title (0)" — Jellyfin items with no identity, no year and no poster."""
|
||||
return bool(re.fullmatch(r"(18|19|20)\d\d", y or "")) and 1870 <= int(y) <= 2035
|
||||
|
||||
|
||||
WORD_SPLIT = re.compile(r"[^0-9a-z]+")
|
||||
|
||||
|
||||
def words_of(text):
|
||||
"""The matching words of a name — or of a whole path, which is how callers
|
||||
pass a file together with the folders it sits in.
|
||||
|
||||
Split on everything that is not a letter or a digit: eng.norm does not touch
|
||||
"/", so feeding it "…/Psycho (1960)/Extras" left glued tokens like
|
||||
"(1960)/extras" and "(1954)" that no other name could ever agree with, and a
|
||||
film could not be found by the folder that holds it — which is how one box
|
||||
set's featurettes ended up in another film's folder. Both sides of every
|
||||
comparison below go through here. A bare number counts only when it could be
|
||||
a release year, so release junk ("1080", "2160", "2", "7") never poses as
|
||||
agreement between two names.
|
||||
"""
|
||||
out = set()
|
||||
for w in WORD_SPLIT.split(eng.norm(text)):
|
||||
if not w or w in eng.STOP:
|
||||
continue
|
||||
if w.isdigit() and not 1870 <= int(w) <= 2100:
|
||||
continue
|
||||
out.add(w)
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ Plex titles
|
||||
# Plex has matched a large part of the library's media files to real titles
|
||||
# ("Alien", not "Alien.Directors.Cut.1979.1080p.BRrip.x265…"). Where the cache
|
||||
# holds a MATCH, its title wins; the filename parser carries everything else
|
||||
# (extras, disc trees, TV, and whatever Plex never matched).
|
||||
def plex_titles():
|
||||
if not os.path.exists(PLEX):
|
||||
return {}
|
||||
out = {}
|
||||
with open(PLEX, newline="") as fh:
|
||||
for r in csv.DictReader(fh, delimiter="\t"):
|
||||
t, y = r["leaf_title"].strip(), r["leaf_year"].strip()
|
||||
# Only a match is an authority, and a match carries a year: an
|
||||
# unmatched row echoes the filename, which the parser reads better
|
||||
# than a copied release name. The junk filter stays for the matches
|
||||
# whose title still carries the release.
|
||||
if t and ok_year(y) and not re.search(
|
||||
r"2160p|1080p|x26[45]|hevc|bluray|web-?dl", t.lower()):
|
||||
out[r["file"]] = (t, y)
|
||||
return out
|
||||
|
||||
|
||||
PLEX_T = plex_titles()
|
||||
|
||||
# ------------------------------------------------------------- classification
|
||||
erows = eng.load(PROBE)
|
||||
for r in erows:
|
||||
for k, v in OVERRIDES.items():
|
||||
if k in r["title"]:
|
||||
r["title"] = v
|
||||
r["key"] = eng.key_of(r["title"])
|
||||
r["base"] = eng.base_words(r["title"], r["key"])
|
||||
marks = {r["path"] for r in erows if eng.is_watermark(r["path"], r["bytes"])}
|
||||
extras = {r["path"] for r in erows
|
||||
if r["path"] not in marks
|
||||
and eng.is_extra(r["path"], r["title"], r["dur"], ROOT,
|
||||
matched=r["path"] in PLEX_T)}
|
||||
eps = collections.defaultdict(list)
|
||||
for r in erows:
|
||||
if r["path"] in marks | extras:
|
||||
continue
|
||||
if eng.ep_of(r["path"]):
|
||||
eps[eng.key_of(r["title"])].append(r)
|
||||
tv_keys = {k for k, v in eps.items() if len(v) >= 5}
|
||||
tv_set = {r["path"] for k, v in eps.items() if k in tv_keys for r in v}
|
||||
# A folder INSIDE a movie's folder is bonus territory even when its own name is
|
||||
# not bonus vocabulary: "Saboteur (1942)/Exrtas/Saboteur - A Closer Look" is a
|
||||
# misspelled Extras folder, so the engine's folder rule never fired and the
|
||||
# featurette became a movie folder of its own — whose words then out-scored the
|
||||
# real Saboteur and drew that box set's other extras into it. The rule needs no
|
||||
# metadata: the folder holding the longest film owns the shorter video filed
|
||||
# under it, provided that shorter file's own name speaks the film's name — a
|
||||
# whole other film filed there stays a film, exactly as the engine's folder rule
|
||||
# exempts it. A different year in the file's own name IS that other film
|
||||
# speaking, so it vetoes the rule.
|
||||
def longest_of(rows):
|
||||
"""The longest video in each folder: that folder's feature."""
|
||||
out = {}
|
||||
for r in rows:
|
||||
if r["dur"] > 0:
|
||||
d = os.path.dirname(r["path"])
|
||||
if d not in out or r["dur"] > out[d]["dur"]:
|
||||
out[d] = r
|
||||
return out
|
||||
|
||||
|
||||
def not_disc(r):
|
||||
"""A disc tree's innards belong to the tree, never to the folder's film."""
|
||||
return not any(c in DISC_DIRS for c in r["path"].split("/"))
|
||||
|
||||
|
||||
FEATURE_OF = longest_of([r for r in erows if r["path"] not in marks and not_disc(r)])
|
||||
|
||||
|
||||
def under_a_feature(path, dur):
|
||||
"""True when a folder above this file holds a longer film that this file's
|
||||
own name names."""
|
||||
d = os.path.dirname(os.path.dirname(path))
|
||||
while len(d) > len(ROOT):
|
||||
f = FEATURE_OF.get(d)
|
||||
if f:
|
||||
if f["dur"] < dur:
|
||||
return False
|
||||
theirs = (words_of(os.path.basename(f["path"]))
|
||||
| words_of(os.path.basename(d)))
|
||||
if not words_of(os.path.basename(path)) & theirs:
|
||||
return False
|
||||
y1 = re.search(r"(?:18|19|20)\d\d", os.path.basename(path))
|
||||
y2 = re.search(r"(?:18|19|20)\d\d", os.path.basename(f["path"]))
|
||||
return not (y1 and y2 and y1.group() != y2.group())
|
||||
d = os.path.dirname(d)
|
||||
return False
|
||||
|
||||
|
||||
extras |= {r["path"] for r in erows
|
||||
if r["dur"] > 0 and r["path"] not in marks | extras | tv_set
|
||||
and not_disc(r) and under_a_feature(r["path"], r["dur"])}
|
||||
film_rows = [r for r in erows if r["path"] not in marks | extras | tv_set]
|
||||
|
||||
buckets = collections.defaultdict(list)
|
||||
for r in film_rows:
|
||||
buckets[eng.key_of(r["title"])].append(r)
|
||||
|
||||
|
||||
def res_label(r):
|
||||
m = RES_RE.search(r["res"] or "")
|
||||
if m:
|
||||
n, kind = m.group(1).lower(), m.group(2).lower()
|
||||
return "2160p" if n in ("2160", "4k") else f"{n}{kind}"
|
||||
try:
|
||||
w = int((r["res"] or "0x0").split("x")[0])
|
||||
except (ValueError, IndexError):
|
||||
w = 0
|
||||
return "2160p" if w >= 3000 else ("1080p" if w >= 1400 else "SD")
|
||||
|
||||
|
||||
def version_label(r):
|
||||
bits = [res_label(r)]
|
||||
vc = (r["vcodec"] or "").lower()
|
||||
if vc in ("av1", "libaom", "libsvtav1"):
|
||||
bits.append("AV1")
|
||||
elif vc in ("vc1", "wmv3"):
|
||||
bits.append("VC1")
|
||||
if r["bytes"] > 40 * 2**30:
|
||||
bits.append("REMUX")
|
||||
if "imax" in r["path"].lower():
|
||||
bits.append("IMAX")
|
||||
return sanitize(" ".join(bits))
|
||||
|
||||
|
||||
def group_name(items):
|
||||
titles = collections.Counter(r["title"] for r in items)
|
||||
ascii_t = [t for t in titles if t.isascii()]
|
||||
title = (collections.Counter(ascii_t) if ascii_t else titles).most_common(1)[0][0]
|
||||
years = [r["year"] for r in items if r["year"] != "?"] or \
|
||||
[r["soft"] for r in items if r["soft"]]
|
||||
year = collections.Counter(years).most_common(1)[0][0] if years else None
|
||||
return sanitize(f"{title} ({year})" if year else title)
|
||||
|
||||
|
||||
groups = []
|
||||
for key, items in buckets.items():
|
||||
for c in eng.dur_clusters(items):
|
||||
g = {"rows": c, "base": group_name(c)}
|
||||
known = [PLEX_T[r["path"]] for r in c if r["path"] in PLEX_T]
|
||||
if known:
|
||||
# one film, one name: a Plex match for any copy names every copy
|
||||
t, yr = collections.Counter(known).most_common(1)[0][0]
|
||||
g["plex"] = t
|
||||
g["base"] = sanitize(f"{t} ({yr})" if ok_year(yr) else t)
|
||||
groups.append(g)
|
||||
clash = {b for b, n in collections.Counter(g["base"] for g in groups).items() if n > 1}
|
||||
for g in groups:
|
||||
if g["base"] in clash:
|
||||
# two genuinely different films share title+year: separate folders, and
|
||||
# say so in the name, because Jellyfin can only match one item per folder
|
||||
g["base"] = sanitize(g["base"] + " - " + version_label(g["rows"][0]))
|
||||
g["flag"] = "same title and year as another film"
|
||||
|
||||
moves, mkdirs, problems, readmes = [], set(), [], []
|
||||
# --------------------------------------------------------------- unnamed groups
|
||||
# A movie folder has to READ as a title. The engine's runtime veto says any file
|
||||
# over 45 minutes that is not named like an extra is a film whatever it is called
|
||||
# — true for Jellyfin, but a 118-minute documentary named "Extra 1" then produced
|
||||
# a movie folder named "1", and folders named "1", "2", "3" are Jellyfin items
|
||||
# with no identity whose numeric keys went on to capture every subtitle named
|
||||
# "3_Eng.srt" in the library. A year proves a dated feature ("300 (2006)" is a
|
||||
# real film), so the gate is: a year, or a word that is not release noise. With
|
||||
# neither, the rows are bonus material until a human names them.
|
||||
def named(g):
|
||||
if re.search(r"\(\d{4}\)$", g["base"]):
|
||||
return True
|
||||
return any(w.isalpha() and len(w) >= 2 for w in eng.name_words(g["base"]))
|
||||
|
||||
|
||||
unnamed = [g for g in groups if not named(g)]
|
||||
groups = [g for g in groups if named(g)]
|
||||
for g in unnamed:
|
||||
for r in g["rows"]:
|
||||
extras.add(r["path"])
|
||||
problems.append(
|
||||
f"UNNAMED film {rel(r['path'])} parses to {g['base']!r} with no year: "
|
||||
f"bonus pile, not a folder — give the file a real name and re-plan")
|
||||
|
||||
|
||||
group_of = {}
|
||||
for g in groups:
|
||||
g["dir"] = f"{ROOT}/{g['base']}"
|
||||
for r in g["rows"]:
|
||||
group_of[r["path"]] = g
|
||||
dr = disc_root(r["path"])
|
||||
if dr:
|
||||
group_of[dr] = g
|
||||
by_path = {r["path"]: r for r in erows}
|
||||
|
||||
# -------------------------------------------------------------- bookkeeping
|
||||
claimed = {}
|
||||
existing = set(rows_map := {r["path"] for r in rows})
|
||||
|
||||
|
||||
at_target, booked_src = set(), set()
|
||||
|
||||
|
||||
def book(src, dst, kind):
|
||||
if src == dst:
|
||||
at_target.add(src)
|
||||
booked_src.add(src)
|
||||
return
|
||||
if os.environ.get("JF_TRACE") and (src in booked_src or dst.lower() in claimed):
|
||||
why = "rebook" if src in booked_src else "collision"
|
||||
problems.append(f"TRACE {why} kind={kind} src={rel(src)} dst={rel(dst)}")
|
||||
if src in booked_src: # one source may only ever move once
|
||||
problems.append(f"RE-BOOK {rel(src)} -> {rel(dst)} (already planned)")
|
||||
return
|
||||
if dst.lower() in claimed:
|
||||
problems.append(f"COLLISION {rel(dst)}\n from {rel(src)}\n and {rel(claimed[dst.lower()])}")
|
||||
return
|
||||
claimed[dst.lower()] = src
|
||||
booked_src.add(src)
|
||||
comp = dst.split("/")
|
||||
for i in range(1, len(comp)):
|
||||
mkdirs.add("/".join(comp[:i]))
|
||||
moves.append((src, dst, kind))
|
||||
|
||||
|
||||
def uniq_dir(d, stem, ext, hint=""):
|
||||
"""A destination that no other source already took."""
|
||||
cand = f"{d}/{stem}{ext}"
|
||||
if cand.lower() not in claimed:
|
||||
return cand
|
||||
n = 2
|
||||
while True:
|
||||
cand = f"{d}/{stem} {n}{ext}"
|
||||
if cand.lower() not in claimed:
|
||||
return cand
|
||||
n += 1
|
||||
|
||||
|
||||
def media_dir(p):
|
||||
dr = disc_root(p)
|
||||
return os.path.dirname(dr) if dr else os.path.dirname(p)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ films
|
||||
# film / disc moves: the largest copy carries the folder's own name, the rest
|
||||
# become Jellyfin versions; a disc tree moves as a whole into the movie folder
|
||||
|
||||
|
||||
def vext(p):
|
||||
return ext_by.get(p) or os.path.splitext(p)[1].lower()
|
||||
disc_children = collections.defaultdict(list)
|
||||
for q in rows:
|
||||
dr = disc_root(q["path"])
|
||||
if dr:
|
||||
disc_children[dr].append(q["path"])
|
||||
for g in groups:
|
||||
primary_taken = False
|
||||
disc_kinds = set()
|
||||
for r in sorted(g["rows"], key=lambda r: -r["bytes"]):
|
||||
dr = disc_root(r["path"])
|
||||
if dr:
|
||||
inner = os.path.basename(dr).upper()
|
||||
if inner in disc_kinds:
|
||||
problems.append(
|
||||
f"DISC-VERSION {g['base']}: extra disc tree {rel(dr)} cannot "
|
||||
f"share the folder — discs have no multiple versions")
|
||||
continue
|
||||
disc_kinds.add(inner)
|
||||
for src in disc_children[dr]:
|
||||
tail = os.path.relpath(src, dr)
|
||||
fold = disc_root(src, fold=False)
|
||||
if fold != dr and os.path.dirname(src) != dr:
|
||||
tail = os.path.basename(dr) + "/" + tail
|
||||
book(src, f"{g['dir']}/{tail}", "disc-file")
|
||||
continue
|
||||
label = "" if not primary_taken else f" - {version_label(r)}"
|
||||
dst = f"{g['dir']}/{g['base']}{label}{vext(r['path'])}"
|
||||
n = 2
|
||||
while dst.lower() in claimed:
|
||||
dst = f"{g['dir']}/{g['base']}{label} {n}{vext(r['path'])}"
|
||||
n += 1
|
||||
r["dst"] = dst
|
||||
primary_taken = True
|
||||
book(r["path"], dst, "film")
|
||||
g["primary"] = primary_taken
|
||||
|
||||
# ------------------------------------------------------------------ extras
|
||||
EXTRA_KIND = [("trailers", r"trailer|teaser|preview|promo|tv spot|advert"),
|
||||
("deleted scenes", r"deleted|alt scene|gag reel|blooper|outtake"),
|
||||
("interviews", r"interview|cast and crew|session piece|commentary"),
|
||||
("featurettes", r"featurette|behind the scenes|making of|documentary"
|
||||
r"|heritage|archival|video archive|music video"),
|
||||
("samples", r"sample"),
|
||||
("shorts", r"\bshort\b|short film"),
|
||||
("clips", r"\bclip\b|scene selection|montage|epilogue|isolated score"),
|
||||
("other", r"menu|logo|gallery|still|trivia|storyboard|screen test"
|
||||
r"|audition|test footage|watermark|workprint|poster"
|
||||
r"|picture in picture|singalong|karaoke|original score")]
|
||||
THEME_RE = re.compile(r"theme|opening song|ending song|soundtrack|score"
|
||||
r"|\bost\b|music|song", re.I)
|
||||
BACKDROP_RE = re.compile(r"backdrop|intro", re.I)
|
||||
|
||||
|
||||
def extras_kind(path):
|
||||
low = path.lower()
|
||||
if THEME_RE.search(low):
|
||||
return "theme-music"
|
||||
if BACKDROP_RE.search(low):
|
||||
return "backdrops"
|
||||
for name, rx in EXTRA_KIND:
|
||||
if re.search(rx, low):
|
||||
return name
|
||||
return "extras"
|
||||
|
||||
|
||||
def attachable_title(path):
|
||||
t, _y = eng.title_of(path)
|
||||
return eng.key_of(t) if t and not eng.is_watermark(path, size_by.get(path, 0)) else None
|
||||
|
||||
|
||||
for g in groups:
|
||||
g["tops"] = {parts_of(r["path"])[0] for r in g["rows"]}
|
||||
top_groups = collections.defaultdict(list)
|
||||
for g in groups:
|
||||
for t in g["tops"]:
|
||||
top_groups[t].append(g)
|
||||
group_by_key = collections.defaultdict(list)
|
||||
for g in groups:
|
||||
for r in g["rows"]:
|
||||
group_by_key[eng.key_of(r["title"])].append(g)
|
||||
group_by_dir = {}
|
||||
for g in groups:
|
||||
for r in g["rows"]:
|
||||
group_by_dir.setdefault(media_dir(r["path"]), g)
|
||||
dr = disc_root(r["path"])
|
||||
if dr:
|
||||
group_by_dir.setdefault(os.path.dirname(dr), g)
|
||||
|
||||
|
||||
def group_words(g):
|
||||
"""The words that identify a film group: its titles and the names of the
|
||||
folders its files live in."""
|
||||
if "words" not in g:
|
||||
w = set()
|
||||
if g.get("plex"):
|
||||
w |= words_of(g["plex"])
|
||||
for r in g["rows"]:
|
||||
w |= words_of(r["title"])
|
||||
w |= words_of(os.path.basename(media_dir(r["path"])))
|
||||
g["words"] = w
|
||||
return g["words"]
|
||||
|
||||
|
||||
def score_words(text, groups):
|
||||
"""Ranks film groups by how many of their words a path carries."""
|
||||
words = words_of(text)
|
||||
out = [(len(words & group_words(g)), g) for g in groups]
|
||||
out.sort(key=lambda x: -x[0])
|
||||
return out
|
||||
|
||||
|
||||
def group_for_orphan(path):
|
||||
"""Which movie folder an extra/sidecar belongs to. A collection folder holding
|
||||
several films must be resolved by WORDS, never by position: taking the first
|
||||
film of the folder attached every loose subtitle in a box set to whichever
|
||||
movie happened to sort first."""
|
||||
stem = os.path.splitext(os.path.basename(path))[0]
|
||||
key = attachable_title(path)
|
||||
if key and group_by_key.get(key):
|
||||
cands = group_by_key[key]
|
||||
top = parts_of(path)[0] if len(parts_of(path)) > 1 else ""
|
||||
same_top = [g for g in cands if any(parts_of(r["path"])[0] == top
|
||||
for r in g["rows"] if len(parts_of(r["path"])) > 1)]
|
||||
pool = same_top or cands
|
||||
return pool[0] if len(pool) == 1 else score_words(stem, pool)[0][1]
|
||||
gtop = parts_of(path)[0] if len(parts_of(path)) > 1 else ""
|
||||
pool = top_groups.get(gtop, [])
|
||||
if len(pool) > 1: # a loose collection folder (box set, bonus
|
||||
n, g = score_words(stem + " " + os.path.dirname(path), pool)[0]
|
||||
return g if n >= 2 else None # album) is resolved by words, not position
|
||||
if len(pool) == 1:
|
||||
return pool[0]
|
||||
d = os.path.dirname(path)
|
||||
while d and d != ROOT:
|
||||
if d in group_by_dir:
|
||||
return group_by_dir[d]
|
||||
d = os.path.dirname(d)
|
||||
return None
|
||||
|
||||
|
||||
tv_tops = {parts_of(p)[0] for p in tv_set}
|
||||
def film_folder_for(path):
|
||||
"""A movie folder for a lone feature that has no group: derived from its own
|
||||
title when a year proves it, else None (goes to the review pile)."""
|
||||
t, y = eng.title_of(path)
|
||||
if not t or y == "?":
|
||||
return None
|
||||
d = uniq_dir(ROOT, sanitize(f"{t} ({y})"), "", "")
|
||||
mkdirs.add(d)
|
||||
return d
|
||||
|
||||
|
||||
for p in sorted(extras):
|
||||
if disc_root(p):
|
||||
continue
|
||||
if len(parts_of(p)) > 1 and parts_of(p)[0] in tv_tops:
|
||||
continue # a special inside a TV set: staged with it
|
||||
g = group_for_orphan(p)
|
||||
ext = ext_by.get(p, os.path.splitext(p)[1].lower())
|
||||
stem = os.path.splitext(os.path.basename(p))[0]
|
||||
if g is None:
|
||||
solo = film_folder_for(p)
|
||||
if solo:
|
||||
dst = f"{solo}/{sanitize(stem)}{ext}"
|
||||
problems.append(f"REVIEW lone feature {rel(p)}: gets its own folder "
|
||||
f"{os.path.basename(solo)}")
|
||||
else:
|
||||
dst = f"{ROOT}/_unsorted bonus/{extras_kind(p)}/{sanitize(stem)}{ext}"
|
||||
problems.append(f"NO-FILM extra {rel(p)}")
|
||||
else:
|
||||
dst = f"{g['dir']}/{extras_kind(p)}/{sanitize(stem)}{ext}"
|
||||
book(p, uniq_dir(os.path.dirname(dst), os.path.basename(dst)[:-(len(ext) or 1)],
|
||||
ext, os.path.dirname(p)), "extra")
|
||||
|
||||
# Disc trees the engine never saw (a MakeMKV scratch folder, an AACS remnant):
|
||||
# they belong to the film of the folder they sit in, kept as a named subtree.
|
||||
for q in rows:
|
||||
pth = q["path"]
|
||||
dr = disc_root(pth)
|
||||
if not dr or pth in by_path or pth in booked_src:
|
||||
continue
|
||||
g = group_by_dir.get(dr) or (top_groups[parts_of(pth)[0]][0]
|
||||
if len(top_groups.get(parts_of(pth)[0], [])) == 1 else None)
|
||||
if g is None:
|
||||
continue
|
||||
inner = os.path.basename(dr)
|
||||
problems.append(f"REVIEW disc remnant {rel(dr)} moves inside {g['base']} "
|
||||
f"(a second master: compare before keeping both)")
|
||||
for src in disc_children[dr]:
|
||||
if src not in booked_src:
|
||||
book(src, f"{g['dir']}/{inner}/{os.path.relpath(src, dr)}", "disc-file")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- sidecars
|
||||
media_by_dir = collections.defaultdict(list)
|
||||
for r in erows:
|
||||
if r["path"] in marks:
|
||||
continue
|
||||
dst = r.get("dst")
|
||||
if dst:
|
||||
media_by_dir[media_dir(r["path"])].append(r)
|
||||
|
||||
|
||||
|
||||
def sidecar_target(path, e):
|
||||
d = os.path.dirname(path)
|
||||
stem = os.path.splitext(os.path.basename(path))[0]
|
||||
if d == ROOT:
|
||||
# A file loose in the library ROOT names no film. media_by_dir[ROOT] is the
|
||||
# set of root-level MOVIE files, and scoring a stray name against it
|
||||
# attached every root-level leftover to whichever root-level film was
|
||||
# largest — the plan sent ".DS_Store" into "Titanic (1997)/other/" that
|
||||
# way. A root-level stray is nobody's sidecar: it goes to the review pile.
|
||||
return None, None
|
||||
cands = media_by_dir.get(d, [])
|
||||
if not cands:
|
||||
nd = os.path.dirname(d) # a subs/ or Extras/ folder beside the media
|
||||
while nd and nd != ROOT and not media_by_dir.get(nd):
|
||||
nd = os.path.dirname(nd)
|
||||
if media_by_dir.get(nd):
|
||||
if nd != d: # a NAMED subfolder (Subs/, Extras/, the
|
||||
g = group_for_orphan(path) # box set's own root) carries evidence
|
||||
dirs = {os.path.dirname(r.get("dst", "")) for r in media_by_dir[nd]}
|
||||
if g is None or g["dir"] not in dirs:
|
||||
# ...but only a STRONG word match outranks the folder the file
|
||||
# sits in. A weak one is an accident: "3_Eng.srt" scored 1
|
||||
# against a junk group named "3" and escaped its own movie's
|
||||
# Subs/ folder, while "03 xXx 3 Return Of Xander Cage" scores 4
|
||||
# and genuinely belongs to the other film of the set.
|
||||
n = score_words(stem, [g])[0][0] if g else 0
|
||||
if n >= 2:
|
||||
return (g, None)
|
||||
cands = media_by_dir[nd]
|
||||
if not cands:
|
||||
g = group_for_orphan(path)
|
||||
return (g, None) if g else (None, None)
|
||||
if len(cands) > 1:
|
||||
w = [t for t in eng.norm(stem).split() if t not in NOISE]
|
||||
cands = sorted(cands, key=lambda r: (
|
||||
-len([t for t in eng.norm(os.path.basename(r["path"])).split() if t in w]),
|
||||
-r["bytes"]))
|
||||
return (None, cands[0])
|
||||
|
||||
|
||||
for q in rows:
|
||||
p, e = q["path"], q["ext"]
|
||||
if p in by_path or p in extras or p in marks or is_qb(p) or disc_root(p):
|
||||
continue
|
||||
if os.path.basename(p).startswith("._"):
|
||||
# macOS AppleDouble resource forks: a few hundred bytes carrying a name
|
||||
# COPIED FROM ANOTHER FILM. Word-matching them to "their" film scattered
|
||||
# dead files through four movie folders (and they are video extensions in
|
||||
# the AppleDouble case, so they also skipped every other filter). They
|
||||
# claim nothing: one pile, and they stay in it — nothing is deleted here.
|
||||
book(p, uniq_dir(f"{ROOT}/_macos sidecars", sanitize(os.path.basename(p))[:120],
|
||||
"", os.path.dirname(p)), "junk")
|
||||
problems.append(f"JUNK AppleDouble sidecar {rel(p)} (macOS resource fork)")
|
||||
continue
|
||||
if e == ".iso": # a DVD image: Jellyfin plays it as a movie file
|
||||
g = group_for_orphan(p)
|
||||
stem = os.path.splitext(os.path.basename(p))[0]
|
||||
if g is not None:
|
||||
book(p, uniq_dir(g["dir"], g["base"], e, stem) if g.get("primary") is not None
|
||||
else f"{g['dir']}/{g['base']}{e}", "iso")
|
||||
else:
|
||||
solo = film_folder_for(p)
|
||||
if solo:
|
||||
book(p, f"{solo}/{sanitize(stem)}.iso", "iso")
|
||||
problems.append(f"REVIEW disc image {rel(p)}: DVD rip, alone in its "
|
||||
f"folder -> {os.path.basename(solo)}")
|
||||
else:
|
||||
book(p, f"{ROOT}/_unsorted files/{sanitize(os.path.basename(p))}", "orphan")
|
||||
problems.append(f"NO-FILM disc image {rel(p)}")
|
||||
continue
|
||||
if e in VIDEO_EXT:
|
||||
# A video the engine never probed (unreadable header) and not an AppleDouble
|
||||
# remnant: the plan does not guess at it, so it stays where it is.
|
||||
continue
|
||||
stem = os.path.splitext(os.path.basename(p))[0]
|
||||
g, media = sidecar_target(p, e)
|
||||
if media is not None:
|
||||
base = os.path.splitext(os.path.basename(media["dst"]))[0]
|
||||
d = os.path.dirname(media["dst"])
|
||||
elif g is not None:
|
||||
base, d = g["base"], g["dir"]
|
||||
else:
|
||||
# Nothing names this file's film. NEVER scatter it: an archive set or an
|
||||
# album tree must move as the unit it is, so the whole loose folder goes
|
||||
# to the review pile and a film folder's own leftovers stay with it.
|
||||
ps = parts_of(p)
|
||||
if len(ps) < 2:
|
||||
dst = f"{ROOT}/_unsorted files/{sanitize(stem)}{e}"
|
||||
elif len(top_groups.get(ps[0], [])):
|
||||
dst = f"{top_groups[ps[0]][0]['dir']}/other/{'/'.join(sanitize(c) for c in ps[1:])}"
|
||||
else:
|
||||
dst = f"{ROOT}/_unsorted files/{'/'.join(sanitize(c) for c in ps)}"
|
||||
book(dst=dst, src=p, kind="orphan")
|
||||
problems.append(f"NO-OWNER sidecar {rel(p)}")
|
||||
continue
|
||||
if e in IMG_EXT:
|
||||
low = eng.norm(stem)
|
||||
word = next((w for w in IMG_PRIMARY if re.search(rf"\b{w}\b", low)), None)
|
||||
kind = "poster" if word else next(
|
||||
(IMG_OTHER[w] for w in IMG_OTHER if re.search(rf"\b{w}\b", low)), None)
|
||||
if re.search(r"\btheme\b", low):
|
||||
dst = f"{d}/theme{e}"
|
||||
else:
|
||||
dst = f"{d}/{base}-{kind or 'art'}{e}" if kind else f"{d}/{sanitize(stem)}{e}"
|
||||
book(p, uniq_dir(os.path.dirname(dst), os.path.basename(dst)[:-(len(e) or 1)],
|
||||
e, os.path.dirname(p)), "image")
|
||||
elif e in NFO_EXT:
|
||||
book(p, uniq_dir(d, base, ".nfo", os.path.dirname(p)), "nfo")
|
||||
elif e in SUB_EXT | AUDIO_EXT:
|
||||
toks = [t for t in re.split(r"[._\-\s]+", stem) if t]
|
||||
langs = [t.lower() for t in toks if t.lower() in LANGS]
|
||||
flags = [t.lower() for t in toks if t.lower() in FLAGS]
|
||||
keep = [sanitize(t) for t in toks
|
||||
if t.lower() not in langs + flags + list(NOISE)
|
||||
and not re.fullmatch(r"\d{3,4}p?|cd\d+|\d{4}|\d+ch|\d+.\d", t.lower())]
|
||||
bits = [base] + ([sanitize(" ".join(keep[:4]))] if keep else []) + langs + flags
|
||||
if e in SUB_EXT:
|
||||
# subtitles: <movie>/srt[/language]
|
||||
d = f"{d}/srt"
|
||||
pd = os.path.basename(os.path.dirname(p))
|
||||
if SUB_KEEP_RE.match(pd) and not SUB_DIR_OK.match(pd):
|
||||
d = f"{d}/{sanitize(pd)}" # a real language folder survives
|
||||
elif e in AUDIO_EXT and AUDIO_DIR_RE.search(os.path.dirname(p)):
|
||||
d = f"{d}/theme-music" # bonus audio: one canonical folder
|
||||
pd = os.path.basename(os.path.dirname(p))
|
||||
if not MUSIC_TOP_RE.search(pd): # an album folder keeps its name
|
||||
d = f"{d}/{sanitize(pd)}"
|
||||
else:
|
||||
pd = os.path.basename(os.path.dirname(p))
|
||||
if not AUDIO_DIR_OK.match(pd):
|
||||
d = f"{d}/{sanitize(pd)}"
|
||||
stem2 = ".".join(bits) or sanitize(os.path.basename(p))
|
||||
book(p, uniq_dir(d, stem2, e, os.path.basename(p)), "track")
|
||||
else:
|
||||
book(p, uniq_dir(f"{d}/other", sanitize(os.path.basename(p))[:120], e,
|
||||
os.path.dirname(p)), "misc")
|
||||
|
||||
readmes.append(f"README in {ROOT}: this library was relaid out into Jellyfin's "
|
||||
"one-folder-per-movie layout; `_release-group watermarks/`, "
|
||||
"`_unsorted files/` and `_unsorted bonus/` hold files no film claims.")
|
||||
for key in tv_keys:
|
||||
nm = "_TV needs a TV library - " + collections.Counter(
|
||||
r["title"] for r in eps[key]).most_common(1)[0][0]
|
||||
readmes.append(f"README in {ROOT}/{sanitize(nm)}: these are TV episodes found in "
|
||||
"the MOVIES library; move the folder to a TV library source, else "
|
||||
"Jellyfin groups them wrongly as one movie.")
|
||||
|
||||
# --------------------------------------------------------------------- TV
|
||||
for key in tv_keys:
|
||||
items = eps[key]
|
||||
title = collections.Counter(r["title"] for r in items).most_common(1)[0][0]
|
||||
year = collections.Counter(r["year"] for r in items if r["year"] != "?")
|
||||
yr = year.most_common(1)[0][0] if year else None
|
||||
base = f"_TV needs a TV library - {sanitize(title)}" + (f" ({yr})" if yr else "")
|
||||
tops = {parts_of(r["path"])[0] for r in items}
|
||||
members = sorted([r["path"] for r in items]
|
||||
+ [q for q in extras | marks if len(parts_of(q)) > 1
|
||||
and parts_of(q)[0] in tops],
|
||||
key=lambda q: q)
|
||||
for src in members:
|
||||
rel_parts = rel(src).split("/")
|
||||
tail = "/".join(sanitize(c) for c in rel_parts[:-1] + [rel_parts[-1]])
|
||||
book(src, f"{ROOT}/{base}/{tail}", "tv")
|
||||
for p in sorted(marks):
|
||||
stem, ext = os.path.splitext(os.path.basename(p))
|
||||
book(p, uniq_dir(f"{ROOT}/_release-group watermarks", sanitize(stem)[:120], ext,
|
||||
os.path.dirname(p)), "watermark")
|
||||
for q in rows:
|
||||
if is_qb(q["path"]):
|
||||
problems.append(f"SKIPPED qBittorrent partial {rel(q['path'])}")
|
||||
book(q["path"], q["path"], "qb-partial") # stays: resume data
|
||||
|
||||
# --------------------------------------------------------------- debug hook
|
||||
if os.environ.get("JF_ISO"):
|
||||
x = ROOT + "/High Anxiety (Mel Brooks) [1977] nordic subs DVDrip/High Anxiety [1977].iso"
|
||||
print("attachable", attachable_title(x), "| dir", os.path.dirname(x) in group_by_dir,
|
||||
"| g", group_for_orphan(x) and group_for_orphan(x)["base"])
|
||||
raise SystemExit
|
||||
if os.environ.get("JF_DEBUG"):
|
||||
import json
|
||||
unp = sorted({r["path"] for r in rows} - {m[0] for m in moves} - at_target)
|
||||
print("at_target", len(at_target))
|
||||
print("kinds-unplanned", collections.Counter(os.path.splitext(u)[1] for u in unp).most_common(8))
|
||||
for up in unp[:20]:
|
||||
d = os.path.dirname(up)
|
||||
print("UNP", up[:90], "| mediadir?", d in media_by_dir, "| by_path", up in by_path,
|
||||
"| extras", up in extras, "| discroot", disc_root(up), "| ext", os.path.splitext(up)[1])
|
||||
|
||||
# -------------------------------------------------------------- validation
|
||||
srcs = [m[0] for m in moves]
|
||||
dup_src = {s for s, n in collections.Counter(srcs).items() if n > 1}
|
||||
for s in dup_src:
|
||||
problems.append(f"DOUBLE-CLAIM source {rel(s)}")
|
||||
all_src = set(srcs)
|
||||
untouched = {r["path"] for r in rows} - all_src
|
||||
bad_names, into_alive, lost = [], [], []
|
||||
for src, dst, kind in moves:
|
||||
for comp in dst[len(ROOT) + 1:].split("/"):
|
||||
if (BAD_CHARS.search(comp) or comp != comp.strip(". ") or not comp
|
||||
or len(comp) > 200):
|
||||
bad_names.append(f"{kind}: {comp!r} in {rel(dst)}")
|
||||
if dst in untouched: # something staying put is there
|
||||
into_alive.append(f"{rel(src)} -> {rel(dst)} (occupied by a file that stays)")
|
||||
if dst.lower() in {s.lower() for s in untouched}:
|
||||
into_alive.append(f"{rel(src)} -> {rel(dst)} (case-collides with a survivor)")
|
||||
if dst.startswith(src + "/") and src in all_src:
|
||||
lost.append(f"{rel(src)} moves into its own subtree")
|
||||
# every accounted-for source must exist exactly once in the plan
|
||||
covered = {r["path"] for r in rows}
|
||||
not_planned = sorted(covered - all_src - at_target)
|
||||
|
||||
with open(PLAN, "w") as out:
|
||||
out.write("action\tsrc\tdst\tkind\n")
|
||||
for d in sorted(mkdirs, key=lambda q: (q.count("/"), q)):
|
||||
if d.startswith(ROOT + "/") and d != ROOT:
|
||||
out.write(f"mkdir\t\t{d}\tdir\n")
|
||||
for p in sorted(booked_src - {m[0] for m in moves}):
|
||||
out.write(f"keep\t{p}\t{p}\n")
|
||||
for src, dst, kind in sorted(moves, key=lambda m: m[0]):
|
||||
out.write(f"move\t{src}\t{dst}\t{kind}\n")
|
||||
|
||||
kinds = collections.Counter(k for _s, _d, k in moves)
|
||||
films_n = sum(1 for _s, _d, k in moves if k == "film")
|
||||
folders = len({os.path.dirname(d) for _s, d, k in moves if k in ("film", "disc-file")})
|
||||
with open(EXCEPTS, "w") as out:
|
||||
out.write(f"groups={len(groups)} moves={len(moves)} folders={folders}\n")
|
||||
out.write(f"kinds={dict(kinds)}\n")
|
||||
out.write(f"at-target-already={len(at_target)} unplanned={len(not_planned)}\n")
|
||||
out.write("\n=== WHAT THE EXECUTOR MUST ALSO WRITE ===\n")
|
||||
out.write("\n".join(sorted(set(readmes))) + "\n")
|
||||
out.write("\n=== PROBLEMS ===\n")
|
||||
out.write("\n".join(sorted(set(problems))) + "\n")
|
||||
out.write("\n=== BAD NAMES ===\n" + "\n".join(sorted(set(bad_names))) + "\n")
|
||||
out.write("\n=== TARGETS OCCUPIED BY SURVIVORS ===\n"
|
||||
+ "\n".join(sorted(set(into_alive))) + "\n")
|
||||
out.write("\n=== LEFTOVER SOURCES (first 60) ===\n"
|
||||
+ "\n".join(rel(p) for p in not_planned[:60]) + "\n")
|
||||
print(f"groups={len(groups)} folders={folders} moves={len(moves)} "
|
||||
f"mkdirs={len(mkdirs)} film-files={films_n}")
|
||||
print(f"kinds={dict(kinds)}")
|
||||
print(f"problems={len(set(problems))} bad-names={len(set(bad_names))} "
|
||||
f"occupied={len(set(into_alive))} double-claim={len(dup_src)}")
|
||||
print(f"left-unmoved={len(not_planned)}")
|
||||
print(f"tv-keys={len(tv_keys)} extras={len(extras)} marks={len(marks)} "
|
||||
f"disc-trees={len(disc_children)}")
|
||||
Executable
+563
@@ -0,0 +1,563 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""The fleet's release-filename title parser: the grouping authority the relayout
|
||||
plan uses for every file the media server never matched.
|
||||
|
||||
report: media-titles.py report [--probed TSV] [--out TSV]
|
||||
the similar-copies census (DUP / REVIEW / EP-DUP groups) over the
|
||||
ffprobe census scripts/media-census.py writes.
|
||||
parse: media-titles.py parse NAME [NAME…]
|
||||
what title_of() makes of a name — the way to check a naming accident
|
||||
before it merges two films into one folder.
|
||||
|
||||
scripts/media-plan.py imports this file; over there the paths are arguments.
|
||||
|
||||
Method — three independent signals, so a naming accident cannot merge two films:
|
||||
* franchise KEY : first two significant title tokens. Stable across parts of a
|
||||
collection ("resident evil", "jump street") but NOT film-specific — it only
|
||||
selects candidates.
|
||||
* DURATION : the identity check. Same master encodes differ by at most a
|
||||
couple of seconds; different films / cuts do not collide.
|
||||
* SIZE SPREAD + YEAR: decide the tier. Two files same size AND same duration
|
||||
are byte-twins (the hash pass confirms); same duration but different sizes are
|
||||
two different releases of one film (deletable redundancy).
|
||||
|
||||
TV episodes (SxxEyy in the name) are grouped separately: their uniform runtime
|
||||
would otherwise collapse a whole series into one "duplicate" group.
|
||||
"""
|
||||
import argparse
|
||||
import collections, os, re, sys, unicodedata
|
||||
|
||||
# Library root: the folder directly under this one names the film, and every
|
||||
# folder below it is structure (Extras/, Featurettes/, Disc 1/, BDMV/). Tests pass
|
||||
# their own; nothing here reads media, only the census TSVs about it.
|
||||
ROOT = os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies")
|
||||
|
||||
# Release-tag vocabulary: never part of a title, so the title stops here.
|
||||
JUNK = {"5.1", "7.1", "2.0", "1.0", "6.1", "4.0", "extras"}
|
||||
JUNK_RE = re.compile(
|
||||
r"^("
|
||||
r"\d{3,5}p|\d{1,2}k|uhd|fhd|imax|x ?26[45]|hevc|h ?26[45]|avc|av1|x ?av1|vp[89]"
|
||||
r"|divx|xvid|mpeg-?2?|\d{1,2}-?bit|hdr ?10\+?|hdr|dolby ?vision|dvh?|sdr|hybrid"
|
||||
r"|re-?mux|bd-?re-?mux|blu-?ray|bdrip|brrip|bdmux|bd|web-?(rip|dl)?|hdtv|hdrip"
|
||||
r"|dvd-?(rip|r|scr)?|hdcam|telesync|screener|ppv|re-?encode|extras?"
|
||||
r"|ddp? ?\d(\.\d)?|dd\d(\.\d)?|dts-?hd(\.ma)?|dts|truehd|atmos|e-?ac-?3?|ac-?3"
|
||||
r"|aac\d?(\.\d)?|lpcm|flac|opus|mp3|pcm|\d\.\d(\.\d)?|5 ?1|7 ?1|2 ?0|1 ?0|6 ?1|4 ?0"
|
||||
r"|subs?-?(it|en|eng|ita)?|subbed|dual(-?audio)?|multi|dub(bed)?"
|
||||
r"|ita|eng|en|it|de|fr|es|pl|jp|jap|lat|esp|ger|fre|rus|zho|hin|tam|tel"
|
||||
r"|repack|proper|internal|limited|unrated|extended|director'?s|cut|remaster(ed)?|uncut"
|
||||
r"|special|edition|version|crc32|sha1|sfv|par2|nfo|nzb|mux|complete|unpack"
|
||||
r"|certified|scene|p2p|part\d*|cd\d+|disc\d+|disk\d+|vob|idx|srt|ass|ssa|sup"
|
||||
r"|m3u|cue|log|txt|mkv|mp4|avi|imdb\d[\d.]*|yts(\.\w+)?|rarbg|collection"
|
||||
r")$", re.I)
|
||||
RES_RE = re.compile(r"^\d{3,5}p$|^\d{1,2}k$|^\d+x\d+$", re.I)
|
||||
YEAR_RE = re.compile(r"^(19|20)\d{2}$")
|
||||
SEP_RE = re.compile(r"[._\s\-\[\](){}+!~,;:&@#/*'\"`´|]+")
|
||||
IDX_RE = re.compile(r"^\d{1,3}[a-z]?$")
|
||||
WORD_RE = re.compile(r"[A-Za-z0-9']+$")
|
||||
EP_RE = re.compile(r"(?i)(?<![a-z0-9])(s(?:eason[\s._-]*)?\d{1,2}[\s._-]*e(?:p(?:isode)?[\s._-]*)?"
|
||||
r"\d{1,3}|s\d{1,2}\s?e\d{1,3}|\d{1,2}x\d{2,3}|part[\s._-]*\d{1,2})")
|
||||
STOP = {"a", "an", "the", "il", "la", "le", "i", "un", "une", "el", "los", "las", "de",
|
||||
"del", "du", "di", "e", "ed", "y", "and", "et", "for", "of", "to", "in", "on",
|
||||
"vs", "avec", "con"}
|
||||
|
||||
|
||||
def norm(s):
|
||||
s = unicodedata.normalize("NFKD", s)
|
||||
return "".join(c for c in s if not unicodedata.combining(c)).lower().strip()
|
||||
|
||||
|
||||
def fold_sep(s):
|
||||
"""norm(), plus: every character that is left over is not a letter, digit or
|
||||
apostrophe, so it is a SEPARATOR whatever codepoint it hides behind — not part
|
||||
of the word. Release groups file titles with lookalikes for punctuation the
|
||||
filesystem forbids: "V ∕ H ∕ S ∕ 2" carries U+2215 DIVISION SLASH, "S꞉ Viral"
|
||||
U+A789, and en dashes hide in "Star Wars – A New Hope". NFKD does NOT fold
|
||||
those (no decomposition mapping), and the scrubber inside title_of DELETES
|
||||
what it does not recognise, so five different V/H/S films squashed into the one
|
||||
title "vhs2" and would have shared one folder. Deleting is the wrong operation:
|
||||
they separate. Accents are gone by then — norm() removed the combining marks —
|
||||
so "Amélie" stays "amelie" and never splits into "ame lie"."""
|
||||
return " ".join("".join(c if c.isalnum() or c == "'" else " "
|
||||
for c in norm(s)).split())
|
||||
|
||||
def title_of(path):
|
||||
"""(title, year) from the basename.
|
||||
|
||||
Two passes. (1) Tokenize; every 4-digit year becomes a sentinel in place.
|
||||
(2) Find the boundary: the first release-noise token (resolution / codec /
|
||||
source / audio / language / group vocabulary) ends the title, so trailing
|
||||
tags can never leak in. (3) Collect up to that boundary: leading index
|
||||
numbers ("03."), leading year sentinels ("1940 - Fantasia") and leading noise
|
||||
are skipped while nothing is collected; once a real word appears a sentinel
|
||||
CLOSES the title. A title sitting BEFORE the year sentinel ("300.2006.…" or
|
||||
"12. Monkeys. 1995") is a numeric title and is prepended to what follows.
|
||||
"""
|
||||
# fold_sep BEFORE tokenizing, so a lookalike separator separates (see its
|
||||
# docstring): the tokens below are then plain alnum/apostrophe words.
|
||||
name = fold_sep(os.path.splitext(os.path.basename(path))[0])
|
||||
raw = [x for x in SEP_RE.split(name) if x]
|
||||
years = [t for t in raw if YEAR_RE.match(t)]
|
||||
two_year = len(years) > 1 # "2001…1968": one of them is the title
|
||||
toks, year = [], None
|
||||
for t in raw:
|
||||
if YEAR_RE.match(t):
|
||||
year = t # last year wins
|
||||
toks.append("\x00")
|
||||
else:
|
||||
toks.append(t)
|
||||
stop, seen_word = len(toks), False
|
||||
for i, t in enumerate(toks):
|
||||
if t == "\x00":
|
||||
continue
|
||||
c = norm(re.sub(r"[^A-Za-z0-9']", "", t))
|
||||
if (not c or RES_RE.match(c) or JUNK_RE.match(c) or c in JUNK
|
||||
or not WORD_RE.match(c)) and not (IDX_RE.match(t) and not seen_word):
|
||||
stop = i
|
||||
break
|
||||
if c.isalpha():
|
||||
seen_word = True
|
||||
# A leading number is a track index ("01. Bye Bye Baby", "05. Casablanca") unless
|
||||
# it is three digits or more ("007", "000", "1408") or the name carries no year
|
||||
# at all ("12 Monkeys", "831"). A 1-2 digit prefix beside a year is an index;
|
||||
# the rare real title that looks like one ("0 Things I Hate About You (1999)")
|
||||
# loses its zero to the parser — Plex's matched title overrides it anyway.
|
||||
lead = re.split(r"[\s._-]", name, maxsplit=1)[0]
|
||||
track_idx = bool(re.fullmatch(r"\d{1,2}[a-z]?", lead)) and bool(year)
|
||||
keep, num_title = [], None
|
||||
for t in toks[:stop]:
|
||||
if t == "\x00":
|
||||
if keep:
|
||||
break # year closes the title
|
||||
if two_year and t is not None and years[0] != year:
|
||||
# two DIFFERENT years ("2001…1968"): the leading one is the title;
|
||||
# "1940 - Fantasia (1940)" repeats one year, so it is not a title
|
||||
num_title = num_title or years[0]
|
||||
continue # leading year: not the title
|
||||
if track_idx and IDX_RE.match(t) and not keep and t == lead:
|
||||
continue # "03." / "1c." ordering
|
||||
c = norm(re.sub(r"[^A-Za-z0-9']", "", t))
|
||||
if not keep and not c.isalpha():
|
||||
num_title = num_title or (c or None)
|
||||
continue # numeric title candidate
|
||||
keep.append(c)
|
||||
if len(keep) >= 8:
|
||||
break
|
||||
keep = [k for k in keep if not re.fullmatch(r"(disc|disk|cd|dvd|part)\d*", k)]
|
||||
title = " ".join(keep)
|
||||
if not keep:
|
||||
# nothing collected: a pure-numeric leading token is the title even when it
|
||||
# sits where the release year sits ("1917.2019.2160p…"), and "300.2006…" too
|
||||
for t in raw:
|
||||
c = norm(re.sub(r"[^A-Za-z0-9']", "", t))
|
||||
if re.fullmatch(r"(?:[12]\d{3}|\d{1,3})", c) and (c != year or two_year):
|
||||
return c, year or "?" # never "2160p" / "1080": they are not pure digits
|
||||
if (num_title and title and not keep[0].isdigit()
|
||||
and (two_year or not track_idx)):
|
||||
# a leading number that only echoes the release year ("1917.2019.2160p…") is
|
||||
# not a title; one that stands alone ("300.2006…", "1900") is
|
||||
title = f"{num_title} {title}"
|
||||
title = title or num_title or norm(name).replace("\x00", " ").strip()[:40]
|
||||
return title, year or "?"
|
||||
|
||||
|
||||
# Bonus-material vocabulary: names that say nothing about the film, so an extras
|
||||
# file otherwise pairs with an extras file of another movie. "Short", "Spot" and
|
||||
# "Clip" are deliberately absent — they are film titles (Short Circuit, Spot,
|
||||
# Clips) — and the running-time guard in is_extra() carries the rest.
|
||||
SEP_CLS = r"[\s._-]+"
|
||||
|
||||
|
||||
def _bonus_rx(word):
|
||||
return r"(?<![a-z0-9])" + re.sub(r"[ \-]+", lambda _: SEP_CLS, word.lower()) + r"(?![a-z0-9])"
|
||||
|
||||
|
||||
EXTRA_WORDS = [
|
||||
"trailer", "teaser", "featurette", "featurettes", "promo", "promotional",
|
||||
"tv spot", "advert", "commercial", "behind the scenes", "making of",
|
||||
"deleted scene", "deleted scenes", "deleted footage", "gag reel", "blooper",
|
||||
"blooper reel", "bloopers", "outtake", "outtakes", "scene selection", "menu",
|
||||
"menus", "gallery", "still gallery", "stills", "poster", "posters",
|
||||
"montage", "preview", "previews", "epilogue", "visual effects",
|
||||
"music video", "singalong", "karaoke", "trivia", "storyboard", "workprint",
|
||||
"test footage", "screen test", "audition", "cast and crew", "isolated score",
|
||||
"picture in picture", "original score", "session piece", "sample",
|
||||
"watermark",
|
||||
]
|
||||
# Words that describe bonus material which can run LONG (a feature-length
|
||||
# commentary-track cut, a documentary about the film). Trusted only below
|
||||
# LONG_BONUS_MAX, so a 40-minute "commentary" release stays a film.
|
||||
EXTRA_LONG_WORDS = ["interview", "interviews", "commentary", "video archive",
|
||||
"archival", "documentary"]
|
||||
EXTRA_NAME_RE = re.compile(
|
||||
"|".join([_bonus_rx(w) for w in EXTRA_WORDS] + [r"scene[\s._-]?\d{1,3}"]), re.I)
|
||||
EXTRA_LONG_NAME_RE = re.compile(
|
||||
"|".join(_bonus_rx(w) for w in EXTRA_LONG_WORDS), re.I)
|
||||
# Bonus-folder vocabulary. fullmatch = the folder IS an extras subtree; a bare
|
||||
# search = it only mentions extras ("... + Extras" box sets hold feature films).
|
||||
EXTRA_FOLDER_RE = re.compile(
|
||||
r"(?:extras?|bonus(?:[\s._-]material)?|specials?|special[\s._-]featurettes?|"
|
||||
r"featurettes?|deleted(?:[\s._-](?:scenes?|footage))?|trailers?|"
|
||||
r"making[\s._-]of|behind[\s._-]the[\s._-]scenes|shorts?|interviews?|"
|
||||
r"archiv(?:al|es)|video[\s._-]archive)", re.I)
|
||||
VIDEO_SUFFIX = (".mkv", ".mp4", ".avi", ".m4v", ".mov", ".m2ts", ".mpg", ".webm")
|
||||
|
||||
# Anything longer than this is a feature film whatever it is called: a 118-minute
|
||||
# file named "The Menu" or "Trailer" is a movie, not bonus material.
|
||||
SHORT_MAX = 45 * 60
|
||||
|
||||
# Long-shaped bonus vocabulary is only believed below this: a 40-minute
|
||||
# commentary cut is an alternate version of the film, not a featurette.
|
||||
LONG_BONUS_MAX = 25 * 60
|
||||
|
||||
|
||||
def name_words(x):
|
||||
"""Bare words of a file or folder name: no release metadata, no year."""
|
||||
x = os.path.splitext(x)[0] if x.lower().endswith(VIDEO_SUFFIX) else x
|
||||
return [w for w in SEP_RE.split(fold_sep(x))
|
||||
if w and not YEAR_RE.match(w) and not RES_RE.match(w)
|
||||
and not JUNK_RE.match(w) and w not in JUNK]
|
||||
|
||||
|
||||
def mentions(short, long_):
|
||||
"""True when one word list appears in the other, anywhere: a release names
|
||||
the film at the front ("Alien (1979) - Trailer") or in the middle
|
||||
("Movies/Alien (1979)/Extra - Logo.mkv"), so a start-of-string rule fails."""
|
||||
a, b = sorted([short, long_], key=len)
|
||||
if not a:
|
||||
return False
|
||||
return any(b[i:i + len(a)] == a for i in range(len(b) - len(a) + 1))
|
||||
|
||||
|
||||
|
||||
def folder_is_bonus(folder):
|
||||
"""True when a folder name carries bonus vocabulary as WHOLE words: the
|
||||
vocabulary is matched against a padded word string, so "Extras" decides and
|
||||
a word that merely CONTAINS one ("Extraordinary", "The Extras") does not."""
|
||||
return bool(EXTRA_FOLDER_RE.search(" " + " ".join(SEP_RE.split(norm(folder))) + " "))
|
||||
|
||||
|
||||
def is_extra(path, title="", dur=0.0, root=ROOT, matched=False):
|
||||
"""True for bonus material, which must never be counted as a film: a file
|
||||
called "Trailer", "Menu", "Sample" or "Gallery" carries no film identity, so
|
||||
extras otherwise pair with each other across the whole library.
|
||||
|
||||
Evidence, strongest first:
|
||||
* RUNTIME VETOES EVERYTHING. The film "The Menu", the film "Trailer", the
|
||||
franchise "Trailer Park Boys" and "ET — The Extra-Terrestrial" all carry
|
||||
bonus vocabulary and run 100+ minutes: anything over SHORT_MAX is a film —
|
||||
unless it sits in an explicit bonus folder and metadata never matched it.
|
||||
Bonus words that describe long-form material (commentary, documentary,
|
||||
interview) are believed only
|
||||
under LONG_BONUS_MAX, so a feature-length commentary cut stays a film.
|
||||
* The file's OWN name carries bonus vocabulary -> bonus.
|
||||
* ANY folder of the path below the film's own root folder carries bonus
|
||||
vocabulary ("Alien (1979)/Extras/", ".../Featurettes/Deleted Scenes/",
|
||||
"Angel's Egg (1985)/04b. Angel's Egg Extras/") -> bonus. The ROOT folder
|
||||
is exempt because it names the film, and a film may be titled with a
|
||||
bonus word; and a file whose own name IS that folder's film name is the
|
||||
feature, not the bonus ("Ghost in the Shell Movies/Extras/
|
||||
Ghost in the Shell - CM3.mp4" is a bonus, but a whole other film filed
|
||||
there stays a film).
|
||||
Bonus vocabulary is matched as whole words only, so the film
|
||||
"Angel's Egg - Extra - Logo" inside "04b. Angel's Egg Extras" is reached by
|
||||
the folder rule, and "…Extras" as a fragment of a word is never evidence.
|
||||
"""
|
||||
parts = [q for q in path.split("/") if q]
|
||||
if not parts:
|
||||
return False
|
||||
name = (os.path.splitext(parts[-1])[0]
|
||||
if parts[-1].lower().endswith(VIDEO_SUFFIX) else "")
|
||||
# A bonus folder anywhere BELOW the film's own folder decides for its
|
||||
# contents; the film's own folder is the film's name, whatever words it
|
||||
# contains, so it is exempt. It is the first component AFTER ROOT's own
|
||||
# components — those are the mount path, not folders of the library.
|
||||
head = len([q for q in root.split("/") if q]) + 1
|
||||
bonus_folder = False
|
||||
for folder in parts[head:-1]:
|
||||
if folder_is_bonus(folder):
|
||||
bonus_folder = not mentions(name_words(name), name_words(folder))
|
||||
break
|
||||
if dur and dur > SHORT_MAX:
|
||||
# Runtime vetoes vocabulary, with ONE exception: an explicit bonus folder
|
||||
# and no metadata match. A 90-minute "Feature Length Rough Cut" sitting in
|
||||
# a movie's Featurettes/ is a disc extra, and nothing in the library claims
|
||||
# it as a film. Long-form bonus words still describe a film, so a
|
||||
# feature-length commentary cut or documentary keeps its runtime veto.
|
||||
if bonus_folder and not matched and not EXTRA_LONG_NAME_RE.search(name):
|
||||
return True
|
||||
return False
|
||||
if EXTRA_NAME_RE.search(name):
|
||||
return True
|
||||
if dur and dur <= LONG_BONUS_MAX and EXTRA_LONG_NAME_RE.search(name):
|
||||
return True
|
||||
return bonus_folder
|
||||
|
||||
# Release-group watermark clips: a group drops a tiny ad ("ETRG.mp4",
|
||||
# "RARBG.com.mp4") into many different films, so its name carries no film
|
||||
# identity at all and it would otherwise pair with itself across the library.
|
||||
WATERMARK_MAX = 20 * 2**20
|
||||
WATERMARK_RE = re.compile(r"(?i)^(rarbg(\.com)?|etrg|tgx|fbt|cpbg|yify|yts|"
|
||||
r"rarbg\.com|playnomore|imprs|gauntlet|dfx|macmote|"
|
||||
r"psi-axiz|acm|tlf|sa8|ebp|gtv|w4g|don|aAF|enjoy|hd4u)$")
|
||||
|
||||
|
||||
def is_watermark(path, size):
|
||||
"""True for a release group's own watermark video, not a film or an extra."""
|
||||
return (size < WATERMARK_MAX
|
||||
and bool(WATERMARK_RE.match(os.path.splitext(os.path.basename(path))[0])))
|
||||
|
||||
|
||||
|
||||
def key_of(title):
|
||||
"""Franchise key: first two alphabetic tokens (numeric-only title = itself).
|
||||
Deliberately NOT film-specific — it only selects candidates for the duration
|
||||
check, so a two-token collision (two films starting "star trek") cannot by
|
||||
itself create a duplicate group."""
|
||||
toks = [t for t in title.split() if t]
|
||||
words = [t for t in toks if not t.isdigit()]
|
||||
if not words:
|
||||
return " ".join(toks)
|
||||
sig = [t for t in words if t not in STOP] or words
|
||||
return " ".join(sig[:2])
|
||||
|
||||
|
||||
def ep_of(path):
|
||||
"""(show, episode-token) when the basename carries SxxEyy / 1x04 / Part N."""
|
||||
name = os.path.splitext(os.path.basename(path))[0]
|
||||
m = EP_RE.search(name)
|
||||
if not m:
|
||||
return None
|
||||
show, _ = title_of(path)
|
||||
return show, norm(m.group(1)).replace(" ", "")
|
||||
|
||||
|
||||
def base_words(title, key):
|
||||
"""Title tokens beyond the franchise key — the FILM identity part.
|
||||
"evil dead rise" under key "evil dead" -> {rise}; "28 days later" -> {}."""
|
||||
kt = set(key.split())
|
||||
return frozenset(t for t in title.split() if t not in kt and t not in STOP
|
||||
and not t.isdigit() and not YEAR_RE.match(t))
|
||||
|
||||
|
||||
def folder_year(path):
|
||||
"""Soft year evidence: a 19xx/20xx token in any parent directory name
|
||||
(collection folders are written both "(2002)" and "[2009]"; a range such as
|
||||
"(1998-2000)" deliberately yields nothing)."""
|
||||
for d in reversed(os.path.dirname(path).split("/")):
|
||||
ys = [y for y in re.split(r"[^0-9]+", d) if YEAR_RE.match(y)]
|
||||
if len(ys) == 1:
|
||||
return ys[0]
|
||||
return None
|
||||
|
||||
|
||||
def same_film(a, b):
|
||||
"""Two files are copies of ONE film only when the film-identity tokens agree
|
||||
(one side may carry none) AND no year evidence contradicts them. The key says
|
||||
the franchise matches; this says the actual film matches — it is what keeps
|
||||
'Evil Dead' (2013) apart from 'Evil Dead Rise' (2023) and the four Austin
|
||||
Powers apart when their runtimes happen to agree."""
|
||||
ba, bb = a["base"], b["base"]
|
||||
if ba and bb and ba != bb:
|
||||
return False
|
||||
ya = a["year"] if a["year"] != "?" else a["soft"]
|
||||
yb = b["year"] if b["year"] != "?" else b["soft"]
|
||||
if ya and yb and ya != yb:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def dur_close(a, b):
|
||||
if a <= 0 or b <= 0:
|
||||
return True # unknown duration: let titles decide
|
||||
return abs(a - b) <= max(2.0, 0.003 * max(a, b))
|
||||
|
||||
|
||||
# Blu-ray folder trees: ffprobe only sees the clip it is pointed at, so each tree
|
||||
# is folded into ONE entry, described by its longest main-feature clip (measured
|
||||
# with ffprobe on the largest BDMV/STREAM file) and sized by the whole tree.
|
||||
BDMV_MAIN = {
|
||||
"Avengers.Age.of.Ultron.2015.2160p.BluRay.HEVC.TrueHD.7.1.Atmos-TERMiNAL":
|
||||
dict(dur=8478.2, vcodec="hevc", res="3840x2160", pix="yuv420p10le", fps=23.976),
|
||||
"Full.Metal.Jacket.1987.2160p.BluRay.HEVC.DTS-HD.MA.5.1-EATDIK":
|
||||
dict(dur=6993.3, vcodec="hevc", res="3840x2160", pix="yuv420p10le", fps=23.976),
|
||||
}
|
||||
|
||||
|
||||
def load(probed):
|
||||
"""Probe rows -> film entries. AppleDouble (._*) sidecars are dropped; each
|
||||
BDMV tree collapses to one entry carrying the tree's total size."""
|
||||
trees = {}
|
||||
rows = []
|
||||
for line in open(probed):
|
||||
f = line.rstrip("\n").split("\t")
|
||||
if len(f) < 12:
|
||||
continue
|
||||
b, p, dur, vc, res, fps, vbit, pix, aud, subs, _d, fmt = f[:12]
|
||||
base = os.path.basename(p)
|
||||
if base.startswith("._") or base == ".DS_Store":
|
||||
continue # macOS sidecar, not media
|
||||
m = re.search(r"/(BDMV|VIDEO_TS)/", p)
|
||||
root = p[:m.start()] if m else None
|
||||
if root or re.search(r"/(CERTIFICATE|AACS|MLPI|CLIPINF|PLAYLIST|BACKUP|REGISTRL)/", p):
|
||||
if not root:
|
||||
continue # disc bookkeeping: folds into its tree, never a film
|
||||
if root:
|
||||
t = trees.setdefault(root, dict(bytes=0, audio=set(), subs=0))
|
||||
t["bytes"] += int(b)
|
||||
t["audio"].update(a for a in aud.split("+") if a)
|
||||
t["subs"] += int(subs or 0)
|
||||
continue
|
||||
try:
|
||||
dur = float(dur)
|
||||
except ValueError:
|
||||
dur = 0.0
|
||||
rows.append(dict(bytes=int(b), path=p, dur=dur, vcodec=vc, res=res, fps=fps,
|
||||
vbit=int(float(vbit or 0)), pix=pix, audio=aud, subs=subs, fmt=fmt))
|
||||
for root, t in trees.items():
|
||||
folder = os.path.basename(root)
|
||||
m = BDMV_MAIN.get(folder, {})
|
||||
rows.append(dict(bytes=t["bytes"], path=root + "/BDMV/index.bdmv",
|
||||
name_src=folder, dur=m.get("dur", 0.0),
|
||||
vcodec=m.get("vcodec", "hevc"), res=m.get("res", "-"),
|
||||
fps=m.get("fps", 0), vbit=0, pix=m.get("pix", "-"),
|
||||
audio="+".join(sorted(t["audio"])), subs=str(t["subs"]),
|
||||
fmt="BDMV tree"))
|
||||
for r in rows:
|
||||
r["title"], r["year"] = title_of(r.get("name_src") or r["path"])
|
||||
r["key"] = key_of(r["title"])
|
||||
r["base"] = base_words(r["title"], r["key"])
|
||||
r["soft"] = folder_year(r["path"])
|
||||
r["ep"] = ep_of(r["path"])
|
||||
return rows
|
||||
|
||||
|
||||
def dur_clusters(items):
|
||||
"""Group files that are the SAME FILM and the SAME MASTER duration. Candidate
|
||||
buckets come from the franchise key; same_film then rejects the different
|
||||
installments whose runtimes happen to agree."""
|
||||
clusters = []
|
||||
for r in sorted(items, key=lambda r: -r["dur"]):
|
||||
for c in clusters:
|
||||
if dur_close(c[0]["dur"], r["dur"]) and same_film(c[0], r):
|
||||
c.append(r)
|
||||
break
|
||||
else:
|
||||
clusters.append([r])
|
||||
return clusters
|
||||
|
||||
|
||||
def tier(groups):
|
||||
"""Split same-film/same-duration clusters: DUP = different releases of one
|
||||
film (real, deletable redundancy); REVIEW = identical size AND duration
|
||||
(byte-twins — the hash pass is the authority) or clashing year evidence."""
|
||||
dup, review = [], []
|
||||
for g in groups:
|
||||
g.sort(key=lambda r: -r["bytes"])
|
||||
years = {(r["year"] if r["year"] != "?" else r["soft"]) for r in g}
|
||||
years.discard(None)
|
||||
spread = (g[0]["bytes"] - g[-1]["bytes"]) / g[0]["bytes"] if g[0]["bytes"] else 0
|
||||
if len(years) > 1 or spread < 0.005:
|
||||
review.append(g)
|
||||
else:
|
||||
dup.append(g)
|
||||
return dup, review
|
||||
|
||||
|
||||
def redun(g):
|
||||
return sum(r["bytes"] for r in sorted(g, key=lambda r: -r["bytes"])[1:])
|
||||
|
||||
|
||||
def line(r):
|
||||
return (f"{r['bytes']/2**30:8.2f} GiB {r['res']:>9} {r['vcodec']:<7} "
|
||||
f"{r['pix']:<11} {r['dur']/60:5.0f}m {r['audio'][:34]:<34}")
|
||||
|
||||
|
||||
def dump(tag, groups, out, note=""):
|
||||
for gi, g in enumerate(groups, 1):
|
||||
g.sort(key=lambda r: -r["bytes"])
|
||||
print(f"\n### {tag}-{gi}. {g[0]['title']} ({g[0]['year']}) — {len(g)} files, "
|
||||
f"{redun(g)/2**30:.1f} GiB redundant {note}".rstrip())
|
||||
for rank, r in enumerate(g):
|
||||
print(f" {line(r)} [{r['year']}] "
|
||||
f"{r['path'].replace(ROOT + '/', '')}")
|
||||
out.write(f"{tag}\t{gi}\t{rank}\t{r['bytes']}\t{r['res']}\t{r['vcodec']}\t"
|
||||
f"{r['pix']}\t{int(r['dur'])}\t{r['audio']}\t{r['title']}|{r['year']}\t"
|
||||
f"{r['path']}\n")
|
||||
|
||||
|
||||
def report(probed, out_path):
|
||||
rows = load(probed)
|
||||
for r in rows:
|
||||
r["mark"] = is_watermark(r["path"], r["bytes"])
|
||||
r["extra"] = not r["mark"] and is_extra(r["path"], r["title"], r["dur"])
|
||||
marks = [r for r in rows if r["mark"]]
|
||||
movies = [r for r in rows if not r["ep"] and not r["extra"] and not r["mark"]]
|
||||
episodes = [r for r in rows if r["ep"] and not r["extra"] and not r["mark"]]
|
||||
extras = [r for r in rows if r["extra"]]
|
||||
print(f"census: {len(rows)} video entries = {len(movies)} films + {len(episodes)} "
|
||||
f"TV episodes + {len(extras)} bonus/trailers "
|
||||
f"({sum(r['bytes'] for r in extras)/2**30:.1f} GiB) "
|
||||
f"+ {len(marks)} release-group watermarks")
|
||||
|
||||
out = open(out_path, "w")
|
||||
|
||||
buckets = collections.defaultdict(list)
|
||||
for r in movies:
|
||||
buckets[r["key"]].append(r)
|
||||
cand = {k: v for k, v in buckets.items() if len(v) > 1}
|
||||
clusters = [c for v in cand.values() for c in dur_clusters(v) if len(c) > 1]
|
||||
dup, review = tier(clusters)
|
||||
dup.sort(key=lambda g: -redun(g))
|
||||
review.sort(key=lambda g: -redun(g))
|
||||
dump("DUP", dup, out)
|
||||
dump("REVIEW", review, out, "— same key+duration, sizes near-identical OR years clash: RECHECK")
|
||||
|
||||
|
||||
ebuf = collections.defaultdict(list)
|
||||
for r in episodes:
|
||||
ebuf[(r["ep"][0], r["ep"][1])].append(r)
|
||||
edup, ereview = tier([c for v in ebuf.values() if len(v) > 1
|
||||
for c in dur_clusters(v) if len(c) > 1])
|
||||
edup.sort(key=lambda g: -redun(g))
|
||||
dump("EP-DUP", edup, out)
|
||||
|
||||
shows = collections.Counter()
|
||||
for r in episodes:
|
||||
shows[r["ep"][0]] += 1
|
||||
multi = {s: n for s, n in shows.items() if n > 4}
|
||||
print("\n=== TV-series content inside Movies/ (>4 episodes) ===")
|
||||
for s, n in sorted(multi.items(), key=lambda kv: -kv[1]):
|
||||
print(f" {n:4d} episodes {s}")
|
||||
out.close()
|
||||
|
||||
print(f"\nDUP groups={len(dup)} files={sum(len(g) for g in dup)} "
|
||||
f"redundant={sum(redun(g) for g in dup)/2**40:.2f} TiB", file=sys.stderr)
|
||||
print(f"REVIEW groups={len(review)} "
|
||||
f"redundant={sum(redun(g) for g in review)/2**40:.2f} TiB", file=sys.stderr)
|
||||
print(f"EP-DUP groups={len(edup)} "
|
||||
f"ep-redundant={sum(redun(g) for g in edup)/2**30:.1f} GiB", file=sys.stderr)
|
||||
|
||||
|
||||
def parse(names):
|
||||
for n in names:
|
||||
t, y = title_of(n)
|
||||
print(f"{t!r} ({y})\tkey={key_of(t)!r}\t{n[:70]}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
rep = sub.add_parser("report", help="similar-copy census over a probe TSV")
|
||||
rep.add_argument("--probed", default="/tmp/media-probed.tsv")
|
||||
rep.add_argument("--out", default="/tmp/media-similar.tsv")
|
||||
par = sub.add_parser("parse", help="show the parsed title/year of a name")
|
||||
par.add_argument("names", nargs="+")
|
||||
a = ap.parse_args()
|
||||
if a.cmd == "report":
|
||||
report(a.probed, a.out)
|
||||
else:
|
||||
parse(a.names)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user