Frank
73f77a13c3
Keyed on the whole episode title, every episode of a box set became a group of one and never reached the five-episode size that calls something a series, so the live run filed 124 Monk episodes as 124 movie folders, each named after its own episode. ep_of now returns the series — the name before SxxEyy / 1x04 / Part N — and the planner groups and names the quarantine pile by that series, so one show is one pile: 'Monk - S05E03 - Mr. Monk and the Big Game' is series monk. The threshold keeps its job: a lone film named Part 2 stays a film. Re-planning the live census with this moves nothing differently except the 124 Monk files, which now join the TV pile: 168 episode files across 3 series.
882 lines
38 KiB
Python
Executable File
882 lines
38 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
r"""Plan a Jellyfin-format relayout of a movie library. DRY RUN ONLY: nothing moves.
|
|
|
|
media-plan.py [--root DIR] [--enum TSV] [--probed TSV] [--plex TSV]
|
|
[--out TSV] [--exceptions TXT]
|
|
|
|
Inputs are the TSVs scripts/media-census.py exports, plus the optional Plex
|
|
metadata-cache export whose clean titles outrank the filename parser; the title
|
|
authority for everything Plex never matched is scripts/media-titles.py beside this
|
|
file. No media file is opened, so planning the whole library costs seconds.
|
|
|
|
PLAN is a TSV the executor (scripts/media-apply.py) replays verbatim —
|
|
`action<TAB>src<TAB>dst<TAB>kind` with action in mkdir | move | keep, ordered so
|
|
parents exist before children. EXCEPTIONS is the human review list: every file the
|
|
plan could not name a film for, every collision it refused, the READMEs the
|
|
executor must write into the piles it created.
|
|
|
|
Target contract (jellyfin.org/docs/general/server/media/movies):
|
|
* one folder per movie, named Title (year)
|
|
* the video inside is named exactly like the folder
|
|
* extra copies of one film are versions: "<base> - <label>.ext"; the base must
|
|
match the folder name character-for-character or Jellyfin splits them apart
|
|
* extras live in canonical subfolders (trailers, featurettes, deleted scenes,
|
|
interviews, samples, shorts, clips, other, extras, backdrops, theme-music)
|
|
* sidecars <base>.<lang>.<flag>.<ext>; images <base>-<poster|backdrop|...>
|
|
* <>:"/\|?* break the matcher, so they never survive here
|
|
* VIDEO_TS/BDMV folders support neither multiple versions nor parts
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import csv
|
|
import importlib.util
|
|
import os
|
|
import re
|
|
|
|
# Inputs are the census TSVs scripts/media-census.py writes; the engine beside
|
|
# this file is the title authority. Nothing here opens a media file.
|
|
def parse_args():
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--root", default=os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies"),
|
|
help="library root the plan moves files inside")
|
|
ap.add_argument("--enum", default="/tmp/media-enum.tsv", help="file census TSV")
|
|
ap.add_argument("--probed", default="/tmp/media-probed.tsv", help="ffprobe census TSV")
|
|
ap.add_argument("--plex", default="/tmp/plex-movies.tsv",
|
|
help="Plex metadata export (clean titles); absent = parser only")
|
|
ap.add_argument("--out", default="/tmp/jf-plan.tsv")
|
|
ap.add_argument("--exceptions", default="/tmp/jf-exceptions.txt")
|
|
return ap.parse_args()
|
|
|
|
|
|
ARGS = parse_args()
|
|
ROOT = ARGS.root
|
|
ENUM, PROBE, PLEX = ARGS.enum, ARGS.probed, ARGS.plex
|
|
PLAN, EXCEPTS = ARGS.out, ARGS.exceptions
|
|
ENGINE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "media-titles.py")
|
|
|
|
spec = importlib.util.spec_from_file_location("eng", ENGINE)
|
|
eng = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(eng)
|
|
|
|
VIDEO_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg",
|
|
".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".flv", ".vob", ".iso",
|
|
".img", ".mpv", ".3gp", ".rm", ".rmvb", ".m2v", ".mpv"}
|
|
DISC_DIRS = {"BDMV", "VIDEO_TS", "CERTIFICATE", "AACS", "MLPI", "CLIPINF"}
|
|
SUB_EXT = {".srt", ".ass", ".ssa", ".sub", ".vtt"}
|
|
IMG_EXT = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif", ".tbn"}
|
|
AUDIO_DIR_RE = re.compile(r"soundtrack|score|\bost\b|music|song|album", re.I)
|
|
SUB_KEEP_RE = re.compile(r"^(?!.*\d)(?!.*[._])(?!.*\[)(?!.*\()([A-Za-z][A-Za-z -]{1,24})$")
|
|
MUSIC_TOP_RE = re.compile(r"(?i)soundtrack|score|\bost\b|music|album|song")
|
|
SUB_DIR_OK = re.compile(r"(?i)^$|subs?|subtitles?|srt|custom ?subs?(\[.*\])?"
|
|
r"|(additional|extra) languages|languages|resources")
|
|
AUDIO_DIR_OK = re.compile(r"(?i)^$|audio(s)?|extra audio|additional audio"
|
|
r"|(additional|extra) languages|commentar(y|ies)"
|
|
r"|audio commentary|audio tracks?")
|
|
|
|
|
|
AUDIO_EXT = {".mp3", ".flac", ".m4a", ".aac", ".dts", ".wav", ".ac3", ".mka",
|
|
".opus", ".ogg", ".wma"}
|
|
NFO_EXT = {".nfo"}
|
|
IMG_PRIMARY = {"poster", "folder", "cover", "default", "movie", "show"}
|
|
IMG_OTHER = {"backdrop": "backdrop", "fanart": "backdrop", "background": "backdrop",
|
|
"banner": "banner", "logo": "logo", "clearlogo": "logo",
|
|
"thumb": "thumb", "landscape": "thumb", "art": "backdrop",
|
|
"clearart": "backdrop", "jacket": "poster", "cdart": "poster",
|
|
"discart": "poster", "disc": "poster"}
|
|
RES_RE = re.compile(r"\b(2160|4320|4k|1080|720|576|480|360|240)(i|p)\b", re.I)
|
|
# The characters Jellyfin's matcher cannot carry in a file or folder name. "/" is
|
|
# one of them and it is the dangerous one: Plex titles KEEP it ("V/H/S/2", and this
|
|
# library holds six of those films), so leaving it in a name turns ONE movie folder
|
|
# into a nested tree — Movies/V/H/S/2 (2013)/… — and Jellyfin then sees four
|
|
# folders, none of them a movie.
|
|
BAD_CHARS = re.compile(r'[<>:"/|?*]')
|
|
PART_RE = re.compile(r"(?:^|[\s._-])(cd|dvd|part|pt|disc|disk|disco)"
|
|
r"[\s._-]?([1-9abcd])(?:$|[\s._-])", re.I)
|
|
LANGS = {"en", "eng", "it", "ita", "ja", "jpn", "de", "deu", "fr", "fra", "es",
|
|
"spa", "pt", "por", "ru", "rus", "zh", "chi", "ko", "kor", "sv", "no",
|
|
"da", "nl", "pl", "cs", "hu", "fi", "tr", "el", "he", "ar", "uk", "ro",
|
|
"bg", "hr", "sr", "sk", "et", "lv", "lt", "id", "th", "vi", "fa", "la"}
|
|
FLAGS = {"default", "forced", "foreign", "sdh", "cc"}
|
|
NOISE = {"sub", "subs", "subtitle", "subtitles", "srt", "ass", "ext", "external",
|
|
"the", "a", "an", "of", "and", "in", "on", "to", "for", "with", "ita",
|
|
"eng", "english", "italian", "dual", "audio", "forced", "default"}
|
|
# Canonical title where the library only ever carries a localised name.
|
|
OVERRIDES = {"28 giorni dopo": "28 Days Later",
|
|
"28 settimane dopo": "28 Weeks Later"}
|
|
|
|
rows = []
|
|
for line in open(ENUM):
|
|
size, ext, path = line.rstrip("\n").split("\t")
|
|
rows.append({"bytes": int(size), "ext": ext.lower(), "path": path})
|
|
ext_by = {r["path"]: r["ext"] for r in rows}
|
|
size_by = {r["path"]: r["bytes"] for r in rows}
|
|
|
|
|
|
def rel(p):
|
|
return p[len(ROOT) + 1:] if p.startswith(ROOT + "/") else p
|
|
|
|
|
|
def parts_of(p):
|
|
return [q for q in rel(p).split("/") if q]
|
|
|
|
|
|
def is_qb(p):
|
|
return ".!q" in p.lower()
|
|
|
|
|
|
def disc_root(p, fold=True):
|
|
"""Absolute root of the disc tree containing p (the MOVIE folder that holds
|
|
BDMV/ or VIDEO_TS/), else None. `fold` also folds bookkeeping dirs that sit
|
|
beside the tree (CERTIFICATE/, AACS/) into the same movie folder."""
|
|
ps = parts_of(p)
|
|
kinds = DISC_DIRS if fold else {"BDMV", "VIDEO_TS"}
|
|
for i, q in enumerate(ps):
|
|
if q.upper() in kinds:
|
|
return ROOT + "/" + "/".join(ps[:i])
|
|
return None
|
|
|
|
|
|
def row_name(r):
|
|
"""The name a disc-tree entry should be identified by: the engine stores the
|
|
disc's own folder for it, because "BDMV/index.bdmv" is not a title."""
|
|
return r.get("name_src") or r["path"]
|
|
|
|
|
|
def sanitize(name):
|
|
name = BAD_CHARS.sub("-", name).replace("\\", "-")
|
|
name = re.sub(r"\s+", " ", name).strip().strip(". ").strip()
|
|
return (name[:180] or "untitled")
|
|
|
|
|
|
def ok_year(y):
|
|
"""True for a year a real film could carry. Plex stores 0 for a file it indexed
|
|
but never MATCHED, and "0" is a digit string: trusting it named 67 movie
|
|
folders "Title (0)" — Jellyfin items with no identity, no year and no poster."""
|
|
return bool(re.fullmatch(r"(18|19|20)\d\d", y or "")) and 1870 <= int(y) <= 2035
|
|
|
|
|
|
WORD_SPLIT = re.compile(r"[^0-9a-z]+")
|
|
|
|
|
|
def words_of(text):
|
|
"""The matching words of a name — or of a whole path, which is how callers
|
|
pass a file together with the folders it sits in.
|
|
|
|
Split on everything that is not a letter or a digit: eng.norm does not touch
|
|
"/", so feeding it "…/Psycho (1960)/Extras" left glued tokens like
|
|
"(1960)/extras" and "(1954)" that no other name could ever agree with, and a
|
|
film could not be found by the folder that holds it — which is how one box
|
|
set's featurettes ended up in another film's folder. Both sides of every
|
|
comparison below go through here. A bare number counts only when it could be
|
|
a release year, so release junk ("1080", "2160", "2", "7") never poses as
|
|
agreement between two names.
|
|
"""
|
|
out = set()
|
|
for w in WORD_SPLIT.split(eng.norm(text)):
|
|
if not w or w in eng.STOP:
|
|
continue
|
|
if w.isdigit() and not 1870 <= int(w) <= 2100:
|
|
continue
|
|
out.add(w)
|
|
return out
|
|
|
|
|
|
# ------------------------------------------------------------------ Plex titles
|
|
# Plex has matched a large part of the library's media files to real titles
|
|
# ("Alien", not "Alien.Directors.Cut.1979.1080p.BRrip.x265…"). Where the cache
|
|
# holds a MATCH, its title wins; the filename parser carries everything else
|
|
# (extras, disc trees, TV, and whatever Plex never matched).
|
|
def plex_titles():
|
|
if not os.path.exists(PLEX):
|
|
return {}
|
|
out = {}
|
|
with open(PLEX, newline="") as fh:
|
|
for r in csv.DictReader(fh, delimiter="\t"):
|
|
t, y = r["leaf_title"].strip(), r["leaf_year"].strip()
|
|
# Only a match is an authority, and a match carries a year: an
|
|
# unmatched row echoes the filename, which the parser reads better
|
|
# than a copied release name. The junk filter stays for the matches
|
|
# whose title still carries the release.
|
|
if t and ok_year(y) and not re.search(
|
|
r"2160p|1080p|x26[45]|hevc|bluray|web-?dl", t.lower()):
|
|
out[r["file"]] = (t, y)
|
|
return out
|
|
|
|
|
|
PLEX_T = plex_titles()
|
|
|
|
# ------------------------------------------------------------- classification
|
|
erows = eng.load(PROBE)
|
|
for r in erows:
|
|
for k, v in OVERRIDES.items():
|
|
if k in r["title"]:
|
|
r["title"] = v
|
|
r["key"] = eng.key_of(r["title"])
|
|
r["base"] = eng.base_words(r["title"], r["key"])
|
|
marks = {r["path"] for r in erows if eng.is_watermark(r["path"], r["bytes"])}
|
|
extras = {r["path"] for r in erows
|
|
if r["path"] not in marks
|
|
and eng.is_extra(r["path"], r["title"], r["dur"], ROOT,
|
|
matched=r["path"] in PLEX_T)}
|
|
eps = collections.defaultdict(list)
|
|
for r in erows:
|
|
if r["path"] in marks | extras:
|
|
continue
|
|
ep = eng.ep_of(r["path"])
|
|
if ep:
|
|
eps[ep[0]].append(r) # keyed by the SERIES, never the episode title
|
|
tv_keys = {k for k, v in eps.items() if len(v) >= 5}
|
|
tv_set = {r["path"] for k, v in eps.items() if k in tv_keys for r in v}
|
|
# A folder INSIDE a movie's folder is bonus territory even when its own name is
|
|
# not bonus vocabulary: "Saboteur (1942)/Exrtas/Saboteur - A Closer Look" is a
|
|
# misspelled Extras folder, so the engine's folder rule never fired and the
|
|
# featurette became a movie folder of its own — whose words then out-scored the
|
|
# real Saboteur and drew that box set's other extras into it. The rule needs no
|
|
# metadata: the folder holding the longest film owns the shorter video filed
|
|
# under it, provided that shorter file's own name speaks the film's name — a
|
|
# whole other film filed there stays a film, exactly as the engine's folder rule
|
|
# exempts it. A different year in the file's own name IS that other film
|
|
# speaking, so it vetoes the rule.
|
|
def longest_of(rows):
|
|
"""The longest video in each folder: that folder's feature."""
|
|
out = {}
|
|
for r in rows:
|
|
if r["dur"] > 0:
|
|
d = os.path.dirname(r["path"])
|
|
if d not in out or r["dur"] > out[d]["dur"]:
|
|
out[d] = r
|
|
return out
|
|
|
|
|
|
def not_disc(r):
|
|
"""A disc tree's innards belong to the tree, never to the folder's film."""
|
|
return not any(c in DISC_DIRS for c in r["path"].split("/"))
|
|
|
|
|
|
FEATURE_OF = longest_of([r for r in erows if r["path"] not in marks and not_disc(r)])
|
|
|
|
|
|
def under_a_feature(path, dur):
|
|
"""True when a folder above this file holds a longer film that this file's
|
|
own name names."""
|
|
d = os.path.dirname(os.path.dirname(path))
|
|
while len(d) > len(ROOT):
|
|
f = FEATURE_OF.get(d)
|
|
if f:
|
|
if f["dur"] < dur:
|
|
return False
|
|
theirs = (words_of(os.path.basename(f["path"]))
|
|
| words_of(os.path.basename(d)))
|
|
if not words_of(os.path.basename(path)) & theirs:
|
|
return False
|
|
y1 = re.search(r"(?:18|19|20)\d\d", os.path.basename(path))
|
|
y2 = re.search(r"(?:18|19|20)\d\d", os.path.basename(f["path"]))
|
|
return not (y1 and y2 and y1.group() != y2.group())
|
|
d = os.path.dirname(d)
|
|
return False
|
|
|
|
|
|
extras |= {r["path"] for r in erows
|
|
if r["dur"] > 0 and r["path"] not in marks | extras | tv_set
|
|
and not_disc(r) and under_a_feature(r["path"], r["dur"])}
|
|
film_rows = [r for r in erows if r["path"] not in marks | extras | tv_set]
|
|
|
|
buckets = collections.defaultdict(list)
|
|
for r in film_rows:
|
|
buckets[eng.key_of(r["title"])].append(r)
|
|
|
|
|
|
def res_label(r):
|
|
m = RES_RE.search(r["res"] or "")
|
|
if m:
|
|
n, kind = m.group(1).lower(), m.group(2).lower()
|
|
return "2160p" if n in ("2160", "4k") else f"{n}{kind}"
|
|
try:
|
|
w = int((r["res"] or "0x0").split("x")[0])
|
|
except (ValueError, IndexError):
|
|
w = 0
|
|
return "2160p" if w >= 3000 else ("1080p" if w >= 1400 else "SD")
|
|
|
|
|
|
def version_label(r):
|
|
bits = [res_label(r)]
|
|
vc = (r["vcodec"] or "").lower()
|
|
if vc in ("av1", "libaom", "libsvtav1"):
|
|
bits.append("AV1")
|
|
elif vc in ("vc1", "wmv3"):
|
|
bits.append("VC1")
|
|
if r["bytes"] > 40 * 2**30:
|
|
bits.append("REMUX")
|
|
if "imax" in r["path"].lower():
|
|
bits.append("IMAX")
|
|
return sanitize(" ".join(bits))
|
|
|
|
|
|
def group_name(items):
|
|
titles = collections.Counter(r["title"] for r in items)
|
|
ascii_t = [t for t in titles if t.isascii()]
|
|
title = (collections.Counter(ascii_t) if ascii_t else titles).most_common(1)[0][0]
|
|
years = [r["year"] for r in items if r["year"] != "?"] or \
|
|
[r["soft"] for r in items if r["soft"]]
|
|
year = collections.Counter(years).most_common(1)[0][0] if years else None
|
|
return sanitize(f"{title} ({year})" if year else title)
|
|
|
|
|
|
groups = []
|
|
for key, items in buckets.items():
|
|
for c in eng.dur_clusters(items):
|
|
g = {"rows": c, "base": group_name(c)}
|
|
known = [PLEX_T[r["path"]] for r in c if r["path"] in PLEX_T]
|
|
if known:
|
|
# one film, one name: a Plex match for any copy names every copy
|
|
t, yr = collections.Counter(known).most_common(1)[0][0]
|
|
g["plex"] = t
|
|
g["base"] = sanitize(f"{t} ({yr})" if ok_year(yr) else t)
|
|
groups.append(g)
|
|
clash = {b for b, n in collections.Counter(g["base"] for g in groups).items() if n > 1}
|
|
for g in groups:
|
|
if g["base"] in clash:
|
|
# two genuinely different films share title+year: separate folders, and
|
|
# say so in the name, because Jellyfin can only match one item per folder
|
|
g["base"] = sanitize(g["base"] + " - " + version_label(g["rows"][0]))
|
|
g["flag"] = "same title and year as another film"
|
|
|
|
moves, mkdirs, problems, readmes = [], set(), [], []
|
|
# --------------------------------------------------------------- unnamed groups
|
|
# A movie folder has to READ as a title. The engine's runtime veto says any file
|
|
# over 45 minutes that is not named like an extra is a film whatever it is called
|
|
# — true for Jellyfin, but a 118-minute documentary named "Extra 1" then produced
|
|
# a movie folder named "1", and folders named "1", "2", "3" are Jellyfin items
|
|
# with no identity whose numeric keys went on to capture every subtitle named
|
|
# "3_Eng.srt" in the library. A year proves a dated feature ("300 (2006)" is a
|
|
# real film), so the gate is: a year, or a word that is not release noise. With
|
|
# neither, the rows are bonus material until a human names them.
|
|
def named(g):
|
|
if re.search(r"\(\d{4}\)$", g["base"]):
|
|
return True
|
|
return any(w.isalpha() and len(w) >= 2 for w in eng.name_words(g["base"]))
|
|
|
|
|
|
unnamed = [g for g in groups if not named(g)]
|
|
groups = [g for g in groups if named(g)]
|
|
for g in unnamed:
|
|
for r in g["rows"]:
|
|
extras.add(r["path"])
|
|
problems.append(
|
|
f"UNNAMED film {rel(r['path'])} parses to {g['base']!r} with no year: "
|
|
f"bonus pile, not a folder — give the file a real name and re-plan")
|
|
|
|
|
|
group_of = {}
|
|
for g in groups:
|
|
g["dir"] = f"{ROOT}/{g['base']}"
|
|
for r in g["rows"]:
|
|
group_of[r["path"]] = g
|
|
dr = disc_root(r["path"])
|
|
if dr:
|
|
group_of[dr] = g
|
|
by_path = {r["path"]: r for r in erows}
|
|
|
|
# -------------------------------------------------------------- bookkeeping
|
|
claimed = {}
|
|
existing = set(rows_map := {r["path"] for r in rows})
|
|
|
|
|
|
at_target, booked_src = set(), set()
|
|
|
|
|
|
def book(src, dst, kind):
|
|
if src == dst:
|
|
at_target.add(src)
|
|
booked_src.add(src)
|
|
return
|
|
if os.environ.get("JF_TRACE") and (src in booked_src or dst.lower() in claimed):
|
|
why = "rebook" if src in booked_src else "collision"
|
|
problems.append(f"TRACE {why} kind={kind} src={rel(src)} dst={rel(dst)}")
|
|
if src in booked_src: # one source may only ever move once
|
|
problems.append(f"RE-BOOK {rel(src)} -> {rel(dst)} (already planned)")
|
|
return
|
|
if dst.lower() in claimed:
|
|
problems.append(f"COLLISION {rel(dst)}\n from {rel(src)}\n and {rel(claimed[dst.lower()])}")
|
|
return
|
|
claimed[dst.lower()] = src
|
|
booked_src.add(src)
|
|
comp = dst.split("/")
|
|
for i in range(1, len(comp)):
|
|
mkdirs.add("/".join(comp[:i]))
|
|
moves.append((src, dst, kind))
|
|
|
|
|
|
def uniq_dir(d, stem, ext, hint=""):
|
|
"""A destination that no other source already took."""
|
|
cand = f"{d}/{stem}{ext}"
|
|
if cand.lower() not in claimed:
|
|
return cand
|
|
n = 2
|
|
while True:
|
|
cand = f"{d}/{stem} {n}{ext}"
|
|
if cand.lower() not in claimed:
|
|
return cand
|
|
n += 1
|
|
|
|
|
|
def media_dir(p):
|
|
dr = disc_root(p)
|
|
return os.path.dirname(dr) if dr else os.path.dirname(p)
|
|
|
|
|
|
# ------------------------------------------------------------------ films
|
|
# film / disc moves: the largest copy carries the folder's own name, the rest
|
|
# become Jellyfin versions; a disc tree moves as a whole into the movie folder
|
|
|
|
|
|
def vext(p):
|
|
return ext_by.get(p) or os.path.splitext(p)[1].lower()
|
|
disc_children = collections.defaultdict(list)
|
|
for q in rows:
|
|
dr = disc_root(q["path"])
|
|
if dr:
|
|
disc_children[dr].append(q["path"])
|
|
for g in groups:
|
|
primary_taken = False
|
|
disc_kinds = set()
|
|
for r in sorted(g["rows"], key=lambda r: -r["bytes"]):
|
|
dr = disc_root(r["path"])
|
|
if dr:
|
|
inner = os.path.basename(dr).upper()
|
|
if inner in disc_kinds:
|
|
problems.append(
|
|
f"DISC-VERSION {g['base']}: extra disc tree {rel(dr)} cannot "
|
|
f"share the folder — discs have no multiple versions")
|
|
continue
|
|
disc_kinds.add(inner)
|
|
for src in disc_children[dr]:
|
|
tail = os.path.relpath(src, dr)
|
|
fold = disc_root(src, fold=False)
|
|
if fold != dr and os.path.dirname(src) != dr:
|
|
tail = os.path.basename(dr) + "/" + tail
|
|
book(src, f"{g['dir']}/{tail}", "disc-file")
|
|
continue
|
|
label = "" if not primary_taken else f" - {version_label(r)}"
|
|
dst = f"{g['dir']}/{g['base']}{label}{vext(r['path'])}"
|
|
n = 2
|
|
while dst.lower() in claimed:
|
|
dst = f"{g['dir']}/{g['base']}{label} {n}{vext(r['path'])}"
|
|
n += 1
|
|
r["dst"] = dst
|
|
primary_taken = True
|
|
book(r["path"], dst, "film")
|
|
g["primary"] = primary_taken
|
|
|
|
# ------------------------------------------------------------------ extras
|
|
EXTRA_KIND = [("trailers", r"trailer|teaser|preview|promo|tv spot|advert"),
|
|
("deleted scenes", r"deleted|alt scene|gag reel|blooper|outtake"),
|
|
("interviews", r"interview|cast and crew|session piece|commentary"),
|
|
("featurettes", r"featurette|behind the scenes|making of|documentary"
|
|
r"|heritage|archival|video archive|music video"),
|
|
("samples", r"sample"),
|
|
("shorts", r"\bshort\b|short film"),
|
|
("clips", r"\bclip\b|scene selection|montage|epilogue|isolated score"),
|
|
("other", r"menu|logo|gallery|still|trivia|storyboard|screen test"
|
|
r"|audition|test footage|watermark|workprint|poster"
|
|
r"|picture in picture|singalong|karaoke|original score")]
|
|
THEME_RE = re.compile(r"theme|opening song|ending song|soundtrack|score"
|
|
r"|\bost\b|music|song", re.I)
|
|
BACKDROP_RE = re.compile(r"backdrop|intro", re.I)
|
|
|
|
|
|
def extras_kind(path):
|
|
low = path.lower()
|
|
if THEME_RE.search(low):
|
|
return "theme-music"
|
|
if BACKDROP_RE.search(low):
|
|
return "backdrops"
|
|
for name, rx in EXTRA_KIND:
|
|
if re.search(rx, low):
|
|
return name
|
|
return "extras"
|
|
|
|
|
|
def attachable_title(path):
|
|
t, _y = eng.title_of(path)
|
|
return eng.key_of(t) if t and not eng.is_watermark(path, size_by.get(path, 0)) else None
|
|
|
|
|
|
for g in groups:
|
|
g["tops"] = {parts_of(r["path"])[0] for r in g["rows"]}
|
|
top_groups = collections.defaultdict(list)
|
|
for g in groups:
|
|
for t in g["tops"]:
|
|
top_groups[t].append(g)
|
|
group_by_key = collections.defaultdict(list)
|
|
for g in groups:
|
|
for r in g["rows"]:
|
|
group_by_key[eng.key_of(r["title"])].append(g)
|
|
group_by_dir = {}
|
|
for g in groups:
|
|
for r in g["rows"]:
|
|
group_by_dir.setdefault(media_dir(r["path"]), g)
|
|
dr = disc_root(r["path"])
|
|
if dr:
|
|
group_by_dir.setdefault(os.path.dirname(dr), g)
|
|
|
|
|
|
def group_words(g):
|
|
"""The words that identify a film group: its titles and the names of the
|
|
folders its files live in."""
|
|
if "words" not in g:
|
|
w = set()
|
|
if g.get("plex"):
|
|
w |= words_of(g["plex"])
|
|
for r in g["rows"]:
|
|
w |= words_of(r["title"])
|
|
w |= words_of(os.path.basename(media_dir(r["path"])))
|
|
g["words"] = w
|
|
return g["words"]
|
|
|
|
|
|
def score_words(text, groups):
|
|
"""Ranks film groups by how many of their words a path carries."""
|
|
words = words_of(text)
|
|
out = [(len(words & group_words(g)), g) for g in groups]
|
|
out.sort(key=lambda x: -x[0])
|
|
return out
|
|
|
|
|
|
def group_for_orphan(path):
|
|
"""Which movie folder an extra/sidecar belongs to. A collection folder holding
|
|
several films must be resolved by WORDS, never by position: taking the first
|
|
film of the folder attached every loose subtitle in a box set to whichever
|
|
movie happened to sort first."""
|
|
stem = os.path.splitext(os.path.basename(path))[0]
|
|
key = attachable_title(path)
|
|
if key and group_by_key.get(key):
|
|
cands = group_by_key[key]
|
|
top = parts_of(path)[0] if len(parts_of(path)) > 1 else ""
|
|
same_top = [g for g in cands if any(parts_of(r["path"])[0] == top
|
|
for r in g["rows"] if len(parts_of(r["path"])) > 1)]
|
|
pool = same_top or cands
|
|
return pool[0] if len(pool) == 1 else score_words(stem, pool)[0][1]
|
|
gtop = parts_of(path)[0] if len(parts_of(path)) > 1 else ""
|
|
pool = top_groups.get(gtop, [])
|
|
if len(pool) > 1: # a loose collection folder (box set, bonus
|
|
n, g = score_words(stem + " " + os.path.dirname(path), pool)[0]
|
|
return g if n >= 2 else None # album) is resolved by words, not position
|
|
if len(pool) == 1:
|
|
return pool[0]
|
|
d = os.path.dirname(path)
|
|
while d and d != ROOT:
|
|
if d in group_by_dir:
|
|
return group_by_dir[d]
|
|
d = os.path.dirname(d)
|
|
return None
|
|
|
|
|
|
tv_tops = {parts_of(p)[0] for p in tv_set}
|
|
def film_folder_for(path):
|
|
"""A movie folder for a lone feature that has no group: derived from its own
|
|
title when a year proves it, else None (goes to the review pile)."""
|
|
t, y = eng.title_of(path)
|
|
if not t or y == "?":
|
|
return None
|
|
d = uniq_dir(ROOT, sanitize(f"{t} ({y})"), "", "")
|
|
mkdirs.add(d)
|
|
return d
|
|
|
|
|
|
for p in sorted(extras):
|
|
if disc_root(p):
|
|
continue
|
|
if len(parts_of(p)) > 1 and parts_of(p)[0] in tv_tops:
|
|
continue # a special inside a TV set: staged with it
|
|
g = group_for_orphan(p)
|
|
ext = ext_by.get(p, os.path.splitext(p)[1].lower())
|
|
stem = os.path.splitext(os.path.basename(p))[0]
|
|
if g is None:
|
|
solo = film_folder_for(p)
|
|
if solo:
|
|
dst = f"{solo}/{sanitize(stem)}{ext}"
|
|
problems.append(f"REVIEW lone feature {rel(p)}: gets its own folder "
|
|
f"{os.path.basename(solo)}")
|
|
else:
|
|
dst = f"{ROOT}/_unsorted bonus/{extras_kind(p)}/{sanitize(stem)}{ext}"
|
|
problems.append(f"NO-FILM extra {rel(p)}")
|
|
else:
|
|
dst = f"{g['dir']}/{extras_kind(p)}/{sanitize(stem)}{ext}"
|
|
book(p, uniq_dir(os.path.dirname(dst), os.path.basename(dst)[:-(len(ext) or 1)],
|
|
ext, os.path.dirname(p)), "extra")
|
|
|
|
# Disc trees the engine never saw (a MakeMKV scratch folder, an AACS remnant):
|
|
# they belong to the film of the folder they sit in, kept as a named subtree.
|
|
for q in rows:
|
|
pth = q["path"]
|
|
dr = disc_root(pth)
|
|
if not dr or pth in by_path or pth in booked_src:
|
|
continue
|
|
g = group_by_dir.get(dr) or (top_groups[parts_of(pth)[0]][0]
|
|
if len(top_groups.get(parts_of(pth)[0], [])) == 1 else None)
|
|
if g is None:
|
|
continue
|
|
inner = os.path.basename(dr)
|
|
problems.append(f"REVIEW disc remnant {rel(dr)} moves inside {g['base']} "
|
|
f"(a second master: compare before keeping both)")
|
|
for src in disc_children[dr]:
|
|
if src not in booked_src:
|
|
book(src, f"{g['dir']}/{inner}/{os.path.relpath(src, dr)}", "disc-file")
|
|
|
|
|
|
# ----------------------------------------------------------------- sidecars
|
|
media_by_dir = collections.defaultdict(list)
|
|
for r in erows:
|
|
if r["path"] in marks:
|
|
continue
|
|
dst = r.get("dst")
|
|
if dst:
|
|
media_by_dir[media_dir(r["path"])].append(r)
|
|
|
|
|
|
|
|
def sidecar_target(path, e):
|
|
d = os.path.dirname(path)
|
|
stem = os.path.splitext(os.path.basename(path))[0]
|
|
if d == ROOT:
|
|
# A file loose in the library ROOT names no film. media_by_dir[ROOT] is the
|
|
# set of root-level MOVIE files, and scoring a stray name against it
|
|
# attached every root-level leftover to whichever root-level film was
|
|
# largest — the plan sent ".DS_Store" into "Titanic (1997)/other/" that
|
|
# way. A root-level stray is nobody's sidecar: it goes to the review pile.
|
|
return None, None
|
|
cands = media_by_dir.get(d, [])
|
|
if not cands:
|
|
nd = os.path.dirname(d) # a subs/ or Extras/ folder beside the media
|
|
while nd and nd != ROOT and not media_by_dir.get(nd):
|
|
nd = os.path.dirname(nd)
|
|
if media_by_dir.get(nd):
|
|
if nd != d: # a NAMED subfolder (Subs/, Extras/, the
|
|
g = group_for_orphan(path) # box set's own root) carries evidence
|
|
dirs = {os.path.dirname(r.get("dst", "")) for r in media_by_dir[nd]}
|
|
if g is None or g["dir"] not in dirs:
|
|
# ...but only a STRONG word match outranks the folder the file
|
|
# sits in. A weak one is an accident: "3_Eng.srt" scored 1
|
|
# against a junk group named "3" and escaped its own movie's
|
|
# Subs/ folder, while "03 xXx 3 Return Of Xander Cage" scores 4
|
|
# and genuinely belongs to the other film of the set.
|
|
n = score_words(stem, [g])[0][0] if g else 0
|
|
if n >= 2:
|
|
return (g, None)
|
|
cands = media_by_dir[nd]
|
|
if not cands:
|
|
g = group_for_orphan(path)
|
|
return (g, None) if g else (None, None)
|
|
if len(cands) > 1:
|
|
w = [t for t in eng.norm(stem).split() if t not in NOISE]
|
|
cands = sorted(cands, key=lambda r: (
|
|
-len([t for t in eng.norm(os.path.basename(r["path"])).split() if t in w]),
|
|
-r["bytes"]))
|
|
return (None, cands[0])
|
|
|
|
|
|
for q in rows:
|
|
p, e = q["path"], q["ext"]
|
|
if p in by_path or p in extras or p in marks or is_qb(p) or disc_root(p):
|
|
continue
|
|
if os.path.basename(p).startswith("._"):
|
|
# macOS AppleDouble resource forks: a few hundred bytes carrying a name
|
|
# COPIED FROM ANOTHER FILM. Word-matching them to "their" film scattered
|
|
# dead files through four movie folders (and they are video extensions in
|
|
# the AppleDouble case, so they also skipped every other filter). They
|
|
# claim nothing: one pile, and they stay in it — nothing is deleted here.
|
|
book(p, uniq_dir(f"{ROOT}/_macos sidecars", sanitize(os.path.basename(p))[:120],
|
|
"", os.path.dirname(p)), "junk")
|
|
problems.append(f"JUNK AppleDouble sidecar {rel(p)} (macOS resource fork)")
|
|
continue
|
|
if e == ".iso": # a DVD image: Jellyfin plays it as a movie file
|
|
g = group_for_orphan(p)
|
|
stem = os.path.splitext(os.path.basename(p))[0]
|
|
if g is not None:
|
|
book(p, uniq_dir(g["dir"], g["base"], e, stem) if g.get("primary") is not None
|
|
else f"{g['dir']}/{g['base']}{e}", "iso")
|
|
else:
|
|
solo = film_folder_for(p)
|
|
if solo:
|
|
book(p, f"{solo}/{sanitize(stem)}.iso", "iso")
|
|
problems.append(f"REVIEW disc image {rel(p)}: DVD rip, alone in its "
|
|
f"folder -> {os.path.basename(solo)}")
|
|
else:
|
|
book(p, f"{ROOT}/_unsorted files/{sanitize(os.path.basename(p))}", "orphan")
|
|
problems.append(f"NO-FILM disc image {rel(p)}")
|
|
continue
|
|
if e in VIDEO_EXT:
|
|
# A video the engine never probed (unreadable header) and not an AppleDouble
|
|
# remnant: the plan does not guess at it, so it stays where it is.
|
|
continue
|
|
stem = os.path.splitext(os.path.basename(p))[0]
|
|
g, media = sidecar_target(p, e)
|
|
if media is not None:
|
|
base = os.path.splitext(os.path.basename(media["dst"]))[0]
|
|
d = os.path.dirname(media["dst"])
|
|
elif g is not None:
|
|
base, d = g["base"], g["dir"]
|
|
else:
|
|
# Nothing names this file's film. NEVER scatter it: an archive set or an
|
|
# album tree must move as the unit it is, so the whole loose folder goes
|
|
# to the review pile and a film folder's own leftovers stay with it.
|
|
ps = parts_of(p)
|
|
if len(ps) < 2:
|
|
dst = f"{ROOT}/_unsorted files/{sanitize(stem)}{e}"
|
|
elif len(top_groups.get(ps[0], [])):
|
|
dst = f"{top_groups[ps[0]][0]['dir']}/other/{'/'.join(sanitize(c) for c in ps[1:])}"
|
|
else:
|
|
dst = f"{ROOT}/_unsorted files/{'/'.join(sanitize(c) for c in ps)}"
|
|
book(dst=dst, src=p, kind="orphan")
|
|
problems.append(f"NO-OWNER sidecar {rel(p)}")
|
|
continue
|
|
if e in IMG_EXT:
|
|
low = eng.norm(stem)
|
|
word = next((w for w in IMG_PRIMARY if re.search(rf"\b{w}\b", low)), None)
|
|
kind = "poster" if word else next(
|
|
(IMG_OTHER[w] for w in IMG_OTHER if re.search(rf"\b{w}\b", low)), None)
|
|
if re.search(r"\btheme\b", low):
|
|
dst = f"{d}/theme{e}"
|
|
else:
|
|
dst = f"{d}/{base}-{kind or 'art'}{e}" if kind else f"{d}/{sanitize(stem)}{e}"
|
|
book(p, uniq_dir(os.path.dirname(dst), os.path.basename(dst)[:-(len(e) or 1)],
|
|
e, os.path.dirname(p)), "image")
|
|
elif e in NFO_EXT:
|
|
book(p, uniq_dir(d, base, ".nfo", os.path.dirname(p)), "nfo")
|
|
elif e in SUB_EXT | AUDIO_EXT:
|
|
toks = [t for t in re.split(r"[._\-\s]+", stem) if t]
|
|
langs = [t.lower() for t in toks if t.lower() in LANGS]
|
|
flags = [t.lower() for t in toks if t.lower() in FLAGS]
|
|
keep = [sanitize(t) for t in toks
|
|
if t.lower() not in langs + flags + list(NOISE)
|
|
and not re.fullmatch(r"\d{3,4}p?|cd\d+|\d{4}|\d+ch|\d+.\d", t.lower())]
|
|
bits = [base] + ([sanitize(" ".join(keep[:4]))] if keep else []) + langs + flags
|
|
if e in SUB_EXT:
|
|
# subtitles: <movie>/srt[/language]
|
|
d = f"{d}/srt"
|
|
pd = os.path.basename(os.path.dirname(p))
|
|
if SUB_KEEP_RE.match(pd) and not SUB_DIR_OK.match(pd):
|
|
d = f"{d}/{sanitize(pd)}" # a real language folder survives
|
|
elif e in AUDIO_EXT and AUDIO_DIR_RE.search(os.path.dirname(p)):
|
|
d = f"{d}/theme-music" # bonus audio: one canonical folder
|
|
pd = os.path.basename(os.path.dirname(p))
|
|
if not MUSIC_TOP_RE.search(pd): # an album folder keeps its name
|
|
d = f"{d}/{sanitize(pd)}"
|
|
else:
|
|
pd = os.path.basename(os.path.dirname(p))
|
|
if not AUDIO_DIR_OK.match(pd):
|
|
d = f"{d}/{sanitize(pd)}"
|
|
stem2 = ".".join(bits) or sanitize(os.path.basename(p))
|
|
book(p, uniq_dir(d, stem2, e, os.path.basename(p)), "track")
|
|
else:
|
|
book(p, uniq_dir(f"{d}/other", sanitize(os.path.basename(p))[:120], e,
|
|
os.path.dirname(p)), "misc")
|
|
|
|
readmes.append(f"README in {ROOT}: this library was relaid out into Jellyfin's "
|
|
"one-folder-per-movie layout; `_release-group watermarks/`, "
|
|
"`_unsorted files/` and `_unsorted bonus/` hold files no film claims.")
|
|
for key in tv_keys:
|
|
readmes.append(f"README in {ROOT}/{sanitize('_TV needs a TV library - ' + key)}: "
|
|
"these are TV episodes found in the MOVIES library; move the folder "
|
|
"to a TV library source, else Jellyfin groups them wrongly as one "
|
|
"movie.")
|
|
|
|
# --------------------------------------------------------------------- TV
|
|
for key in tv_keys:
|
|
items = eps[key]
|
|
year = collections.Counter(r["year"] for r in items if r["year"] != "?")
|
|
yr = year.most_common(1)[0][0] if year else None
|
|
base = f"_TV needs a TV library - {sanitize(key)}" + (f" ({yr})" if yr else "")
|
|
tops = {parts_of(r["path"])[0] for r in items}
|
|
members = sorted([r["path"] for r in items]
|
|
+ [q for q in extras | marks if len(parts_of(q)) > 1
|
|
and parts_of(q)[0] in tops],
|
|
key=lambda q: q)
|
|
for src in members:
|
|
rel_parts = rel(src).split("/")
|
|
tail = "/".join(sanitize(c) for c in rel_parts[:-1] + [rel_parts[-1]])
|
|
book(src, f"{ROOT}/{base}/{tail}", "tv")
|
|
for p in sorted(marks):
|
|
stem, ext = os.path.splitext(os.path.basename(p))
|
|
book(p, uniq_dir(f"{ROOT}/_release-group watermarks", sanitize(stem)[:120], ext,
|
|
os.path.dirname(p)), "watermark")
|
|
for q in rows:
|
|
if is_qb(q["path"]):
|
|
problems.append(f"SKIPPED qBittorrent partial {rel(q['path'])}")
|
|
book(q["path"], q["path"], "qb-partial") # stays: resume data
|
|
|
|
# --------------------------------------------------------------- debug hook
|
|
if os.environ.get("JF_ISO"):
|
|
x = ROOT + "/High Anxiety (Mel Brooks) [1977] nordic subs DVDrip/High Anxiety [1977].iso"
|
|
print("attachable", attachable_title(x), "| dir", os.path.dirname(x) in group_by_dir,
|
|
"| g", group_for_orphan(x) and group_for_orphan(x)["base"])
|
|
raise SystemExit
|
|
if os.environ.get("JF_DEBUG"):
|
|
import json
|
|
unp = sorted({r["path"] for r in rows} - {m[0] for m in moves} - at_target)
|
|
print("at_target", len(at_target))
|
|
print("kinds-unplanned", collections.Counter(os.path.splitext(u)[1] for u in unp).most_common(8))
|
|
for up in unp[:20]:
|
|
d = os.path.dirname(up)
|
|
print("UNP", up[:90], "| mediadir?", d in media_by_dir, "| by_path", up in by_path,
|
|
"| extras", up in extras, "| discroot", disc_root(up), "| ext", os.path.splitext(up)[1])
|
|
|
|
# -------------------------------------------------------------- validation
|
|
srcs = [m[0] for m in moves]
|
|
dup_src = {s for s, n in collections.Counter(srcs).items() if n > 1}
|
|
for s in dup_src:
|
|
problems.append(f"DOUBLE-CLAIM source {rel(s)}")
|
|
all_src = set(srcs)
|
|
untouched = {r["path"] for r in rows} - all_src
|
|
bad_names, into_alive, lost = [], [], []
|
|
for src, dst, kind in moves:
|
|
for comp in dst[len(ROOT) + 1:].split("/"):
|
|
if (BAD_CHARS.search(comp) or comp != comp.strip(". ") or not comp
|
|
or len(comp) > 200):
|
|
bad_names.append(f"{kind}: {comp!r} in {rel(dst)}")
|
|
if dst in untouched: # something staying put is there
|
|
into_alive.append(f"{rel(src)} -> {rel(dst)} (occupied by a file that stays)")
|
|
if dst.lower() in {s.lower() for s in untouched}:
|
|
into_alive.append(f"{rel(src)} -> {rel(dst)} (case-collides with a survivor)")
|
|
if dst.startswith(src + "/") and src in all_src:
|
|
lost.append(f"{rel(src)} moves into its own subtree")
|
|
# every accounted-for source must exist exactly once in the plan
|
|
covered = {r["path"] for r in rows}
|
|
not_planned = sorted(covered - all_src - at_target)
|
|
|
|
with open(PLAN, "w") as out:
|
|
out.write("action\tsrc\tdst\tkind\n")
|
|
for d in sorted(mkdirs, key=lambda q: (q.count("/"), q)):
|
|
if d.startswith(ROOT + "/") and d != ROOT:
|
|
out.write(f"mkdir\t\t{d}\tdir\n")
|
|
for p in sorted(booked_src - {m[0] for m in moves}):
|
|
out.write(f"keep\t{p}\t{p}\n")
|
|
for src, dst, kind in sorted(moves, key=lambda m: m[0]):
|
|
out.write(f"move\t{src}\t{dst}\t{kind}\n")
|
|
|
|
kinds = collections.Counter(k for _s, _d, k in moves)
|
|
films_n = sum(1 for _s, _d, k in moves if k == "film")
|
|
folders = len({os.path.dirname(d) for _s, d, k in moves if k in ("film", "disc-file")})
|
|
with open(EXCEPTS, "w") as out:
|
|
out.write(f"groups={len(groups)} moves={len(moves)} folders={folders}\n")
|
|
out.write(f"kinds={dict(kinds)}\n")
|
|
out.write(f"at-target-already={len(at_target)} unplanned={len(not_planned)}\n")
|
|
out.write("\n=== WHAT THE EXECUTOR MUST ALSO WRITE ===\n")
|
|
out.write("\n".join(sorted(set(readmes))) + "\n")
|
|
out.write("\n=== PROBLEMS ===\n")
|
|
out.write("\n".join(sorted(set(problems))) + "\n")
|
|
out.write("\n=== BAD NAMES ===\n" + "\n".join(sorted(set(bad_names))) + "\n")
|
|
out.write("\n=== TARGETS OCCUPIED BY SURVIVORS ===\n"
|
|
+ "\n".join(sorted(set(into_alive))) + "\n")
|
|
out.write("\n=== LEFTOVER SOURCES (first 60) ===\n"
|
|
+ "\n".join(rel(p) for p in not_planned[:60]) + "\n")
|
|
print(f"groups={len(groups)} folders={folders} moves={len(moves)} "
|
|
f"mkdirs={len(mkdirs)} film-files={films_n}")
|
|
print(f"kinds={dict(kinds)}")
|
|
print(f"problems={len(set(problems))} bad-names={len(set(bad_names))} "
|
|
f"occupied={len(set(into_alive))} double-claim={len(dup_src)}")
|
|
print(f"left-unmoved={len(not_planned)}")
|
|
print(f"tv-keys={len(tv_keys)} extras={len(extras)} marks={len(marks)} "
|
|
f"disc-trees={len(disc_children)}")
|