#!/usr/bin/env python3 r"""The fleet's release-filename title parser: the grouping authority the relayout plan uses for every file the media server never matched. report: media-titles.py report [--probed TSV] [--out TSV] the similar-copies census (DUP / REVIEW / EP-DUP groups) over the ffprobe census scripts/media-census.py writes. parse: media-titles.py parse NAME [NAME…] what title_of() makes of a name — the way to check a naming accident before it merges two films into one folder. scripts/media-plan.py imports this file; over there the paths are arguments. Method — three independent signals, so a naming accident cannot merge two films: * franchise KEY : first two significant title tokens. Stable across parts of a collection ("resident evil", "jump street") but NOT film-specific — it only selects candidates. * DURATION : the identity check. Same master encodes differ by at most a couple of seconds; different films / cuts do not collide. * SIZE SPREAD + YEAR: decide the tier. Two files same size AND same duration are byte-twins (the hash pass confirms); same duration but different sizes are two different releases of one film (deletable redundancy). TV episodes (SxxEyy in the name) are grouped separately: their uniform runtime would otherwise collapse a whole series into one "duplicate" group. """ import argparse import collections, os, re, sys, unicodedata # Library root: the folder directly under this one names the film, and every # folder below it is structure (Extras/, Featurettes/, Disc 1/, BDMV/). Tests pass # their own; nothing here reads media, only the census TSVs about it. ROOT = os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies") # Release-tag vocabulary: never part of a title, so the title stops here. JUNK = {"5.1", "7.1", "2.0", "1.0", "6.1", "4.0", "extras"} JUNK_RE = re.compile( r"^(" r"\d{3,5}p|\d{1,2}k|uhd|fhd|imax|x ?26[45]|hevc|h ?26[45]|avc|av1|x ?av1|vp[89]" r"|divx|xvid|mpeg-?2?|\d{1,2}-?bit|hdr ?10\+?|hdr|dolby ?vision|dvh?|sdr|hybrid" r"|re-?mux|bd-?re-?mux|blu-?ray|bdrip|brrip|bdmux|bd|web-?(rip|dl)?|hdtv|hdrip" r"|dvd-?(rip|r|scr)?|hdcam|telesync|screener|ppv|re-?encode|extras?" r"|ddp? ?\d(\.\d)?|dd\d(\.\d)?|dts-?hd(\.ma)?|dts|truehd|atmos|e-?ac-?3?|ac-?3" r"|aac\d?(\.\d)?|lpcm|flac|opus|mp3|pcm|\d\.\d(\.\d)?|5 ?1|7 ?1|2 ?0|1 ?0|6 ?1|4 ?0" r"|subs?-?(it|en|eng|ita)?|subbed|dual(-?audio)?|multi|dub(bed)?" r"|ita|eng|en|it|de|fr|es|pl|jp|jap|lat|esp|ger|fre|rus|zho|hin|tam|tel" r"|repack|proper|internal|limited|unrated|extended|director'?s|cut|remaster(ed)?|uncut" r"|special|edition|version|crc32|sha1|sfv|par2|nfo|nzb|mux|complete|unpack" r"|certified|scene|p2p|part\d*|cd\d+|disc\d+|disk\d+|vob|idx|srt|ass|ssa|sup" r"|m3u|cue|log|txt|mkv|mp4|avi|imdb\d[\d.]*|yts(\.\w+)?|rarbg|collection" r")$", re.I) RES_RE = re.compile(r"^\d{3,5}p$|^\d{1,2}k$|^\d+x\d+$", re.I) YEAR_RE = re.compile(r"^(19|20)\d{2}$") SEP_RE = re.compile(r"[._\s\-\[\](){}+!~,;:&@#/*'\"`´|]+") IDX_RE = re.compile(r"^\d{1,3}[a-z]?$") WORD_RE = re.compile(r"[A-Za-z0-9']+$") EP_RE = re.compile(r"(?i)(? 1 # "2001…1968": one of them is the title toks, year = [], None for t in raw: if YEAR_RE.match(t): year = t # last year wins toks.append("\x00") else: toks.append(t) stop, seen_word = len(toks), False for i, t in enumerate(toks): if t == "\x00": continue c = norm(re.sub(r"[^A-Za-z0-9']", "", t)) if (not c or RES_RE.match(c) or JUNK_RE.match(c) or c in JUNK or not WORD_RE.match(c)) and not (IDX_RE.match(t) and not seen_word): stop = i break if c.isalpha(): seen_word = True # A leading number is a track index ("01. Bye Bye Baby", "05. Casablanca") unless # it is three digits or more ("007", "000", "1408") or the name carries no year # at all ("12 Monkeys", "831"). A 1-2 digit prefix beside a year is an index; # the rare real title that looks like one ("0 Things I Hate About You (1999)") # loses its zero to the parser — Plex's matched title overrides it anyway. lead = re.split(r"[\s._-]", name, maxsplit=1)[0] track_idx = bool(re.fullmatch(r"\d{1,2}[a-z]?", lead)) and bool(year) keep, num_title = [], None for t in toks[:stop]: if t == "\x00": if keep: break # year closes the title if two_year and t is not None and years[0] != year: # two DIFFERENT years ("2001…1968"): the leading one is the title; # "1940 - Fantasia (1940)" repeats one year, so it is not a title num_title = num_title or years[0] continue # leading year: not the title if track_idx and IDX_RE.match(t) and not keep and t == lead: continue # "03." / "1c." ordering c = norm(re.sub(r"[^A-Za-z0-9']", "", t)) if not keep and not c.isalpha(): num_title = num_title or (c or None) continue # numeric title candidate keep.append(c) if len(keep) >= 8: break keep = [k for k in keep if not re.fullmatch(r"(disc|disk|cd|dvd|part)\d*", k)] title = " ".join(keep) if not keep: # nothing collected: a pure-numeric leading token is the title even when it # sits where the release year sits ("1917.2019.2160p…"), and "300.2006…" too for t in raw: c = norm(re.sub(r"[^A-Za-z0-9']", "", t)) if re.fullmatch(r"(?:[12]\d{3}|\d{1,3})", c) and (c != year or two_year): return c, year or "?" # never "2160p" / "1080": they are not pure digits if (num_title and title and not keep[0].isdigit() and (two_year or not track_idx)): # a leading number that only echoes the release year ("1917.2019.2160p…") is # not a title; one that stands alone ("300.2006…", "1900") is title = f"{num_title} {title}" title = title or num_title or norm(name).replace("\x00", " ").strip()[:40] return title, year or "?" # Bonus-material vocabulary: names that say nothing about the film, so an extras # file otherwise pairs with an extras file of another movie. "Short", "Spot" and # "Clip" are deliberately absent — they are film titles (Short Circuit, Spot, # Clips) — and the running-time guard in is_extra() carries the rest. SEP_CLS = r"[\s._-]+" def _bonus_rx(word): return r"(? bonus. * ANY folder of the path below the film's own root folder carries bonus vocabulary ("Alien (1979)/Extras/", ".../Featurettes/Deleted Scenes/", "Angel's Egg (1985)/04b. Angel's Egg Extras/") -> bonus. The ROOT folder is exempt because it names the film, and a film may be titled with a bonus word; and a file whose own name IS that folder's film name is the feature, not the bonus ("Ghost in the Shell Movies/Extras/ Ghost in the Shell - CM3.mp4" is a bonus, but a whole other film filed there stays a film). Bonus vocabulary is matched as whole words only, so the film "Angel's Egg - Extra - Logo" inside "04b. Angel's Egg Extras" is reached by the folder rule, and "…Extras" as a fragment of a word is never evidence. """ parts = [q for q in path.split("/") if q] if not parts: return False name = (os.path.splitext(parts[-1])[0] if parts[-1].lower().endswith(VIDEO_SUFFIX) else "") # A bonus folder anywhere BELOW the film's own folder decides for its # contents; the film's own folder is the film's name, whatever words it # contains, so it is exempt. It is the first component AFTER ROOT's own # components — those are the mount path, not folders of the library. head = len([q for q in root.split("/") if q]) + 1 bonus_folder = False for folder in parts[head:-1]: if folder_is_bonus(folder): # The folder's words are often the film's own name ("Angel's Egg # Extras/"), and then they are not bonus evidence. Only WORDS can name # a film: "extras" is release junk that name_words drops, so a folder # like "01. Extras" reduces to its index — and an index matching an # index ("NCOP 01") is a coincidence, not a title. nw = [w for w in name_words(name) if not w.isdigit()] fw = [w for w in name_words(folder) if not w.isdigit()] bonus_folder = not fw or not mentions(nw, fw) break if dur and dur > SHORT_MAX: # Runtime vetoes vocabulary, with ONE exception: an explicit bonus folder # and no metadata match. A 90-minute "Feature Length Rough Cut" sitting in # a movie's Featurettes/ is a disc extra, and nothing in the library claims # it as a film. Long-form bonus words still describe a film, so a # feature-length commentary cut or documentary keeps its runtime veto. if bonus_folder and not matched and not EXTRA_LONG_NAME_RE.search(name): return True return False if EXTRA_NAME_RE.search(name): return True if dur and dur <= LONG_BONUS_MAX and EXTRA_LONG_NAME_RE.search(name): return True return bonus_folder # Release-group watermark clips: a group drops a tiny ad ("ETRG.mp4", # "RARBG.com.mp4") into many different films, so its name carries no film # identity at all and it would otherwise pair with itself across the library. WATERMARK_MAX = 20 * 2**20 WATERMARK_RE = re.compile(r"(?i)^(rarbg(\.com)?|etrg|tgx|fbt|cpbg|yify|yts|" r"rarbg\.com|playnomore|imprs|gauntlet|dfx|macmote|" r"psi-axiz|acm|tlf|sa8|ebp|gtv|w4g|don|aAF|enjoy|hd4u)$") def is_watermark(path, size): """True for a release group's own watermark video, not a film or an extra.""" return (size < WATERMARK_MAX and bool(WATERMARK_RE.match(os.path.splitext(os.path.basename(path))[0]))) def key_of(title): """Franchise key: first two alphabetic tokens (numeric-only title = itself). Deliberately NOT film-specific — it only selects candidates for the duration check, so a two-token collision (two films starting "star trek") cannot by itself create a duplicate group.""" toks = [t for t in title.split() if t] words = [t for t in toks if not t.isdigit()] if not words: return " ".join(toks) sig = [t for t in words if t not in STOP] or words return " ".join(sig[:2]) def ep_of(path): """(series, episode-token) when the basename carries SxxEyy / 1x04 / Part N. The series is the name BEFORE the marker. The box-set convention writes "Monk - S05E03 - Mr. Monk and the Big Game", and the episode title that follows the marker is not the show's name: keying on the whole title gave every episode of one series its own group, so a 124-episode box set never reached the size that calls something a series and each episode was filed as its own film. A file named only by its marker has nothing before it, so its whole title stands in.""" name = os.path.splitext(os.path.basename(path))[0] m = EP_RE.search(name) if not m: return None head = name[:m.start()].strip(" ._\u2013-") series = title_of(head)[0] if head else None return series or title_of(path)[0], norm(m.group(1)).replace(" ", "") def base_words(title, key): """Title tokens beyond the franchise key — the FILM identity part. "evil dead rise" under key "evil dead" -> {rise}; "28 days later" -> {}.""" kt = set(key.split()) return frozenset(t for t in title.split() if t not in kt and t not in STOP and not t.isdigit() and not YEAR_RE.match(t)) def folder_year(path): """Soft year evidence: a 19xx/20xx token in any parent directory name (collection folders are written both "(2002)" and "[2009]"; a range such as "(1998-2000)" deliberately yields nothing).""" for d in reversed(os.path.dirname(path).split("/")): ys = [y for y in re.split(r"[^0-9]+", d) if YEAR_RE.match(y)] if len(ys) == 1: return ys[0] return None def same_film(a, b): """Two files are copies of ONE film only when the film-identity tokens agree (one side may carry none) AND no year evidence contradicts them. The key says the franchise matches; this says the actual film matches — it is what keeps 'Evil Dead' (2013) apart from 'Evil Dead Rise' (2023) and the four Austin Powers apart when their runtimes happen to agree.""" ba, bb = a["base"], b["base"] if ba and bb and ba != bb: return False ya = a["year"] if a["year"] != "?" else a["soft"] yb = b["year"] if b["year"] != "?" else b["soft"] if ya and yb and ya != yb: return False return True def dur_close(a, b): if a <= 0 or b <= 0: return True # unknown duration: let titles decide return abs(a - b) <= max(2.0, 0.003 * max(a, b)) # Blu-ray folder trees: ffprobe only sees the clip it is pointed at, so each tree # is folded into ONE entry, described by its longest main-feature clip (measured # with ffprobe on the largest BDMV/STREAM file) and sized by the whole tree. BDMV_MAIN = { "Avengers.Age.of.Ultron.2015.2160p.BluRay.HEVC.TrueHD.7.1.Atmos-TERMiNAL": dict(dur=8478.2, vcodec="hevc", res="3840x2160", pix="yuv420p10le", fps=23.976), "Full.Metal.Jacket.1987.2160p.BluRay.HEVC.DTS-HD.MA.5.1-EATDIK": dict(dur=6993.3, vcodec="hevc", res="3840x2160", pix="yuv420p10le", fps=23.976), } def load(probed): """Probe rows -> film entries. AppleDouble (._*) sidecars are dropped; each BDMV tree collapses to one entry carrying the tree's total size.""" trees = {} rows = [] for line in open(probed): f = line.rstrip("\n").split("\t") if len(f) < 12: continue b, p, dur, vc, res, fps, vbit, pix, aud, subs, _d, fmt = f[:12] base = os.path.basename(p) if base.startswith("._") or base == ".DS_Store": continue # macOS sidecar, not media m = re.search(r"/(BDMV|VIDEO_TS)/", p) root = p[:m.start()] if m else None if root or re.search(r"/(CERTIFICATE|AACS|MLPI|CLIPINF|PLAYLIST|BACKUP|REGISTRL)/", p): if not root: continue # disc bookkeeping: folds into its tree, never a film if root: t = trees.setdefault(root, dict(bytes=0, audio=set(), subs=0)) t["bytes"] += int(b) t["audio"].update(a for a in aud.split("+") if a) t["subs"] += int(subs or 0) continue try: dur = float(dur) except ValueError: dur = 0.0 rows.append(dict(bytes=int(b), path=p, dur=dur, vcodec=vc, res=res, fps=fps, vbit=int(float(vbit or 0)), pix=pix, audio=aud, subs=subs, fmt=fmt)) for root, t in trees.items(): folder = os.path.basename(root) m = BDMV_MAIN.get(folder, {}) rows.append(dict(bytes=t["bytes"], path=root + "/BDMV/index.bdmv", name_src=folder, dur=m.get("dur", 0.0), vcodec=m.get("vcodec", "hevc"), res=m.get("res", "-"), fps=m.get("fps", 0), vbit=0, pix=m.get("pix", "-"), audio="+".join(sorted(t["audio"])), subs=str(t["subs"]), fmt="BDMV tree")) for r in rows: r["title"], r["year"] = title_of(r.get("name_src") or r["path"]) r["key"] = key_of(r["title"]) r["base"] = base_words(r["title"], r["key"]) r["soft"] = folder_year(r["path"]) r["ep"] = ep_of(r["path"]) return rows def dur_clusters(items): """Group files that are the SAME FILM and the SAME MASTER duration. Candidate buckets come from the franchise key; same_film then rejects the different installments whose runtimes happen to agree.""" clusters = [] for r in sorted(items, key=lambda r: -r["dur"]): for c in clusters: if dur_close(c[0]["dur"], r["dur"]) and same_film(c[0], r): c.append(r) break else: clusters.append([r]) return clusters def tier(groups): """Split same-film/same-duration clusters: DUP = different releases of one film (real, deletable redundancy); REVIEW = identical size AND duration (byte-twins — the hash pass is the authority) or clashing year evidence.""" dup, review = [], [] for g in groups: g.sort(key=lambda r: -r["bytes"]) years = {(r["year"] if r["year"] != "?" else r["soft"]) for r in g} years.discard(None) spread = (g[0]["bytes"] - g[-1]["bytes"]) / g[0]["bytes"] if g[0]["bytes"] else 0 if len(years) > 1 or spread < 0.005: review.append(g) else: dup.append(g) return dup, review def redun(g): return sum(r["bytes"] for r in sorted(g, key=lambda r: -r["bytes"])[1:]) def line(r): return (f"{r['bytes']/2**30:8.2f} GiB {r['res']:>9} {r['vcodec']:<7} " f"{r['pix']:<11} {r['dur']/60:5.0f}m {r['audio'][:34]:<34}") def dump(tag, groups, out, note=""): for gi, g in enumerate(groups, 1): g.sort(key=lambda r: -r["bytes"]) print(f"\n### {tag}-{gi}. {g[0]['title']} ({g[0]['year']}) — {len(g)} files, " f"{redun(g)/2**30:.1f} GiB redundant {note}".rstrip()) for rank, r in enumerate(g): print(f" {line(r)} [{r['year']}] " f"{r['path'].replace(ROOT + '/', '')}") out.write(f"{tag}\t{gi}\t{rank}\t{r['bytes']}\t{r['res']}\t{r['vcodec']}\t" f"{r['pix']}\t{int(r['dur'])}\t{r['audio']}\t{r['title']}|{r['year']}\t" f"{r['path']}\n") def report(probed, out_path): rows = load(probed) for r in rows: r["mark"] = is_watermark(r["path"], r["bytes"]) r["extra"] = not r["mark"] and is_extra(r["path"], r["title"], r["dur"]) marks = [r for r in rows if r["mark"]] movies = [r for r in rows if not r["ep"] and not r["extra"] and not r["mark"]] episodes = [r for r in rows if r["ep"] and not r["extra"] and not r["mark"]] extras = [r for r in rows if r["extra"]] print(f"census: {len(rows)} video entries = {len(movies)} films + {len(episodes)} " f"TV episodes + {len(extras)} bonus/trailers " f"({sum(r['bytes'] for r in extras)/2**30:.1f} GiB) " f"+ {len(marks)} release-group watermarks") out = open(out_path, "w") buckets = collections.defaultdict(list) for r in movies: buckets[r["key"]].append(r) cand = {k: v for k, v in buckets.items() if len(v) > 1} clusters = [c for v in cand.values() for c in dur_clusters(v) if len(c) > 1] dup, review = tier(clusters) dup.sort(key=lambda g: -redun(g)) review.sort(key=lambda g: -redun(g)) dump("DUP", dup, out) dump("REVIEW", review, out, "— same key+duration, sizes near-identical OR years clash: RECHECK") ebuf = collections.defaultdict(list) for r in episodes: ebuf[(r["ep"][0], r["ep"][1])].append(r) edup, ereview = tier([c for v in ebuf.values() if len(v) > 1 for c in dur_clusters(v) if len(c) > 1]) edup.sort(key=lambda g: -redun(g)) dump("EP-DUP", edup, out) shows = collections.Counter() for r in episodes: shows[r["ep"][0]] += 1 multi = {s: n for s, n in shows.items() if n > 4} print("\n=== TV-series content inside Movies/ (>4 episodes) ===") for s, n in sorted(multi.items(), key=lambda kv: -kv[1]): print(f" {n:4d} episodes {s}") out.close() print(f"\nDUP groups={len(dup)} files={sum(len(g) for g in dup)} " f"redundant={sum(redun(g) for g in dup)/2**40:.2f} TiB", file=sys.stderr) print(f"REVIEW groups={len(review)} " f"redundant={sum(redun(g) for g in review)/2**40:.2f} TiB", file=sys.stderr) print(f"EP-DUP groups={len(edup)} " f"ep-redundant={sum(redun(g) for g in edup)/2**30:.1f} GiB", file=sys.stderr) def parse(names): for n in names: t, y = title_of(n) print(f"{t!r} ({y})\tkey={key_of(t)!r}\t{n[:70]}") def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) sub = ap.add_subparsers(dest="cmd", required=True) rep = sub.add_parser("report", help="similar-copy census over a probe TSV") rep.add_argument("--probed", default="/tmp/media-probed.tsv") rep.add_argument("--out", default="/tmp/media-similar.tsv") par = sub.add_parser("parse", help="show the parsed title/year of a name") par.add_argument("names", nargs="+") a = ap.parse_args() if a.cmd == "report": report(a.probed, a.out) else: parse(a.names) if __name__ == "__main__": main()