#!/usr/bin/env python3 r"""Read-only census of a media library — the inputs a relayout plan is built from. media-census.py enumerate [--root DIR] [--out TSV] media-census.py probe [--enum TSV] [--out TSV] [--jobs N] media-census.py plex [--root DIR] [--db PATH] [--out TSV] media-census.py jellyfin [--root DIR] [--db PATH] [--container PREFIX] [--out TSV] Run it on the machine that mounts the library. Nothing here writes inside the library and nothing is deleted; the only outputs are the TSVs below, and the scripts/media-plan.py and scripts/media-shows.py read them instead of touching media. enum bytes ext path every file under the root probed bytes path dur vcodec WxH fps vbit pix audio subs spare format one row per video-container file; `audio` is "codec:lang+codec:lang", `subs` the subtitle-stream count, `spare` the audio streams carrying neither a default nor a forced tag plex file leaf_title … (see PLEX_COLS) the titles Plex already matched — the authority over any filename guess jellyfin file series year season episode extra tmdb (see JF_COLS) one row per episode Jellyfin has matched and numbered, its path mapped from the container's view back to the host; `year` and `tmdb` are the SERIES' values, `extra` is Jellyfin's own extra type when the row is bonus material rather than an episode music file artist album year disc track title mb performer (see MUSIC_COLS) one row per track Jellyfin has tagged: the tag read is what the row IS, so `artist` is the album artist (the track artist when that is empty) and `album`/`year` are the album item's provider values when the folder is an album item and the tags' otherwise; `mb` is the album's MusicBrainz release group, `performer` the track artist the tags carry `probe` is the slow step (every container is opened for its header, ~10 min for 4,800 files on a spindle array at 6 workers) so it is `nice`d: a census beside a live service must not starve it. `enumerate` is a stat pass, so re-censusing before a plan is cheap — and a plan is only ever as good as these files. """ import argparse import csv import json import os import re import sqlite3 import subprocess import sys from concurrent.futures import ThreadPoolExecutor # Containers ffprobe can read. .iso/.img are deliberately NOT here: a disc image is # a mounted volume, not a stream, and the planner treats it as a movie file instead. PROBE_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg", ".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".bdmv", ".vob"} # Trash and thumbnail caches are not library content, and this repo's trash policy # means .AGENTTRASH can sit inside a movie folder at any time. SKIP_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir|lost\+found)(/|$)") PLEX_DB = ("/var/lib/plexmediaserver/Library/Application Support/Plex Media Server" "/Plug-in Support/Databases/com.plexapp.plugins.library.db") PLEX_COLS = ["file", "leaf_title", "leaf_orig", "leaf_edition", "leaf_year", "parent_title", "parent_orig", "parent_year", "parent_edition", "container", "vcodec", "acodec", "width", "height"] PLEX_SQL = """ select p.file, coalesce(mi.title,''), coalesce(mi.original_title,''), coalesce(mi.edition_title,''), coalesce(mi.year,0), coalesce(par.title,''), coalesce(par.original_title,''), coalesce(par.year,0), coalesce(par.edition_title,''), m.container, m.video_codec, m.audio_codec, m.width, m.height from media_parts p join media_items m on m.id = p.media_item_id join metadata_items mi on mi.id = m.metadata_item_id left join metadata_items par on par.id = mi.parent_id where p.file like ? and p.deleted_at is null and mi.deleted_at is null """ JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db" JF_COLS = ["file", "series", "year", "season", "episode", "extra", "tmdb"] # Jellyfin stores the entity's .NET class as its type; the TV tree is these three. JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode", "MediaBrowser.Controller.Entities.TV.Season", "MediaBrowser.Controller.Entities.TV.Series") JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, IsVirtualItem from BaseItems where Type = ? and Path like ?""") # The music tree is these two entity classes. The track row carries the tags # Jellyfin read out of the file; the album row carries what the provider matched, # which is the better folder name of the two. JF_AUDIO = "MediaBrowser.Controller.Entities.Audio.Audio" JF_ALBUM = "MediaBrowser.Controller.Entities.Audio.MusicAlbum" MUSIC_COLS = ["file", "artist", "album", "year", "disc", "track", "title", "mb", "performer"] MUSIC_SQL = ("""select b.Path, b.AlbumArtists, b.Artists, b.Album, b.ProductionYear, b.ParentIndexNumber, b.IndexNumber, b.Name, b.Id, a.Name, a.ProductionYear, a.Id from BaseItems b left join BaseItems a on a.Id = b.ParentId and a.Type = ? where b.Type = ? and b.Path like ?""") # The album's identity, in preference order. A release group is the album; a bare # release id is one reissue of it. MUSIC_PROVIDERS = ("MusicBrainzReleaseGroup", "MusicBrainzAlbum") def enumerate_lib(root, out): """One row per real file under root: size, extension, path.""" rows = [] for dirpath, dirnames, filenames in os.walk(root): dirnames[:] = [d for d in dirnames if not SKIP_DIR.search(os.path.join(dirpath, d))] for fn in filenames: p = os.path.join(dirpath, fn) if SKIP_DIR.search(p): continue try: st = os.lstat(p) except OSError: continue if not os.path.isfile(p) or os.path.islink(p): continue # links and sockets are not library content rows.append((st.st_size, os.path.splitext(fn)[1].lower(), p)) with open(out, "w") as fh: for size, ext, p in rows: fh.write(f"{size}\t{ext}\t{p}\n") vid = [r for r in rows if r[1] in PROBE_EXT] print(f"files={len(rows)} bytes={sum(r[0] for r in rows)} " f"probes={len(vid)} probe_bytes={sum(r[0] for r in vid)} -> {out}") def probe_one(path): """ffprobe header read -> the probed TSV row (never the whole file).""" try: os.nice(19) # a census must never outrank the media server except OSError: pass try: size = os.stat(path).st_size except OSError: return None cmd = ["ffprobe", "-v", "error", "-print_format", "json", "-show_format", "-show_streams", path] try: done = subprocess.run(cmd, capture_output=True, text=True, timeout=180) data = json.loads(done.stdout or "{}") except (subprocess.TimeoutExpired, subprocess.SubprocessError, json.JSONDecodeError, OSError): data = {} st = data.get("streams", []) fmt = data.get("format", {}) v = next((s for s in st if s.get("codec_type") == "video"), None) aud = [s for s in st if s.get("codec_type") == "audio"] subs = [s for s in st if s.get("codec_type") == "subtitle"] fps = "-" if v and v.get("r_frame_rate"): try: n, d = (int(x) for x in v["r_frame_rate"].split("/", 1)) fps = f"{round(n / d * 100) / 100:g}" if d else "0" except ValueError: pass def num(x): try: return round(float(x)) except (TypeError, ValueError): return 0 return [size, path, num(fmt.get("duration")), (v or {}).get("codec_name", "-"), f"{v['width']}x{v['height']}" if v else "-", fps, num(fmt.get("bit_rate")), (v or {}).get("pix_fmt", "-"), "+".join(f"{a.get('codec_name', '?')}:{a.get('language', '?')}" for a in aud), len(subs), sum(1 for a in aud if not re.search( r"default|forced", json.dumps(a.get("tags", {})), re.I)), fmt.get("format_name", "-") or "-"] def probe(enum, out, jobs): """Probe every video-container row of the enum TSV, in parallel.""" paths = [] with open(enum) as fh: for line in fh: f = line.rstrip("\n").split("\t", 2) if len(f) == 3 and f[1] in PROBE_EXT: paths.append(f[2]) print(f"probing {len(paths)} files with {jobs} workers", flush=True) done = unreadable = 0 with open(out, "w") as fh, ThreadPoolExecutor(max_workers=jobs) as pool: for row in pool.map(probe_one, paths): if row is None: continue if row[3] == "-": unreadable += 1 # header unreadable: the row still gets written done += 1 fh.write("\t".join(str(x) for x in row) + "\n") if done % 250 == 0: print(f" {done}/{len(paths)}", flush=True) print(f"probed={done} unreadable={unreadable} -> {out}") def plex(root, db, out): """Plex's matched titles for files under root, from its DB, read-only.""" uri = f"file:{db}?mode=ro&immutable=1" try: con = sqlite3.connect(uri, uri=True) except sqlite3.Error as e: sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n" "Plex runs rootless here: run this as the container's owner, never " "with sudo — rootless storage is per-user, so root sees nothing.") with open(out, "w", newline="") as fh: w = csv.writer(fh, delimiter="\t", lineterminator="\n") w.writerow(PLEX_COLS) n = 0 for r in con.execute(PLEX_SQL, (root + "/%",)): w.writerow([str(x).replace("\t", " ").replace("\n", " ") for x in r]) n += 1 print(f"plex-matched={n} -> {out}") def jellyfin(root, db, container, out): """The episodes Jellyfin has already numbered, for the library at root. Jellyfin stores each path as the CONTAINER sees it (its quadlet binds the share at /data), so `container` is the in-container prefix that stands for root and every exported path is mapped back to the host before it is written — the planner reads host paths, because it plans moves on the array. The numbering is what matters, not the name: a release group's file name says nothing about which episode a file is, and Jellyfin has already answered that question against a provider. Season/episode come from the episode row; the series name, its year and its TMDb id come from the Series row the episode climbs to, because those are the only place Jellyfin keeps them. """ try: con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) except sqlite3.Error as e: sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n" "It is a WAL database: copy it with its -wal and -shm beside it and" " point --db at the copy. Never sudo it — the data dir belongs to" " the container's owner, and rootless storage is per-user.") have = {r[1] for r in con.execute("pragma table_info(BaseItems)")} need = {"Id", "ParentId", "Type", "Path", "IndexNumber", "ParentIndexNumber", "ExtraType", "IsVirtualItem", "ProductionYear", "Data"} if not need <= have: sys.exit(f"{db} is not a Jellyfin library database (BaseItems has no" f" {','.join(sorted(need - have))})") node = {} for i, pid, typ, name, year, data in con.execute( "select Id, ParentId, Type, Name, ProductionYear, Data from BaseItems" " where Type in (?,?,?)", JF_TV): node[i] = (pid, typ, name, year, data) def series_of(item_id): """Climb episode -> season -> series; empty when the chain is broken.""" for _ in range(4): if item_id not in node: return "", "", "" pid, typ, name, year, data = node[item_id] if typ.endswith(".Series"): try: tmdb = (json.loads(data or "{}") .get("ProviderIds") or {}).get("Tmdb", "") except ValueError: tmdb = "" return name or "", year or "", tmdb item_id = pid return "", "", "" # The fleet's quadlets bind the share at /data, so the container's view of # this library is /data/ unless told otherwise. if not container: container = "/data/" + os.path.basename(os.path.normpath(root)) prefix = container.rstrip("/") + "/" written = skipped = 0 with open(out, "w", newline="") as fh: w = csv.writer(fh, delimiter="\t", lineterminator="\n") w.writerow(JF_COLS) for iid, path, idx, pidx, extra, virt in con.execute(JF_SQL, (JF_TV[0], prefix + "%")): if virt or not path or not path.startswith(prefix): skipped += 1 # a virtual item is a variant, not a file continue sname, syear, tmdb = series_of(iid) w.writerow([root + "/" + path[len(prefix):], sname.replace("\t", " "), syear, "" if pidx is None else pidx, "" if idx is None else idx, (extra or "").replace("\t", " "), tmdb]) written += 1 print(f"episodes={written} virtual-or-blank={skipped} -> {out}") def music(root, db, container, out): """The tracks Jellyfin has already tagged, for the library at root. Jellyfin reads music metadata out of the file itself, so an Audio row IS the tag read, done once, by the vendor's own parser: its album-artist, album, year, disc and track columns are what the embedded tags say and its Name is the track title. That makes this export the whole authority for a music relayout — the path a ripper chose is not evidence of anything, and the folder Jellyfin put a track under is often a release pack, not an album. Two gaps this export answers from the row rather than the tree. Many tracks sit under a folder that is not an album item at all (a disc folder, or a mixed one), so the album name and year fall back to the tags when the album row is absent. And an album's MusicBrainz ids are not in the row's own Data — the current schema leaves that empty for music — they are rows of BaseItemProviders keyed by the item's Id. """ try: con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) except sqlite3.Error as e: sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n" "It is a WAL database: copy it with its -wal and -shm beside it and" " point --db at the copy. Never sudo it — the data dir belongs to" " the container's owner, and rootless storage is per-user.") have = {r[1] for r in con.execute("pragma table_info(BaseItems)")} need = {"Id", "ParentId", "Type", "Path", "Name", "IndexNumber", "ParentIndexNumber", "ProductionYear", "Album", "AlbumArtists", "Artists"} if not need <= have: sys.exit(f"{db} is not a Jellyfin library database with music columns" f" (BaseItems has no {','.join(sorted(need - have))})") # Provider ids, best identity first: the release group is the album, a bare # release id is one reissue of it and never overrides a group. mb = {} if "BaseItemProviders" in {r[0] for r in con.execute( "select name from sqlite_master where type='table'")}: rank = {p: i for i, p in enumerate(MUSIC_PROVIDERS)} best = {} for iid, pid, val in con.execute( "select ItemId, ProviderId, ProviderValue from BaseItemProviders" " where ProviderId in (?, ?)", MUSIC_PROVIDERS): if iid not in best or rank[pid] < best[iid][0]: best[iid] = (rank[pid], val) mb = {k: v[1] for k, v in best.items()} # The fleet's quadlets bind the share at /data, so the container's view of # this library is /data/ unless told otherwise. if not container: container = "/data/" + os.path.basename(os.path.normpath(root)) prefix = container.rstrip("/") + "/" written = skipped = 0 with open(out, "w", newline="") as fh: w = csv.writer(fh, delimiter="\t", lineterminator="\n") w.writerow(MUSIC_COLS) for (path, aal, arts, tag_album, tag_year, disc, track, name, iid, alb_name, alb_year, alb_id) in con.execute( MUSIC_SQL, (JF_ALBUM, JF_AUDIO, prefix + "%")): if not path or not path.startswith(prefix): skipped += 1 # a row for a file outside this library continue # The album artist owns the folder; the track artist is the fallback # for the compilations whose album-artist tag was left empty. w.writerow([root + "/" + path[len(prefix):], ((aal or "").strip() or (arts or "").strip()), (alb_name or tag_album or "").replace("\t", " "), alb_year or tag_year or "", "" if disc is None else disc, "" if track is None else track, (name or "").replace("\t", " "), mb.get(alb_id or "") or mb.get(iid or "") or "", (arts or "").replace("\t", " ")]) written += 1 print(f"tracks={written} outside-root-or-blank={skipped} -> {out}") def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) sub = ap.add_subparsers(dest="cmd", required=True) root = {"default": os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies"), "help": "library root"} e = sub.add_parser("enumerate", help="stat every file under the root") e.add_argument("--root", **root) e.add_argument("--out", default="/tmp/media-enum.tsv") p = sub.add_parser("probe", help="ffprobe every video container") p.add_argument("--enum", default="/tmp/media-enum.tsv") p.add_argument("--out", default="/tmp/media-probed.tsv") p.add_argument("--jobs", type=int, default=6) x = sub.add_parser("plex", help="export Plex's matched titles for the root") x.add_argument("--root", **root) x.add_argument("--db", default=PLEX_DB) x.add_argument("--out", default="/tmp/plex-movies.tsv") j = sub.add_parser("jellyfin", help="export the episodes Jellyfin has numbered for the root") j.add_argument("--root", default=os.environ.get("MEDIA_ROOT", "/mnt/entropy/TVShows"), help="TV library root on the host") j.add_argument("--db", default=JF_DB) j.add_argument("--container", default=None, help="the in-container prefix that stands for --root" " (default /data/)") j.add_argument("--out", default="/tmp/jellyfin-tv.tsv") m = sub.add_parser("music", help="export the tracks Jellyfin has tagged for the root") m.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT", "/mnt/entropy/Music"), help="music library root on the host") m.add_argument("--db", default=JF_DB) m.add_argument("--container", default=None, help="the in-container prefix that stands for --root" " (default /data/)") m.add_argument("--out", default="/tmp/jellyfin-music.tsv") a = ap.parse_args() if a.cmd == "enumerate": enumerate_lib(a.root, a.out) elif a.cmd == "probe": probe(a.enum, a.out, a.jobs) elif a.cmd == "plex": plex(a.root, a.db, a.out) elif a.cmd == "music": music(a.root, a.db, a.container, a.out) else: jellyfin(a.root, a.db, a.container, a.out) if __name__ == "__main__": main()