Frank
b03931f7ce
The music library is rips and discography packs whose filenames say nothing the matcher needs. The authority is Jellyfin's own audio rows: one per track, with the album and album-artist ids it carries in BaseItemProviders (the item JSON holds an empty ProviderIds for albums in this build). Grouping is what the tags assert, not where files sit: (album-artist tag, album, year), case-folded. A compilation is one album whose tracks name many performers, so it files once under Various Artists; two album-artist tags on one album name are two albums. The folder spelling is the dominant one, because a name taken from whichever row sorted first depends on row order and re-names albums that are already filed. An occupied path is never a destination: the executor refuses occupied targets, so a plan that aims at one fails after the renames meant to clear it. A file standing on its own target, or already carrying a collision suffix, is a keep - otherwise the plan shuffles instead of being a fixed point.
417 lines
20 KiB
Python
Executable File
417 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
r"""Read-only census of a media library — the inputs a relayout plan is built from.
|
|
|
|
media-census.py enumerate [--root DIR] [--out TSV]
|
|
media-census.py probe [--enum TSV] [--out TSV] [--jobs N]
|
|
media-census.py plex [--root DIR] [--db PATH] [--out TSV]
|
|
media-census.py jellyfin [--root DIR] [--db PATH] [--container PREFIX] [--out TSV]
|
|
|
|
Run it on the machine that mounts the library. Nothing here writes inside the
|
|
library and nothing is deleted; the only outputs are the TSVs below, and the
|
|
scripts/media-plan.py and scripts/media-shows.py read them instead of touching media.
|
|
|
|
enum bytes <TAB> ext <TAB> path every file under the root
|
|
probed bytes <TAB> path <TAB> dur <TAB> vcodec <TAB> WxH <TAB> fps <TAB> vbit
|
|
<TAB> pix <TAB> audio <TAB> subs <TAB> spare <TAB> format
|
|
one row per video-container file; `audio` is "codec:lang+codec:lang",
|
|
`subs` the subtitle-stream count, `spare` the audio streams carrying
|
|
neither a default nor a forced tag
|
|
plex file <TAB> leaf_title <TAB> … (see PLEX_COLS)
|
|
the titles Plex already matched — the authority over any filename guess
|
|
|
|
jellyfin file <TAB> series <TAB> year <TAB> season <TAB> episode <TAB> extra
|
|
<TAB> tmdb (see JF_COLS)
|
|
one row per episode Jellyfin has matched and numbered, its path mapped
|
|
from the container's view back to the host; `year` and `tmdb` are the
|
|
SERIES' values, `extra` is Jellyfin's own extra type when the row is
|
|
bonus material rather than an episode
|
|
|
|
music file <TAB> artist <TAB> album <TAB> year <TAB> disc <TAB> track <TAB> title
|
|
<TAB> mb <TAB> performer (see MUSIC_COLS)
|
|
one row per track Jellyfin has tagged: the tag read is what the row IS,
|
|
so `artist` is the album artist (the track artist when that is empty) and
|
|
`album`/`year` are the album item's provider values when the folder is an
|
|
album item and the tags' otherwise; `mb` is the album's MusicBrainz
|
|
release group, `performer` the track artist the tags carry
|
|
|
|
`probe` is the slow step (every container is opened for its header, ~10 min for
|
|
4,800 files on a spindle array at 6 workers) so it is `nice`d: a census beside a
|
|
live service must not starve it. `enumerate` is a stat pass, so re-censusing
|
|
before a plan is cheap — and a plan is only ever as good as these files.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
# Containers ffprobe can read. .iso/.img are deliberately NOT here: a disc image is
|
|
# a mounted volume, not a stream, and the planner treats it as a movie file instead.
|
|
PROBE_EXT = {".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".m2ts", ".mpg",
|
|
".mpeg", ".webm", ".wmv", ".divx", ".ogm", ".bdmv", ".vob"}
|
|
# Trash and thumbnail caches are not library content, and this repo's trash policy
|
|
# means .AGENTTRASH can sit inside a movie folder at any time.
|
|
SKIP_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir|lost\+found)(/|$)")
|
|
PLEX_DB = ("/var/lib/plexmediaserver/Library/Application Support/Plex Media Server"
|
|
"/Plug-in Support/Databases/com.plexapp.plugins.library.db")
|
|
PLEX_COLS = ["file", "leaf_title", "leaf_orig", "leaf_edition", "leaf_year",
|
|
"parent_title", "parent_orig", "parent_year", "parent_edition",
|
|
"container", "vcodec", "acodec", "width", "height"]
|
|
PLEX_SQL = """
|
|
select p.file, coalesce(mi.title,''), coalesce(mi.original_title,''),
|
|
coalesce(mi.edition_title,''), coalesce(mi.year,0),
|
|
coalesce(par.title,''), coalesce(par.original_title,''),
|
|
coalesce(par.year,0), coalesce(par.edition_title,''),
|
|
m.container, m.video_codec, m.audio_codec, m.width, m.height
|
|
from media_parts p
|
|
join media_items m on m.id = p.media_item_id
|
|
join metadata_items mi on mi.id = m.metadata_item_id
|
|
left join metadata_items par on par.id = mi.parent_id
|
|
where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
|
"""
|
|
JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db"
|
|
JF_COLS = ["file", "series", "year", "season", "episode", "extra", "tmdb"]
|
|
# Jellyfin stores the entity's .NET class as its type; the TV tree is these three.
|
|
JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode",
|
|
"MediaBrowser.Controller.Entities.TV.Season",
|
|
"MediaBrowser.Controller.Entities.TV.Series")
|
|
JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, IsVirtualItem
|
|
from BaseItems where Type = ? and Path like ?""")
|
|
|
|
# The music tree is these two entity classes. The track row carries the tags
|
|
# Jellyfin read out of the file; the album row carries what the provider matched,
|
|
# which is the better folder name of the two.
|
|
JF_AUDIO = "MediaBrowser.Controller.Entities.Audio.Audio"
|
|
JF_ALBUM = "MediaBrowser.Controller.Entities.Audio.MusicAlbum"
|
|
MUSIC_COLS = ["file", "artist", "album", "year", "disc", "track", "title", "mb",
|
|
"performer"]
|
|
MUSIC_SQL = ("""select b.Path, b.AlbumArtists, b.Artists, b.Album, b.ProductionYear,
|
|
b.ParentIndexNumber, b.IndexNumber, b.Name, b.Id,
|
|
a.Name, a.ProductionYear, a.Id
|
|
from BaseItems b
|
|
left join BaseItems a on a.Id = b.ParentId and a.Type = ?
|
|
where b.Type = ? and b.Path like ?""")
|
|
# The album's identity, in preference order. A release group is the album; a bare
|
|
# release id is one reissue of it.
|
|
MUSIC_PROVIDERS = ("MusicBrainzReleaseGroup", "MusicBrainzAlbum")
|
|
|
|
|
|
def enumerate_lib(root, out):
|
|
"""One row per real file under root: size, extension, path."""
|
|
rows = []
|
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
dirnames[:] = [d for d in dirnames
|
|
if not SKIP_DIR.search(os.path.join(dirpath, d))]
|
|
for fn in filenames:
|
|
p = os.path.join(dirpath, fn)
|
|
if SKIP_DIR.search(p):
|
|
continue
|
|
try:
|
|
st = os.lstat(p)
|
|
except OSError:
|
|
continue
|
|
if not os.path.isfile(p) or os.path.islink(p):
|
|
continue # links and sockets are not library content
|
|
rows.append((st.st_size, os.path.splitext(fn)[1].lower(), p))
|
|
with open(out, "w") as fh:
|
|
for size, ext, p in rows:
|
|
fh.write(f"{size}\t{ext}\t{p}\n")
|
|
vid = [r for r in rows if r[1] in PROBE_EXT]
|
|
print(f"files={len(rows)} bytes={sum(r[0] for r in rows)} "
|
|
f"probes={len(vid)} probe_bytes={sum(r[0] for r in vid)} -> {out}")
|
|
|
|
|
|
def probe_one(path):
|
|
"""ffprobe header read -> the probed TSV row (never the whole file)."""
|
|
try:
|
|
os.nice(19) # a census must never outrank the media server
|
|
except OSError:
|
|
pass
|
|
try:
|
|
size = os.stat(path).st_size
|
|
except OSError:
|
|
return None
|
|
cmd = ["ffprobe", "-v", "error", "-print_format", "json",
|
|
"-show_format", "-show_streams", path]
|
|
try:
|
|
done = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
|
data = json.loads(done.stdout or "{}")
|
|
except (subprocess.TimeoutExpired, subprocess.SubprocessError,
|
|
json.JSONDecodeError, OSError):
|
|
data = {}
|
|
st = data.get("streams", [])
|
|
fmt = data.get("format", {})
|
|
v = next((s for s in st if s.get("codec_type") == "video"), None)
|
|
aud = [s for s in st if s.get("codec_type") == "audio"]
|
|
subs = [s for s in st if s.get("codec_type") == "subtitle"]
|
|
fps = "-"
|
|
if v and v.get("r_frame_rate"):
|
|
try:
|
|
n, d = (int(x) for x in v["r_frame_rate"].split("/", 1))
|
|
fps = f"{round(n / d * 100) / 100:g}" if d else "0"
|
|
except ValueError:
|
|
pass
|
|
|
|
def num(x):
|
|
try:
|
|
return round(float(x))
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
return [size, path, num(fmt.get("duration")),
|
|
(v or {}).get("codec_name", "-"),
|
|
f"{v['width']}x{v['height']}" if v else "-",
|
|
fps, num(fmt.get("bit_rate")), (v or {}).get("pix_fmt", "-"),
|
|
"+".join(f"{a.get('codec_name', '?')}:{a.get('language', '?')}" for a in aud),
|
|
len(subs),
|
|
sum(1 for a in aud if not re.search(
|
|
r"default|forced", json.dumps(a.get("tags", {})), re.I)),
|
|
fmt.get("format_name", "-") or "-"]
|
|
|
|
|
|
def probe(enum, out, jobs):
|
|
"""Probe every video-container row of the enum TSV, in parallel."""
|
|
paths = []
|
|
with open(enum) as fh:
|
|
for line in fh:
|
|
f = line.rstrip("\n").split("\t", 2)
|
|
if len(f) == 3 and f[1] in PROBE_EXT:
|
|
paths.append(f[2])
|
|
print(f"probing {len(paths)} files with {jobs} workers", flush=True)
|
|
done = unreadable = 0
|
|
with open(out, "w") as fh, ThreadPoolExecutor(max_workers=jobs) as pool:
|
|
for row in pool.map(probe_one, paths):
|
|
if row is None:
|
|
continue
|
|
if row[3] == "-":
|
|
unreadable += 1 # header unreadable: the row still gets written
|
|
done += 1
|
|
fh.write("\t".join(str(x) for x in row) + "\n")
|
|
if done % 250 == 0:
|
|
print(f" {done}/{len(paths)}", flush=True)
|
|
print(f"probed={done} unreadable={unreadable} -> {out}")
|
|
|
|
|
|
def plex(root, db, out):
|
|
"""Plex's matched titles for files under root, from its DB, read-only."""
|
|
uri = f"file:{db}?mode=ro&immutable=1"
|
|
try:
|
|
con = sqlite3.connect(uri, uri=True)
|
|
except sqlite3.Error as e:
|
|
sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n"
|
|
"Plex runs rootless here: run this as the container's owner, never "
|
|
"with sudo — rootless storage is per-user, so root sees nothing.")
|
|
with open(out, "w", newline="") as fh:
|
|
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
|
w.writerow(PLEX_COLS)
|
|
n = 0
|
|
for r in con.execute(PLEX_SQL, (root + "/%",)):
|
|
w.writerow([str(x).replace("\t", " ").replace("\n", " ") for x in r])
|
|
n += 1
|
|
print(f"plex-matched={n} -> {out}")
|
|
|
|
|
|
def jellyfin(root, db, container, out):
|
|
"""The episodes Jellyfin has already numbered, for the library at root.
|
|
|
|
Jellyfin stores each path as the CONTAINER sees it (its quadlet binds the
|
|
share at /data), so `container` is the in-container prefix that stands for
|
|
root and every exported path is mapped back to the host before it is written
|
|
— the planner reads host paths, because it plans moves on the array.
|
|
|
|
The numbering is what matters, not the name: a release group's file name says
|
|
nothing about which episode a file is, and Jellyfin has already answered that
|
|
question against a provider. Season/episode come from the episode row; the
|
|
series name, its year and its TMDb id come from the Series row the episode
|
|
climbs to, because those are the only place Jellyfin keeps them.
|
|
"""
|
|
try:
|
|
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
|
except sqlite3.Error as e:
|
|
sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n"
|
|
"It is a WAL database: copy it with its -wal and -shm beside it and"
|
|
" point --db at the copy. Never sudo it — the data dir belongs to"
|
|
" the container's owner, and rootless storage is per-user.")
|
|
have = {r[1] for r in con.execute("pragma table_info(BaseItems)")}
|
|
need = {"Id", "ParentId", "Type", "Path", "IndexNumber", "ParentIndexNumber",
|
|
"ExtraType", "IsVirtualItem", "ProductionYear", "Data"}
|
|
if not need <= have:
|
|
sys.exit(f"{db} is not a Jellyfin library database (BaseItems has no"
|
|
f" {','.join(sorted(need - have))})")
|
|
node = {}
|
|
for i, pid, typ, name, year, data in con.execute(
|
|
"select Id, ParentId, Type, Name, ProductionYear, Data from BaseItems"
|
|
" where Type in (?,?,?)", JF_TV):
|
|
node[i] = (pid, typ, name, year, data)
|
|
|
|
def series_of(item_id):
|
|
"""Climb episode -> season -> series; empty when the chain is broken."""
|
|
for _ in range(4):
|
|
if item_id not in node:
|
|
return "", "", ""
|
|
pid, typ, name, year, data = node[item_id]
|
|
if typ.endswith(".Series"):
|
|
try:
|
|
tmdb = (json.loads(data or "{}")
|
|
.get("ProviderIds") or {}).get("Tmdb", "")
|
|
except ValueError:
|
|
tmdb = ""
|
|
return name or "", year or "", tmdb
|
|
item_id = pid
|
|
return "", "", ""
|
|
|
|
# The fleet's quadlets bind the share at /data, so the container's view of
|
|
# this library is /data/<the library's own folder name> unless told otherwise.
|
|
if not container:
|
|
container = "/data/" + os.path.basename(os.path.normpath(root))
|
|
prefix = container.rstrip("/") + "/"
|
|
written = skipped = 0
|
|
with open(out, "w", newline="") as fh:
|
|
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
|
w.writerow(JF_COLS)
|
|
for iid, path, idx, pidx, extra, virt in con.execute(JF_SQL,
|
|
(JF_TV[0], prefix + "%")):
|
|
if virt or not path or not path.startswith(prefix):
|
|
skipped += 1 # a virtual item is a variant, not a file
|
|
continue
|
|
sname, syear, tmdb = series_of(iid)
|
|
w.writerow([root + "/" + path[len(prefix):],
|
|
sname.replace("\t", " "), syear,
|
|
"" if pidx is None else pidx,
|
|
"" if idx is None else idx,
|
|
(extra or "").replace("\t", " "), tmdb])
|
|
written += 1
|
|
print(f"episodes={written} virtual-or-blank={skipped} -> {out}")
|
|
|
|
|
|
def music(root, db, container, out):
|
|
"""The tracks Jellyfin has already tagged, for the library at root.
|
|
|
|
Jellyfin reads music metadata out of the file itself, so an Audio row IS the
|
|
tag read, done once, by the vendor's own parser: its album-artist, album,
|
|
year, disc and track columns are what the embedded tags say and its Name is
|
|
the track title. That makes this export the whole authority for a music
|
|
relayout — the path a ripper chose is not evidence of anything, and the
|
|
folder Jellyfin put a track under is often a release pack, not an album.
|
|
|
|
Two gaps this export answers from the row rather than the tree. Many tracks
|
|
sit under a folder that is not an album item at all (a disc folder, or a
|
|
mixed one), so the album name and year fall back to the tags when the album
|
|
row is absent. And an album's MusicBrainz ids are not in the row's own Data
|
|
— the current schema leaves that empty for music — they are rows of
|
|
BaseItemProviders keyed by the item's Id.
|
|
"""
|
|
try:
|
|
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
|
except sqlite3.Error as e:
|
|
sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n"
|
|
"It is a WAL database: copy it with its -wal and -shm beside it and"
|
|
" point --db at the copy. Never sudo it — the data dir belongs to"
|
|
" the container's owner, and rootless storage is per-user.")
|
|
have = {r[1] for r in con.execute("pragma table_info(BaseItems)")}
|
|
need = {"Id", "ParentId", "Type", "Path", "Name", "IndexNumber",
|
|
"ParentIndexNumber", "ProductionYear", "Album", "AlbumArtists",
|
|
"Artists"}
|
|
if not need <= have:
|
|
sys.exit(f"{db} is not a Jellyfin library database with music columns"
|
|
f" (BaseItems has no {','.join(sorted(need - have))})")
|
|
# Provider ids, best identity first: the release group is the album, a bare
|
|
# release id is one reissue of it and never overrides a group.
|
|
mb = {}
|
|
if "BaseItemProviders" in {r[0] for r in con.execute(
|
|
"select name from sqlite_master where type='table'")}:
|
|
rank = {p: i for i, p in enumerate(MUSIC_PROVIDERS)}
|
|
best = {}
|
|
for iid, pid, val in con.execute(
|
|
"select ItemId, ProviderId, ProviderValue from BaseItemProviders"
|
|
" where ProviderId in (?, ?)", MUSIC_PROVIDERS):
|
|
if iid not in best or rank[pid] < best[iid][0]:
|
|
best[iid] = (rank[pid], val)
|
|
mb = {k: v[1] for k, v in best.items()}
|
|
|
|
# The fleet's quadlets bind the share at /data, so the container's view of
|
|
# this library is /data/<the library's own folder name> unless told otherwise.
|
|
if not container:
|
|
container = "/data/" + os.path.basename(os.path.normpath(root))
|
|
prefix = container.rstrip("/") + "/"
|
|
written = skipped = 0
|
|
with open(out, "w", newline="") as fh:
|
|
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
|
w.writerow(MUSIC_COLS)
|
|
for (path, aal, arts, tag_album, tag_year, disc, track, name, iid,
|
|
alb_name, alb_year, alb_id) in con.execute(
|
|
MUSIC_SQL, (JF_ALBUM, JF_AUDIO, prefix + "%")):
|
|
if not path or not path.startswith(prefix):
|
|
skipped += 1 # a row for a file outside this library
|
|
continue
|
|
# The album artist owns the folder; the track artist is the fallback
|
|
# for the compilations whose album-artist tag was left empty.
|
|
w.writerow([root + "/" + path[len(prefix):],
|
|
((aal or "").strip() or (arts or "").strip()),
|
|
(alb_name or tag_album or "").replace("\t", " "),
|
|
alb_year or tag_year or "",
|
|
"" if disc is None else disc,
|
|
"" if track is None else track,
|
|
(name or "").replace("\t", " "),
|
|
mb.get(alb_id or "") or mb.get(iid or "") or "",
|
|
(arts or "").replace("\t", " ")])
|
|
written += 1
|
|
print(f"tracks={written} outside-root-or-blank={skipped} -> {out}")
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
root = {"default": os.environ.get("MEDIA_ROOT", "/mnt/entropy/Movies"),
|
|
"help": "library root"}
|
|
e = sub.add_parser("enumerate", help="stat every file under the root")
|
|
e.add_argument("--root", **root)
|
|
e.add_argument("--out", default="/tmp/media-enum.tsv")
|
|
p = sub.add_parser("probe", help="ffprobe every video container")
|
|
p.add_argument("--enum", default="/tmp/media-enum.tsv")
|
|
p.add_argument("--out", default="/tmp/media-probed.tsv")
|
|
p.add_argument("--jobs", type=int, default=6)
|
|
x = sub.add_parser("plex", help="export Plex's matched titles for the root")
|
|
x.add_argument("--root", **root)
|
|
x.add_argument("--db", default=PLEX_DB)
|
|
x.add_argument("--out", default="/tmp/plex-movies.tsv")
|
|
j = sub.add_parser("jellyfin",
|
|
help="export the episodes Jellyfin has numbered for the root")
|
|
j.add_argument("--root", default=os.environ.get("MEDIA_ROOT",
|
|
"/mnt/entropy/TVShows"),
|
|
help="TV library root on the host")
|
|
j.add_argument("--db", default=JF_DB)
|
|
j.add_argument("--container", default=None,
|
|
help="the in-container prefix that stands for --root"
|
|
" (default /data/<root's folder name>)")
|
|
j.add_argument("--out", default="/tmp/jellyfin-tv.tsv")
|
|
m = sub.add_parser("music",
|
|
help="export the tracks Jellyfin has tagged for the root")
|
|
m.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT",
|
|
"/mnt/entropy/Music"),
|
|
help="music library root on the host")
|
|
m.add_argument("--db", default=JF_DB)
|
|
m.add_argument("--container", default=None,
|
|
help="the in-container prefix that stands for --root"
|
|
" (default /data/<root's folder name>)")
|
|
m.add_argument("--out", default="/tmp/jellyfin-music.tsv")
|
|
a = ap.parse_args()
|
|
if a.cmd == "enumerate":
|
|
enumerate_lib(a.root, a.out)
|
|
elif a.cmd == "probe":
|
|
probe(a.enum, a.out, a.jobs)
|
|
elif a.cmd == "plex":
|
|
plex(a.root, a.db, a.out)
|
|
elif a.cmd == "music":
|
|
music(a.root, a.db, a.container, a.out)
|
|
else:
|
|
jellyfin(a.root, a.db, a.container, a.out)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|