Frank
b03931f7ce
The music library is rips and discography packs whose filenames say nothing the matcher needs. The authority is Jellyfin's own audio rows: one per track, with the album and album-artist ids it carries in BaseItemProviders (the item JSON holds an empty ProviderIds for albums in this build). Grouping is what the tags assert, not where files sit: (album-artist tag, album, year), case-folded. A compilation is one album whose tracks name many performers, so it files once under Various Artists; two album-artist tags on one album name are two albums. The folder spelling is the dominant one, because a name taken from whichever row sorted first depends on row order and re-names albums that are already filed. An occupied path is never a destination: the executor refuses occupied targets, so a plan that aims at one fails after the renames meant to clear it. A file standing on its own target, or already carrying a collision suffix, is a keep - otherwise the plan shuffles instead of being a fixed point.
446 lines
21 KiB
Python
Executable File
446 lines
21 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
r"""media-music.py — a plan that re-files the music library into Jellyfin's own shape.
|
|
|
|
media-music.py --root DIR --enum TSV --rows TSV [--out TSV] [--exceptions FILE]
|
|
|
|
Jellyfin's music contract (https://jellyfin.org/docs/general/server/media/music/)
|
|
is not the movie one. Jellyfin reads the metadata out of the file, so a folder is
|
|
an ALBUM and nothing else, every disc of an album belongs in that one folder, the
|
|
tags decide the track order, and a file name only matters when it carries a
|
|
character known to break playback (`< > : " / \ | ? *`) or when a lyric file has
|
|
to match the basename of the track it belongs to. The plan is therefore:
|
|
|
|
Music/<album artist>/<Album> (<year>)/NN - <Title>.<ext>
|
|
... the same folder, D.NN - <Title> when the album has discs
|
|
Music/Various Artists/... when its tracks name
|
|
more than one artist
|
|
|
|
The authority is the tag row Jellyfin already read — `scripts/media-census.py
|
|
music` — never the release-pack folder a ripper made. A file with no row is not
|
|
guessed at: it parks under `<root>/_unsorted` with the relative path it has today.
|
|
|
|
The rules below exist because of what a downloaded discography pack actually
|
|
contains:
|
|
|
|
* A cuesheet beside exactly one audio file means the album is one image with a
|
|
track list inside it. That file keeps its own name: numbering it `01` would
|
|
claim it is one track, and Jellyfin shows one track either way.
|
|
* Sidecars — images, cuesheets, EAC logs, checksums, booklets, playlists —
|
|
follow the album the audio in their own folder belongs to. A folder holding
|
|
two albums gives a sidecar no single owner, so it parks instead of being
|
|
guessed into one.
|
|
* A video container (.mkv, .vob, .ifo, …) inside a music library is not music.
|
|
Renaming it to .mka/.m4a would tell Jellyfin to index video as audio, so it
|
|
parks for a human decision.
|
|
* Playlists move with their album and their CONTENTS are never rewritten: the
|
|
paths inside an .m3u go stale. Editing a file the operator owns to keep a
|
|
convenience file tidy is not this tool's job.
|
|
|
|
Nothing is deleted and nothing is overwritten. The output is a list of rename(2)
|
|
intents for `scripts/media-apply.py`, whose preflight refuses any target that is
|
|
already claimed and whose journal is the rollback.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import os
|
|
import re
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
|
|
# What Jellyfin will index as music. .mp4/.mkv/.webm are deliberately NOT here:
|
|
# the vendor doc says an audio-only stream in those containers is never read as
|
|
# music, and this tool renames for a reason it can prove, not to guess at one.
|
|
AUDIO_EXT = {".flac", ".mp3", ".m4a", ".m4b", ".ogg", ".oga", ".opus", ".wav",
|
|
".wma", ".aiff", ".aif", ".ape", ".dsf", ".dff", ".wv", ".mka",
|
|
".mpc", ".aac"}
|
|
VIDEO_EXT = {".mkv", ".vob", ".ifo", ".bup", ".mpg", ".mpeg", ".avi", ".mov",
|
|
".ts", ".m2ts", ".wmv", ".webm", ".mp4", ".m4v", ".iso", ".img",
|
|
".bdmv", ".divx", ".ogm"}
|
|
# A lyric has to carry the basename of its track, so it is named after the audio
|
|
# file and not after itself. .txt counts only when its stem matches an audio stem:
|
|
# a loose .txt in an album folder is a note.
|
|
LYRIC_EXT = {".lrc", ".elrc", ".txt"}
|
|
IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tiff"}
|
|
# The album-artist strings that mean "compilation", so such an album is not filed
|
|
# under one artist who does not own it.
|
|
VARIOUS = {"various", "various artists", "unknown", "unknown artist", "unknow"}
|
|
# The characters Jellyfin documents as breaking playback. They are dropped, not
|
|
# substituted: a replacement character is a guess at what the title meant.
|
|
BAD_CHARS = set('<>:"/\\|?*')
|
|
TRASH_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir"
|
|
r"|lost\+found)(/|$)")
|
|
|
|
ARGS = None
|
|
HELD = {} # path -> (bytes, ext)
|
|
ROWS = {} # path -> the tag row
|
|
PLACED = {} # audio path -> the destination the plan gave it
|
|
CLAIMED = set() # every destination path the plan has handed out
|
|
MOVES, KEEPS = [], []
|
|
PARKED = defaultdict(list) # reason -> [path]
|
|
NOTES = defaultdict(list) # section -> [line]
|
|
CUE_DIRS = set() # source dirs holding a .cue
|
|
AUDIO_BY_DIR = defaultdict(set) # source dir -> audio files a row names
|
|
CHILD_OF = defaultdict(set) # dir -> the dirs directly below it holding audio
|
|
FOLDERS_SEEN = {} # (dir, depth) -> album folders placed below it
|
|
SUBTREE_DEPTH = 2 # how far below a folder its audio may sit
|
|
ALBUMS = {} # (artist, album, year) -> the album group
|
|
|
|
AMBIGUOUS = object()
|
|
|
|
|
|
def dominant(values):
|
|
"""The spelling the tags mostly agree on: the most frequent value, ties broken
|
|
by sort so the answer never depends on the order rows arrived in."""
|
|
counts = Counter(values)
|
|
return min(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0]
|
|
|
|
|
|
def rel(path):
|
|
"""The path as seen from the library root."""
|
|
return os.path.relpath(path, ARGS.root)
|
|
|
|
|
|
def safe(name):
|
|
"""A name every client can carry: no reserved character, nothing
|
|
unprintable, no trailing dot or space, and short enough to rename."""
|
|
out = "".join(c for c in (name or "")
|
|
if c not in BAD_CHARS and (c.isprintable() or c == " "))
|
|
out = re.sub(r"\s+", " ", out).strip().rstrip(". ")
|
|
while out and len(out.encode("utf-8", "ignore")) > 200:
|
|
out = out[:-4].rstrip()
|
|
return out
|
|
|
|
|
|
def assign(src, folder, name, ext, kind, note):
|
|
"""Book one file into `folder` under `name`, renamed if that exact destination
|
|
is already taken — by another file in the plan, or by a file standing there
|
|
today. Two files claiming one path is the fault that costs a re-run of a 90k
|
|
plan, and so is a plan that only works if the renames happen in the right
|
|
order: the executor refuses any target that is occupied, so an occupied path
|
|
is never a destination, even when its occupant is scheduled to leave.
|
|
A file already standing where the contract puts it is a keep, not a move."""
|
|
stem = safe(name) or "file"
|
|
dst = os.path.join(folder, stem + ext)
|
|
if dst == src:
|
|
CLAIMED.add(dst.lower()) # it is standing on its own target
|
|
KEEPS.append((src, dst, kind))
|
|
return dst
|
|
# A file already in this folder under this name — or under the numbered
|
|
# variant the collision rule gives it — keeps the name it has. Renumbering it
|
|
# every run would turn the plan into a shuffle instead of a fixed point.
|
|
base = os.path.splitext(os.path.basename(src))[0]
|
|
if os.path.dirname(src) == folder and (base == stem
|
|
or base.startswith(stem + " (")):
|
|
CLAIMED.add(src.lower())
|
|
KEEPS.append((src, src, kind))
|
|
return src
|
|
n = 1
|
|
while dst.lower() in CLAIMED:
|
|
n += 1
|
|
dst = os.path.join(folder, f"{stem} ({n}){ext}")
|
|
if n > 1:
|
|
NOTES[note].append(rel(dst))
|
|
CLAIMED.add(dst.lower())
|
|
MOVES.append((src, dst, kind))
|
|
return dst
|
|
|
|
|
|
def park(path, reason):
|
|
"""Hold a file where it is, under the park folder, keeping its relative path."""
|
|
PARKED[reason].append(path)
|
|
dst = os.path.join(ARGS.park_dir, rel(path))
|
|
if dst != path:
|
|
CLAIMED.add(dst.lower())
|
|
MOVES.append((path, dst, "park"))
|
|
|
|
|
|
def under_park(path):
|
|
"""True for a file already parked: that pile is for a human to review, and
|
|
re-parking it every run would only push it deeper."""
|
|
return path.startswith(ARGS.park_dir + os.sep)
|
|
|
|
|
|
# ---------------------------------------------------------------------- albums
|
|
def group_albums():
|
|
"""Every tagged track into one group per (album-artist tag, album, year).
|
|
|
|
The key is the identity the tags assert, not the folder the file sits in: one
|
|
album arrives as three discs in three folders, and two copies of an album in
|
|
one pack are one album to Jellyfin, whose album item is that name, artist and
|
|
year. The artist in the key is the ALBUM-artist tag and never the performer,
|
|
because a compilation is one album whose tracks name many performers — keying
|
|
on the performer would file one album once per artist.
|
|
"""
|
|
for path, r in sorted(ROWS.items()):
|
|
if path not in HELD or under_park(path):
|
|
continue
|
|
d = os.path.dirname(path)
|
|
AUDIO_BY_DIR[d].add(path)
|
|
CHILD_OF[os.path.dirname(d)].add(d) # so a cover above CD1/CD2 sees both
|
|
tag = (r["artist"] or "").strip()
|
|
album = (r["album"] or "").strip()
|
|
performer = (r["performer"] or "").strip()
|
|
if not album:
|
|
park(path, "a track no album names")
|
|
continue
|
|
if not tag and not performer:
|
|
park(path, "a track no artist names")
|
|
continue
|
|
key = (tag.casefold(), album.casefold(), (r["year"] or "").strip())
|
|
g = ALBUMS.setdefault(key, {"tags": [], "albums": [], "year": key[2],
|
|
"members": [], "performers": set(),
|
|
"mb": set()})
|
|
g["members"].append(path)
|
|
g["tags"].append(tag)
|
|
g["albums"].append(album)
|
|
g["performers"].add(performer or tag)
|
|
if r.get("mb"):
|
|
g["mb"].add(r["mb"])
|
|
|
|
|
|
def folder_artist(g):
|
|
"""Whose shelf the album sits on. The album-artist tag owns it when it names
|
|
one; when the tags name only performers, a single performer names the album
|
|
and several make it a compilation, which belongs to no one of them. The key is
|
|
case-folded, so the spelling that reaches the folder is the one the tracks
|
|
mostly carry — see dominant()."""
|
|
tags = [t for t in g["tags"] if t]
|
|
if tags:
|
|
name = dominant(tags)
|
|
return "Various Artists" if name.casefold() in VARIOUS else name
|
|
if len({p.casefold() for p in g["performers"]}) == 1:
|
|
return dominant(g["performers"])
|
|
return "Various Artists"
|
|
|
|
|
|
def album_folder(g):
|
|
name = safe(dominant(g["albums"])) or "Untitled"
|
|
return os.path.join(ARGS.root, safe(folder_artist(g)) or "Unknown Artist",
|
|
f"{name} ({g['year']})" if g["year"] else name)
|
|
|
|
|
|
def is_cue_image(path):
|
|
"""A folder holding this one audio file and a cuesheet is an album image."""
|
|
d = os.path.dirname(path)
|
|
return len(AUDIO_BY_DIR.get(d, ())) == 1 and d in CUE_DIRS
|
|
|
|
|
|
def number_of(value, width):
|
|
return f"{int(value):0{width}d}" if str(value or "").isdigit() else ""
|
|
|
|
|
|
def place_albums():
|
|
for key in sorted(ALBUMS):
|
|
g = ALBUMS[key]
|
|
folder = album_folder(g)
|
|
discs = {g0["disc"] for g0 in (ROWS[p] for p in g["members"]) if g0["disc"]}
|
|
tracks = [int(ROWS[p]["track"]) for p in g["members"]
|
|
if str(ROWS[p]["track"] or "").isdigit()]
|
|
# More than one disc, or a disc numbered past the first: the track number
|
|
# alone cannot name a file uniquely, so the disc goes in front of it.
|
|
multi = len(discs) > 1 or any(str(d).isdigit() and int(d) > 1 for d in discs)
|
|
width = 3 if multi or (max(tracks, default=0) > 99) else 2
|
|
for path in sorted(g["members"]):
|
|
r = ROWS[path]
|
|
ext = HELD[path][1]
|
|
base = os.path.splitext(os.path.basename(path))[0]
|
|
if is_cue_image(path):
|
|
name = safe(base) or "image"
|
|
else:
|
|
title = safe(r["title"] or base) or "untitled"
|
|
num = number_of(r["track"], 2 if multi else width)
|
|
if multi and num:
|
|
num = f"{r['disc'] or '1'}.{num}"
|
|
name = f"{num} - {title}" if num else title
|
|
PLACED[path] = assign(path, folder, name, ext, "track",
|
|
"two tracks claimed one name inside one album")
|
|
|
|
|
|
# --------------------------------------------------------------------- sidecars
|
|
def folders_below(d, depth):
|
|
"""The album folders the audio in `d`, or within `depth` directories below it,
|
|
was placed in. Memoised, because every loose file in a big discography folder
|
|
asks the same question about it."""
|
|
if (d, depth) not in FOLDERS_SEEN:
|
|
seen = {os.path.dirname(PLACED[a]) for a in AUDIO_BY_DIR.get(d, ())
|
|
if a in PLACED}
|
|
if depth:
|
|
for child in CHILD_OF.get(d, ()):
|
|
seen |= folders_below(child, depth - 1)
|
|
FOLDERS_SEEN[(d, depth)] = seen
|
|
return FOLDERS_SEEN[(d, depth)]
|
|
|
|
|
|
def owner_folder(d):
|
|
"""The album folder that owns this directory — the first place up the tree
|
|
where audio sits, in the directory itself or within SUBTREE_DEPTH levels below
|
|
it, which is how a box with its discs in CD1/CD2 owns the cover lying above
|
|
them. None when no audio is near enough to say, AMBIGUOUS when the audio there
|
|
belongs to more than one album."""
|
|
while d.startswith(ARGS.root + os.sep) and d != ARGS.root:
|
|
folders = folders_below(d, SUBTREE_DEPTH)
|
|
if folders:
|
|
return next(iter(folders)) if len(folders) == 1 else AMBIGUOUS
|
|
d = os.path.dirname(d)
|
|
return None
|
|
|
|
|
|
def place_sidecars():
|
|
"""Everything that is not a tagged track: lyrics, images, cuesheets, logs,
|
|
booklets, playlists — and the clutter that is none of those."""
|
|
# The lyric rule needs the audio's stem as it is named TODAY, before the plan
|
|
# renames it, because that is what the sidecar's own name was copied from.
|
|
stem = {(d, os.path.splitext(os.path.basename(a))[0]): a
|
|
for d, audios in AUDIO_BY_DIR.items() for a in audios}
|
|
for path, (_size, ext) in sorted(HELD.items()):
|
|
if path in ROWS or under_park(path) or TRASH_DIR.search(path):
|
|
continue
|
|
d, base = os.path.dirname(path), os.path.splitext(os.path.basename(path))[0]
|
|
if ext in VIDEO_EXT:
|
|
park(path, "a video container inside the music library: Jellyfin will "
|
|
"not read it as music, and renaming it would claim it is")
|
|
elif ext in AUDIO_EXT:
|
|
park(path, "an audio file no provider row names — its tags are not in "
|
|
"the library Jellyfin has matched")
|
|
elif ext in (LYRIC_EXT | IMAGE_EXT) and (d, base) in stem:
|
|
# A lyric, or a picture named after its track: it follows that track
|
|
# and takes the name the track now has, or Jellyfin cannot find it.
|
|
audio = stem[(d, base)]
|
|
dst = PLACED.get(audio)
|
|
if dst:
|
|
folder, name = os.path.dirname(dst), os.path.splitext(
|
|
os.path.basename(dst))[0]
|
|
assign(path, folder, name, ext, "lyric",
|
|
"a sidecar whose name collided with another in its album")
|
|
else:
|
|
park(path, "a sidecar of a track the plan could not place")
|
|
else:
|
|
owner = owner_folder(d)
|
|
if owner is AMBIGUOUS:
|
|
park(path, "a sidecar whose folder holds more than one album, so "
|
|
"nothing says which one it belongs to")
|
|
elif owner:
|
|
assign(path, owner, base, ext,
|
|
"playlist" if ext in (".m3u", ".m3u8") else "sidecar",
|
|
"one album folder would have carried two files of the same "
|
|
"sidecar name")
|
|
else:
|
|
park(path, "not music, or music with nothing beside it to belong to")
|
|
|
|
|
|
# ---------------------------------------------------------------------- outputs
|
|
def write_plan():
|
|
"""mkdir rows for every destination folder and its whole chain, then the moves.
|
|
Parents first: the executor creates directories in the order they are listed."""
|
|
chain = set()
|
|
for _s, dst, _k in MOVES:
|
|
d = os.path.dirname(dst)
|
|
while d and d != ARGS.root and d != os.path.dirname(d):
|
|
chain.add(d)
|
|
d = os.path.dirname(d)
|
|
dirs = sorted(chain, key=lambda p: (p.count("/"), p))
|
|
with open(ARGS.out, "w") as fh:
|
|
fh.write("action\tsrc\tdst\tkind\n")
|
|
for d in dirs:
|
|
fh.write(f"mkdir\t\t{d}\tdir\n")
|
|
for src, dst, kind in sorted(MOVES, key=lambda m: (m[1], m[0])):
|
|
fh.write(f"move\t{src}\t{dst}\t{kind}\n")
|
|
for src, dst, kind in sorted(KEEPS, key=lambda m: (m[1], m[0])):
|
|
fh.write(f"keep\t{src}\t{dst}\t{kind}\n")
|
|
parked = [m for m in MOVES if m[2] == "park"]
|
|
sized = sum(HELD[s][0] for s, _d, _k in MOVES if s in HELD)
|
|
print(f"plan: {len(MOVES) - len(parked)} moves, {len(parked)} parked"
|
|
f" ({sized / 2**40:.2f} TiB touched), {len(KEEPS)} already placed,"
|
|
f" {len(dirs)} folders -> {ARGS.out}")
|
|
|
|
|
|
def write_exceptions():
|
|
with open(ARGS.exceptions, "w") as fh:
|
|
fh.write(f"Music relayout exceptions for {ARGS.root}\n\n")
|
|
fh.write(f"Everything below is parked under {ARGS.park_dir}, keeping the\n"
|
|
"relative path it has today. Nothing is deleted and nothing is\n"
|
|
"overwritten: a parked file is one rename from wherever the plan is\n"
|
|
"pointed next.\n\n")
|
|
for reason in sorted(PARKED, key=lambda r: -len(PARKED[r])):
|
|
rs = PARKED[reason]
|
|
fh.write(f"{len(rs):>6} files {reason}\n")
|
|
for p in sorted(rs, key=rel)[:400]:
|
|
fh.write(f" {rel(p)}\n")
|
|
if len(rs) > 400:
|
|
fh.write(f" … and {len(rs) - 400} more\n")
|
|
fh.write("\n")
|
|
for section in sorted(NOTES, key=lambda s: -len(NOTES[s])):
|
|
lines = sorted(set(NOTES[section]))
|
|
fh.write(f"{len(lines):>6} {section}\n")
|
|
for line in lines[:200]:
|
|
fh.write(f" {line}\n")
|
|
if len(lines) > 200:
|
|
fh.write(f" … and {len(lines) - 200} more\n")
|
|
fh.write("\n")
|
|
no_year = sorted(f"{folder_artist(g)}/{dominant(g['albums'])}"
|
|
for g in ALBUMS.values() if not g["year"])
|
|
if no_year:
|
|
fh.write(f"{len(no_year):>6} albums carry no year in any tag, so their folders\n"
|
|
" are named without one; Jellyfin still matches them.\n")
|
|
fh.write(" " + ", ".join(no_year[:300]) + "\n\n")
|
|
clash = sorted(f"{folder_artist(g)}/{dominant(g['albums'])}"
|
|
for g in ALBUMS.values() if len(g["mb"]) > 1)
|
|
if clash:
|
|
fh.write(f"{len(clash):>6} albums put one folder name on more than one\n"
|
|
" MusicBrainz release group — different releases, one name.\n"
|
|
" They share the folder Jellyfin's own album item names.\n")
|
|
fh.write(" " + ", ".join(clash[:200]) + "\n\n")
|
|
fh.write(" Playlists move with their album and their contents are never\n"
|
|
" rewritten: the paths inside an .m3u go stale.\n")
|
|
print(f"exceptions -> {ARGS.exceptions}")
|
|
|
|
|
|
def main():
|
|
global ARGS
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT",
|
|
"/mnt/entropy/Music"))
|
|
ap.add_argument("--enum", default="/tmp/music-enum.tsv",
|
|
help="the enumerate TSV (bytes, ext, path)")
|
|
ap.add_argument("--rows", default="/tmp/jellyfin-music.tsv",
|
|
help="the tag rows scripts/media-census.py music wrote")
|
|
ap.add_argument("--out", default="/tmp/music-plan.tsv")
|
|
ap.add_argument("--exceptions", default="/tmp/music-exceptions.txt")
|
|
ap.add_argument("--park", default="_unsorted",
|
|
help="the folder inside the root that holds what cannot be filed")
|
|
ARGS = ap.parse_args()
|
|
ARGS.root = os.path.normpath(ARGS.root)
|
|
ARGS.park_dir = os.path.join(ARGS.root, ARGS.park)
|
|
read_inputs()
|
|
# Every path a file occupies today is claimed before a single destination is
|
|
# handed out: the executor refuses an occupied target, so a plan that aimed at
|
|
# one would only fail after the renames that were supposed to clear it.
|
|
CLAIMED.update(p.lower() for p in HELD)
|
|
for path, (_size, ext) in HELD.items():
|
|
if ext == ".cue":
|
|
CUE_DIRS.add(os.path.dirname(path))
|
|
group_albums()
|
|
place_albums()
|
|
place_sidecars()
|
|
write_plan()
|
|
write_exceptions()
|
|
|
|
|
|
def read_inputs():
|
|
with open(ARGS.enum) as fh:
|
|
for line in fh:
|
|
parts = line.rstrip("\n").split("\t")
|
|
if len(parts) >= 3:
|
|
HELD[parts[2]] = (int(parts[0]), parts[1].lower())
|
|
with open(ARGS.rows, newline="") as fh:
|
|
for row in csv.DictReader(fh, delimiter="\t"):
|
|
if row.get("file"):
|
|
ROWS[row["file"]] = row
|
|
gone = [p for p in ROWS if p not in HELD]
|
|
if gone:
|
|
NOTES["a tag row whose file the census no longer sees"].extend(gone)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|