diff --git a/scripts/media-census.py b/scripts/media-census.py index e9e09ab..62114f7 100755 --- a/scripts/media-census.py +++ b/scripts/media-census.py @@ -26,6 +26,14 @@ scripts/media-plan.py and scripts/media-shows.py read them instead of touching m SERIES' values, `extra` is Jellyfin's own extra type when the row is bonus material rather than an episode + music file artist album year disc track title + mb performer (see MUSIC_COLS) + one row per track Jellyfin has tagged: the tag read is what the row IS, + so `artist` is the album artist (the track artist when that is empty) and + `album`/`year` are the album item's provider values when the folder is an + album item and the tags' otherwise; `mb` is the album's MusicBrainz + release group, `performer` the track artist the tags carry + `probe` is the slow step (every container is opened for its header, ~10 min for 4,800 files on a spindle array at 6 workers) so it is `nice`d: a census beside a live service must not starve it. `enumerate` is a stat pass, so re-censusing @@ -74,6 +82,23 @@ JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode", JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType, IsVirtualItem from BaseItems where Type = ? and Path like ?""") +# The music tree is these two entity classes. The track row carries the tags +# Jellyfin read out of the file; the album row carries what the provider matched, +# which is the better folder name of the two. +JF_AUDIO = "MediaBrowser.Controller.Entities.Audio.Audio" +JF_ALBUM = "MediaBrowser.Controller.Entities.Audio.MusicAlbum" +MUSIC_COLS = ["file", "artist", "album", "year", "disc", "track", "title", "mb", + "performer"] +MUSIC_SQL = ("""select b.Path, b.AlbumArtists, b.Artists, b.Album, b.ProductionYear, + b.ParentIndexNumber, b.IndexNumber, b.Name, b.Id, + a.Name, a.ProductionYear, a.Id + from BaseItems b + left join BaseItems a on a.Id = b.ParentId and a.Type = ? + where b.Type = ? and b.Path like ?""") +# The album's identity, in preference order. A release group is the album; a bare +# release id is one reissue of it. +MUSIC_PROVIDERS = ("MusicBrainzReleaseGroup", "MusicBrainzAlbum") + def enumerate_lib(root, out): """One row per real file under root: size, extension, path.""" @@ -263,6 +288,81 @@ def jellyfin(root, db, container, out): print(f"episodes={written} virtual-or-blank={skipped} -> {out}") +def music(root, db, container, out): + """The tracks Jellyfin has already tagged, for the library at root. + + Jellyfin reads music metadata out of the file itself, so an Audio row IS the + tag read, done once, by the vendor's own parser: its album-artist, album, + year, disc and track columns are what the embedded tags say and its Name is + the track title. That makes this export the whole authority for a music + relayout — the path a ripper chose is not evidence of anything, and the + folder Jellyfin put a track under is often a release pack, not an album. + + Two gaps this export answers from the row rather than the tree. Many tracks + sit under a folder that is not an album item at all (a disc folder, or a + mixed one), so the album name and year fall back to the tags when the album + row is absent. And an album's MusicBrainz ids are not in the row's own Data + — the current schema leaves that empty for music — they are rows of + BaseItemProviders keyed by the item's Id. + """ + try: + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + except sqlite3.Error as e: + sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n" + "It is a WAL database: copy it with its -wal and -shm beside it and" + " point --db at the copy. Never sudo it — the data dir belongs to" + " the container's owner, and rootless storage is per-user.") + have = {r[1] for r in con.execute("pragma table_info(BaseItems)")} + need = {"Id", "ParentId", "Type", "Path", "Name", "IndexNumber", + "ParentIndexNumber", "ProductionYear", "Album", "AlbumArtists", + "Artists"} + if not need <= have: + sys.exit(f"{db} is not a Jellyfin library database with music columns" + f" (BaseItems has no {','.join(sorted(need - have))})") + # Provider ids, best identity first: the release group is the album, a bare + # release id is one reissue of it and never overrides a group. + mb = {} + if "BaseItemProviders" in {r[0] for r in con.execute( + "select name from sqlite_master where type='table'")}: + rank = {p: i for i, p in enumerate(MUSIC_PROVIDERS)} + best = {} + for iid, pid, val in con.execute( + "select ItemId, ProviderId, ProviderValue from BaseItemProviders" + " where ProviderId in (?, ?)", MUSIC_PROVIDERS): + if iid not in best or rank[pid] < best[iid][0]: + best[iid] = (rank[pid], val) + mb = {k: v[1] for k, v in best.items()} + + # The fleet's quadlets bind the share at /data, so the container's view of + # this library is /data/ unless told otherwise. + if not container: + container = "/data/" + os.path.basename(os.path.normpath(root)) + prefix = container.rstrip("/") + "/" + written = skipped = 0 + with open(out, "w", newline="") as fh: + w = csv.writer(fh, delimiter="\t", lineterminator="\n") + w.writerow(MUSIC_COLS) + for (path, aal, arts, tag_album, tag_year, disc, track, name, iid, + alb_name, alb_year, alb_id) in con.execute( + MUSIC_SQL, (JF_ALBUM, JF_AUDIO, prefix + "%")): + if not path or not path.startswith(prefix): + skipped += 1 # a row for a file outside this library + continue + # The album artist owns the folder; the track artist is the fallback + # for the compilations whose album-artist tag was left empty. + w.writerow([root + "/" + path[len(prefix):], + ((aal or "").strip() or (arts or "").strip()), + (alb_name or tag_album or "").replace("\t", " "), + alb_year or tag_year or "", + "" if disc is None else disc, + "" if track is None else track, + (name or "").replace("\t", " "), + mb.get(alb_id or "") or mb.get(iid or "") or "", + (arts or "").replace("\t", " ")]) + written += 1 + print(f"tracks={written} outside-root-or-blank={skipped} -> {out}") + + def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) sub = ap.add_subparsers(dest="cmd", required=True) @@ -289,6 +389,16 @@ def main(): help="the in-container prefix that stands for --root" " (default /data/)") j.add_argument("--out", default="/tmp/jellyfin-tv.tsv") + m = sub.add_parser("music", + help="export the tracks Jellyfin has tagged for the root") + m.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT", + "/mnt/entropy/Music"), + help="music library root on the host") + m.add_argument("--db", default=JF_DB) + m.add_argument("--container", default=None, + help="the in-container prefix that stands for --root" + " (default /data/)") + m.add_argument("--out", default="/tmp/jellyfin-music.tsv") a = ap.parse_args() if a.cmd == "enumerate": enumerate_lib(a.root, a.out) @@ -296,6 +406,8 @@ def main(): probe(a.enum, a.out, a.jobs) elif a.cmd == "plex": plex(a.root, a.db, a.out) + elif a.cmd == "music": + music(a.root, a.db, a.container, a.out) else: jellyfin(a.root, a.db, a.container, a.out) diff --git a/scripts/media-music.py b/scripts/media-music.py new file mode 100755 index 0000000..da66dd1 --- /dev/null +++ b/scripts/media-music.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +r"""media-music.py — a plan that re-files the music library into Jellyfin's own shape. + + media-music.py --root DIR --enum TSV --rows TSV [--out TSV] [--exceptions FILE] + +Jellyfin's music contract (https://jellyfin.org/docs/general/server/media/music/) +is not the movie one. Jellyfin reads the metadata out of the file, so a folder is +an ALBUM and nothing else, every disc of an album belongs in that one folder, the +tags decide the track order, and a file name only matters when it carries a +character known to break playback (`< > : " / \ | ? *`) or when a lyric file has +to match the basename of the track it belongs to. The plan is therefore: + + Music// ()/NN - .<ext> + ... the same folder, D.NN - <Title> when the album has discs + Music/Various Artists/... when its tracks name + more than one artist + +The authority is the tag row Jellyfin already read — `scripts/media-census.py +music` — never the release-pack folder a ripper made. A file with no row is not +guessed at: it parks under `<root>/_unsorted` with the relative path it has today. + +The rules below exist because of what a downloaded discography pack actually +contains: + + * A cuesheet beside exactly one audio file means the album is one image with a + track list inside it. That file keeps its own name: numbering it `01` would + claim it is one track, and Jellyfin shows one track either way. + * Sidecars — images, cuesheets, EAC logs, checksums, booklets, playlists — + follow the album the audio in their own folder belongs to. A folder holding + two albums gives a sidecar no single owner, so it parks instead of being + guessed into one. + * A video container (.mkv, .vob, .ifo, …) inside a music library is not music. + Renaming it to .mka/.m4a would tell Jellyfin to index video as audio, so it + parks for a human decision. + * Playlists move with their album and their CONTENTS are never rewritten: the + paths inside an .m3u go stale. Editing a file the operator owns to keep a + convenience file tidy is not this tool's job. + +Nothing is deleted and nothing is overwritten. The output is a list of rename(2) +intents for `scripts/media-apply.py`, whose preflight refuses any target that is +already claimed and whose journal is the rollback. +""" +import argparse +import csv +import os +import re +import sys +from collections import Counter, defaultdict + +# What Jellyfin will index as music. .mp4/.mkv/.webm are deliberately NOT here: +# the vendor doc says an audio-only stream in those containers is never read as +# music, and this tool renames for a reason it can prove, not to guess at one. +AUDIO_EXT = {".flac", ".mp3", ".m4a", ".m4b", ".ogg", ".oga", ".opus", ".wav", + ".wma", ".aiff", ".aif", ".ape", ".dsf", ".dff", ".wv", ".mka", + ".mpc", ".aac"} +VIDEO_EXT = {".mkv", ".vob", ".ifo", ".bup", ".mpg", ".mpeg", ".avi", ".mov", + ".ts", ".m2ts", ".wmv", ".webm", ".mp4", ".m4v", ".iso", ".img", + ".bdmv", ".divx", ".ogm"} +# A lyric has to carry the basename of its track, so it is named after the audio +# file and not after itself. .txt counts only when its stem matches an audio stem: +# a loose .txt in an album folder is a note. +LYRIC_EXT = {".lrc", ".elrc", ".txt"} +IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tiff"} +# The album-artist strings that mean "compilation", so such an album is not filed +# under one artist who does not own it. +VARIOUS = {"various", "various artists", "unknown", "unknown artist", "unknow"} +# The characters Jellyfin documents as breaking playback. They are dropped, not +# substituted: a replacement character is a guess at what the title meant. +BAD_CHARS = set('<>:"/\\|?*') +TRASH_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir" + r"|lost\+found)(/|$)") + +ARGS = None +HELD = {} # path -> (bytes, ext) +ROWS = {} # path -> the tag row +PLACED = {} # audio path -> the destination the plan gave it +CLAIMED = set() # every destination path the plan has handed out +MOVES, KEEPS = [], [] +PARKED = defaultdict(list) # reason -> [path] +NOTES = defaultdict(list) # section -> [line] +CUE_DIRS = set() # source dirs holding a .cue +AUDIO_BY_DIR = defaultdict(set) # source dir -> audio files a row names +CHILD_OF = defaultdict(set) # dir -> the dirs directly below it holding audio +FOLDERS_SEEN = {} # (dir, depth) -> album folders placed below it +SUBTREE_DEPTH = 2 # how far below a folder its audio may sit +ALBUMS = {} # (artist, album, year) -> the album group + +AMBIGUOUS = object() + + +def dominant(values): + """The spelling the tags mostly agree on: the most frequent value, ties broken + by sort so the answer never depends on the order rows arrived in.""" + counts = Counter(values) + return min(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0] + + +def rel(path): + """The path as seen from the library root.""" + return os.path.relpath(path, ARGS.root) + + +def safe(name): + """A name every client can carry: no reserved character, nothing + unprintable, no trailing dot or space, and short enough to rename.""" + out = "".join(c for c in (name or "") + if c not in BAD_CHARS and (c.isprintable() or c == " ")) + out = re.sub(r"\s+", " ", out).strip().rstrip(". ") + while out and len(out.encode("utf-8", "ignore")) > 200: + out = out[:-4].rstrip() + return out + + +def assign(src, folder, name, ext, kind, note): + """Book one file into `folder` under `name`, renamed if that exact destination + is already taken — by another file in the plan, or by a file standing there + today. Two files claiming one path is the fault that costs a re-run of a 90k + plan, and so is a plan that only works if the renames happen in the right + order: the executor refuses any target that is occupied, so an occupied path + is never a destination, even when its occupant is scheduled to leave. + A file already standing where the contract puts it is a keep, not a move.""" + stem = safe(name) or "file" + dst = os.path.join(folder, stem + ext) + if dst == src: + CLAIMED.add(dst.lower()) # it is standing on its own target + KEEPS.append((src, dst, kind)) + return dst + # A file already in this folder under this name — or under the numbered + # variant the collision rule gives it — keeps the name it has. Renumbering it + # every run would turn the plan into a shuffle instead of a fixed point. + base = os.path.splitext(os.path.basename(src))[0] + if os.path.dirname(src) == folder and (base == stem + or base.startswith(stem + " (")): + CLAIMED.add(src.lower()) + KEEPS.append((src, src, kind)) + return src + n = 1 + while dst.lower() in CLAIMED: + n += 1 + dst = os.path.join(folder, f"{stem} ({n}){ext}") + if n > 1: + NOTES[note].append(rel(dst)) + CLAIMED.add(dst.lower()) + MOVES.append((src, dst, kind)) + return dst + + +def park(path, reason): + """Hold a file where it is, under the park folder, keeping its relative path.""" + PARKED[reason].append(path) + dst = os.path.join(ARGS.park_dir, rel(path)) + if dst != path: + CLAIMED.add(dst.lower()) + MOVES.append((path, dst, "park")) + + +def under_park(path): + """True for a file already parked: that pile is for a human to review, and + re-parking it every run would only push it deeper.""" + return path.startswith(ARGS.park_dir + os.sep) + + +# ---------------------------------------------------------------------- albums +def group_albums(): + """Every tagged track into one group per (album-artist tag, album, year). + + The key is the identity the tags assert, not the folder the file sits in: one + album arrives as three discs in three folders, and two copies of an album in + one pack are one album to Jellyfin, whose album item is that name, artist and + year. The artist in the key is the ALBUM-artist tag and never the performer, + because a compilation is one album whose tracks name many performers — keying + on the performer would file one album once per artist. + """ + for path, r in sorted(ROWS.items()): + if path not in HELD or under_park(path): + continue + d = os.path.dirname(path) + AUDIO_BY_DIR[d].add(path) + CHILD_OF[os.path.dirname(d)].add(d) # so a cover above CD1/CD2 sees both + tag = (r["artist"] or "").strip() + album = (r["album"] or "").strip() + performer = (r["performer"] or "").strip() + if not album: + park(path, "a track no album names") + continue + if not tag and not performer: + park(path, "a track no artist names") + continue + key = (tag.casefold(), album.casefold(), (r["year"] or "").strip()) + g = ALBUMS.setdefault(key, {"tags": [], "albums": [], "year": key[2], + "members": [], "performers": set(), + "mb": set()}) + g["members"].append(path) + g["tags"].append(tag) + g["albums"].append(album) + g["performers"].add(performer or tag) + if r.get("mb"): + g["mb"].add(r["mb"]) + + +def folder_artist(g): + """Whose shelf the album sits on. The album-artist tag owns it when it names + one; when the tags name only performers, a single performer names the album + and several make it a compilation, which belongs to no one of them. The key is + case-folded, so the spelling that reaches the folder is the one the tracks + mostly carry — see dominant().""" + tags = [t for t in g["tags"] if t] + if tags: + name = dominant(tags) + return "Various Artists" if name.casefold() in VARIOUS else name + if len({p.casefold() for p in g["performers"]}) == 1: + return dominant(g["performers"]) + return "Various Artists" + + +def album_folder(g): + name = safe(dominant(g["albums"])) or "Untitled" + return os.path.join(ARGS.root, safe(folder_artist(g)) or "Unknown Artist", + f"{name} ({g['year']})" if g["year"] else name) + + +def is_cue_image(path): + """A folder holding this one audio file and a cuesheet is an album image.""" + d = os.path.dirname(path) + return len(AUDIO_BY_DIR.get(d, ())) == 1 and d in CUE_DIRS + + +def number_of(value, width): + return f"{int(value):0{width}d}" if str(value or "").isdigit() else "" + + +def place_albums(): + for key in sorted(ALBUMS): + g = ALBUMS[key] + folder = album_folder(g) + discs = {g0["disc"] for g0 in (ROWS[p] for p in g["members"]) if g0["disc"]} + tracks = [int(ROWS[p]["track"]) for p in g["members"] + if str(ROWS[p]["track"] or "").isdigit()] + # More than one disc, or a disc numbered past the first: the track number + # alone cannot name a file uniquely, so the disc goes in front of it. + multi = len(discs) > 1 or any(str(d).isdigit() and int(d) > 1 for d in discs) + width = 3 if multi or (max(tracks, default=0) > 99) else 2 + for path in sorted(g["members"]): + r = ROWS[path] + ext = HELD[path][1] + base = os.path.splitext(os.path.basename(path))[0] + if is_cue_image(path): + name = safe(base) or "image" + else: + title = safe(r["title"] or base) or "untitled" + num = number_of(r["track"], 2 if multi else width) + if multi and num: + num = f"{r['disc'] or '1'}.{num}" + name = f"{num} - {title}" if num else title + PLACED[path] = assign(path, folder, name, ext, "track", + "two tracks claimed one name inside one album") + + +# --------------------------------------------------------------------- sidecars +def folders_below(d, depth): + """The album folders the audio in `d`, or within `depth` directories below it, + was placed in. Memoised, because every loose file in a big discography folder + asks the same question about it.""" + if (d, depth) not in FOLDERS_SEEN: + seen = {os.path.dirname(PLACED[a]) for a in AUDIO_BY_DIR.get(d, ()) + if a in PLACED} + if depth: + for child in CHILD_OF.get(d, ()): + seen |= folders_below(child, depth - 1) + FOLDERS_SEEN[(d, depth)] = seen + return FOLDERS_SEEN[(d, depth)] + + +def owner_folder(d): + """The album folder that owns this directory — the first place up the tree + where audio sits, in the directory itself or within SUBTREE_DEPTH levels below + it, which is how a box with its discs in CD1/CD2 owns the cover lying above + them. None when no audio is near enough to say, AMBIGUOUS when the audio there + belongs to more than one album.""" + while d.startswith(ARGS.root + os.sep) and d != ARGS.root: + folders = folders_below(d, SUBTREE_DEPTH) + if folders: + return next(iter(folders)) if len(folders) == 1 else AMBIGUOUS + d = os.path.dirname(d) + return None + + +def place_sidecars(): + """Everything that is not a tagged track: lyrics, images, cuesheets, logs, + booklets, playlists — and the clutter that is none of those.""" + # The lyric rule needs the audio's stem as it is named TODAY, before the plan + # renames it, because that is what the sidecar's own name was copied from. + stem = {(d, os.path.splitext(os.path.basename(a))[0]): a + for d, audios in AUDIO_BY_DIR.items() for a in audios} + for path, (_size, ext) in sorted(HELD.items()): + if path in ROWS or under_park(path) or TRASH_DIR.search(path): + continue + d, base = os.path.dirname(path), os.path.splitext(os.path.basename(path))[0] + if ext in VIDEO_EXT: + park(path, "a video container inside the music library: Jellyfin will " + "not read it as music, and renaming it would claim it is") + elif ext in AUDIO_EXT: + park(path, "an audio file no provider row names — its tags are not in " + "the library Jellyfin has matched") + elif ext in (LYRIC_EXT | IMAGE_EXT) and (d, base) in stem: + # A lyric, or a picture named after its track: it follows that track + # and takes the name the track now has, or Jellyfin cannot find it. + audio = stem[(d, base)] + dst = PLACED.get(audio) + if dst: + folder, name = os.path.dirname(dst), os.path.splitext( + os.path.basename(dst))[0] + assign(path, folder, name, ext, "lyric", + "a sidecar whose name collided with another in its album") + else: + park(path, "a sidecar of a track the plan could not place") + else: + owner = owner_folder(d) + if owner is AMBIGUOUS: + park(path, "a sidecar whose folder holds more than one album, so " + "nothing says which one it belongs to") + elif owner: + assign(path, owner, base, ext, + "playlist" if ext in (".m3u", ".m3u8") else "sidecar", + "one album folder would have carried two files of the same " + "sidecar name") + else: + park(path, "not music, or music with nothing beside it to belong to") + + +# ---------------------------------------------------------------------- outputs +def write_plan(): + """mkdir rows for every destination folder and its whole chain, then the moves. + Parents first: the executor creates directories in the order they are listed.""" + chain = set() + for _s, dst, _k in MOVES: + d = os.path.dirname(dst) + while d and d != ARGS.root and d != os.path.dirname(d): + chain.add(d) + d = os.path.dirname(d) + dirs = sorted(chain, key=lambda p: (p.count("/"), p)) + with open(ARGS.out, "w") as fh: + fh.write("action\tsrc\tdst\tkind\n") + for d in dirs: + fh.write(f"mkdir\t\t{d}\tdir\n") + for src, dst, kind in sorted(MOVES, key=lambda m: (m[1], m[0])): + fh.write(f"move\t{src}\t{dst}\t{kind}\n") + for src, dst, kind in sorted(KEEPS, key=lambda m: (m[1], m[0])): + fh.write(f"keep\t{src}\t{dst}\t{kind}\n") + parked = [m for m in MOVES if m[2] == "park"] + sized = sum(HELD[s][0] for s, _d, _k in MOVES if s in HELD) + print(f"plan: {len(MOVES) - len(parked)} moves, {len(parked)} parked" + f" ({sized / 2**40:.2f} TiB touched), {len(KEEPS)} already placed," + f" {len(dirs)} folders -> {ARGS.out}") + + +def write_exceptions(): + with open(ARGS.exceptions, "w") as fh: + fh.write(f"Music relayout exceptions for {ARGS.root}\n\n") + fh.write(f"Everything below is parked under {ARGS.park_dir}, keeping the\n" + "relative path it has today. Nothing is deleted and nothing is\n" + "overwritten: a parked file is one rename from wherever the plan is\n" + "pointed next.\n\n") + for reason in sorted(PARKED, key=lambda r: -len(PARKED[r])): + rs = PARKED[reason] + fh.write(f"{len(rs):>6} files {reason}\n") + for p in sorted(rs, key=rel)[:400]: + fh.write(f" {rel(p)}\n") + if len(rs) > 400: + fh.write(f" … and {len(rs) - 400} more\n") + fh.write("\n") + for section in sorted(NOTES, key=lambda s: -len(NOTES[s])): + lines = sorted(set(NOTES[section])) + fh.write(f"{len(lines):>6} {section}\n") + for line in lines[:200]: + fh.write(f" {line}\n") + if len(lines) > 200: + fh.write(f" … and {len(lines) - 200} more\n") + fh.write("\n") + no_year = sorted(f"{folder_artist(g)}/{dominant(g['albums'])}" + for g in ALBUMS.values() if not g["year"]) + if no_year: + fh.write(f"{len(no_year):>6} albums carry no year in any tag, so their folders\n" + " are named without one; Jellyfin still matches them.\n") + fh.write(" " + ", ".join(no_year[:300]) + "\n\n") + clash = sorted(f"{folder_artist(g)}/{dominant(g['albums'])}" + for g in ALBUMS.values() if len(g["mb"]) > 1) + if clash: + fh.write(f"{len(clash):>6} albums put one folder name on more than one\n" + " MusicBrainz release group — different releases, one name.\n" + " They share the folder Jellyfin's own album item names.\n") + fh.write(" " + ", ".join(clash[:200]) + "\n\n") + fh.write(" Playlists move with their album and their contents are never\n" + " rewritten: the paths inside an .m3u go stale.\n") + print(f"exceptions -> {ARGS.exceptions}") + + +def main(): + global ARGS + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT", + "/mnt/entropy/Music")) + ap.add_argument("--enum", default="/tmp/music-enum.tsv", + help="the enumerate TSV (bytes, ext, path)") + ap.add_argument("--rows", default="/tmp/jellyfin-music.tsv", + help="the tag rows scripts/media-census.py music wrote") + ap.add_argument("--out", default="/tmp/music-plan.tsv") + ap.add_argument("--exceptions", default="/tmp/music-exceptions.txt") + ap.add_argument("--park", default="_unsorted", + help="the folder inside the root that holds what cannot be filed") + ARGS = ap.parse_args() + ARGS.root = os.path.normpath(ARGS.root) + ARGS.park_dir = os.path.join(ARGS.root, ARGS.park) + read_inputs() + # Every path a file occupies today is claimed before a single destination is + # handed out: the executor refuses an occupied target, so a plan that aimed at + # one would only fail after the renames that were supposed to clear it. + CLAIMED.update(p.lower() for p in HELD) + for path, (_size, ext) in HELD.items(): + if ext == ".cue": + CUE_DIRS.add(os.path.dirname(path)) + group_albums() + place_albums() + place_sidecars() + write_plan() + write_exceptions() + + +def read_inputs(): + with open(ARGS.enum) as fh: + for line in fh: + parts = line.rstrip("\n").split("\t") + if len(parts) >= 3: + HELD[parts[2]] = (int(parts[0]), parts[1].lower()) + with open(ARGS.rows, newline="") as fh: + for row in csv.DictReader(fh, delimiter="\t"): + if row.get("file"): + ROWS[row["file"]] = row + gone = [p for p in ROWS if p not in HELD] + if gone: + NOTES["a tag row whose file the census no longer sees"].extend(gone) + + +if __name__ == "__main__": + main() diff --git a/serverAppConfigs-docs/Dankserver/Media Relayout.md b/serverAppConfigs-docs/Dankserver/Media Relayout.md index d71bca6..00f2ecd 100644 --- a/serverAppConfigs-docs/Dankserver/Media Relayout.md +++ b/serverAppConfigs-docs/Dankserver/Media Relayout.md @@ -12,8 +12,8 @@ The movie library on [[Entropy Volume]] arrived as release folders — `Some.Movie.2019.1080p.WiKiTaLx.x265-FOO/` with the video inside named after the release. [[Jellyfin]] matches a movie by folder and file name, so the release noise cost posters, extras and versions. These scripts turn that -library into Jellyfin's own shape — the movies, the TV packs, and the TV that -turns up in the movie share — and nothing else in the repo touches a media file. +library into Jellyfin's own shape — the movies, the TV packs, the TV that +turns up in the movie share, and the music library — and nothing else in the repo touches a media file. | Script | Touches media? | Output | |---|---|---| @@ -22,8 +22,10 @@ turns up in the movie share — and nothing else in the repo touches a media fil | `scripts/media-apply.py` | **renames, never copies**; sweeps only what a run emptied | the moves, plus a journal | | `scripts/media-tv.py` | no — TSV out | a plan that moves TV into the TV library | | `scripts/media-shows.py` | no — TSV out | a plan that re-files the TV library itself | +| `scripts/media-music.py` | no — TSV out | a plan that re-files the music library by its tags | -The seams are `tests/media_relayout.bats` and `tests/media_shows.bats`; the target contract is quoted from +The seams are `tests/media_relayout.bats`, `tests/media_shows.bats` and +`tests/media_music.bats`; the target contract is quoted from [jellyfin.org/docs/general/server/media/movies](https://jellyfin.org/docs/general/server/media/movies) in `media-plan.py`'s docstring. @@ -214,6 +216,63 @@ filename. Renaming a show is Jellyfin's work — a folder rename never touches i provider ids — so the mover keeps the name it is handed and strips only the pack index a release leads with (`4a. Seed DESTINY` → `Seed DESTINY`). +## The music library's own relayout + +`/mnt/entropy/Music` is rips and discography packs, and Jellyfin has matched and +tagged almost all of it. `scripts/media-music.py` re-files it into the shape +Jellyfin's music scanner reads — + + Music/<album artist>/<Album> (<year>)/NN - <Title>.ext + +— and the authority is the tag row of the track, never the name the ripper gave +the file. + +```bash +# 1. The authority: Jellyfin's audio rows, one per track, with the album and +# artist ids its own database carries (the DB opens read-only). +python3 scripts/media-census.py music --root /mnt/entropy/Music \ + --out /tmp/music_rows.tsv +# 2. What is on disk (stat only). +python3 scripts/media-census.py enumerate --root /mnt/entropy/Music \ + --out /tmp/music_enum.tsv +# 3. The plan. +python3 scripts/media-music.py --root /mnt/entropy/Music --enum /tmp/music_enum.tsv \ + --rows /tmp/music_rows.tsv --out /tmp/music_plan.tsv \ + --exceptions /tmp/music_exceptions.txt +# 4. Read the exceptions file FIRST — it is the review, not a log. Then the same +# executor, journal and rollback as above, on its own journal. +scripts/media-apply.py apply --plan /tmp/music_plan.tsv --root /mnt/entropy \ + --journal /mnt/entropy/media-relayout-music.jsonl --apply +``` + +The group is what the tags assert, not where the files sit: **(album-artist tag, +album, year)**. One album arrives as three disc folders and as two copies of one +rip, and one MusicBrainz release group is one album. A compilation is one album +whose tracks name many performers, so it files once under `Various Artists`: +keying on the performer would file one album once per artist. Two different +album-artist tags on one album name are two albums, which is what Jellyfin's two +album items say. + +What it refuses: + ++- **A sidecar is placed by the audio around it, never by its own name.** A + `.lrc`, `.txt` or picture that shares a track's stem follows that track and + takes the name the track now has, or Jellyfin cannot find it; anything else + goes to the album its own folder was placed in, or to the folder whose discs + sit one level below it. A folder whose audio belongs to two albums parks its + sidecar rather than guessing. ++- **An image album keeps its name.** A folder holding one audio file and a + `.cue` is a whole album in one file; numbering it `01 - …` tells Jellyfin to + index it as one track. ++- **A video container parks**, because Jellyfin will not index it as music and a + rename to `.m4a` would claim it is; **audio no provider row names parks** too, + because unmatched audio is a tagging job, not a renaming job. + +The 2026-10-30 dry run files 114k tracks and parks 6.8k — mostly sidecars +sitting in a folder that holds two albums, plus audio no provider row names. The +plan is NOT applied: the exceptions file is the review, and it is Frank's to +read. + ## Laws - **A moved file is a 404 in [[Plex]] until its section is rescanned.** The @@ -322,6 +381,21 @@ index a release leads with (`4a. Seed DESTINY` → `Seed DESTINY`). whiteout in the upper layer, which hides the array's copy without removing it; the file returns when the view is unmounted. Deletions in this pipeline happen on the raw path only, which is what the journal can undo. +- **The music album key is case-folded; the folder name is not.** `Master Of + Puppets` and `Master of Puppets` are one album to the matcher, so the spelling + that reaches the folder is the one most tracks carry. Taking it from whichever + row sorted first made the folder name depend on row order, and re-planning an + already-filed library renamed albums that were already in place. +- **An occupied path is never a destination**, not even when its occupant is + scheduled to leave: the executor refuses an occupied target, so a plan that + aims at one fails after the renames that were supposed to clear it. A file + standing where the contract puts it is a keep, and a file already carrying a + collision suffix keeps that suffix — otherwise the plan is a shuffle, not a + fixed point, and a re-run moves albums that are already right. +- **`media-census.py music` reads the `BaseItemProviders` table, not the item's + JSON.** This Jellyfin build carries an empty `ProviderIds` inside + `BaseItems.Data` for albums, so a census that parses the JSON finds no + MusicBrainz or Spotify id on any album and reads as "the library is untagged". ## Rollback diff --git a/tests/media_music.bats b/tests/media_music.bats new file mode 100755 index 0000000..ac46211 --- /dev/null +++ b/tests/media_music.bats @@ -0,0 +1,373 @@ +#!/usr/bin/env bats +# Seam: the music-library relayout — scripts/media-census.py music (the tag and +# provider authority export) and scripts/media-music.py (the plan). +# +# Jellyfin's music rule is the opposite of the movie and TV ones: the embedded +# tags are the metadata and the file name is only a fallback title, so the +# authority here is the Audio row Jellyfin already matched (its album-artist, +# album, year, disc and track columns) and NOT the path a ripper chose. Every +# test states which authority a file has — row only, row with no album item, +# sidecar of a file that has a row, nothing at all — and asserts where the PLAN +# files it, never how the planner decided. +# +# The three outcomes that cost a re-run of a 90k-file plan: a track filed under +# the wrong artist (it splits one album across two artists), a destination that +# two tracks claim (the executor refuses, and the operator loses the run), and a +# lyric file that stops matching the basename of the track it belongs to, which +# silently loses the lyrics. Everything below is written to catch those, plus the +# gate that matters most: the plan must replay through scripts/media-apply.py, +# whose preflight is the fleet's only overwrite guard. + +load helpers + +PLAN="$REPO_ROOT/scripts/media-music.py" +CENSUS="$REPO_ROOT/scripts/media-census.py" +APPLY="$REPO_ROOT/scripts/media-apply.py" + +# at PATH SIZE — a file with deterministic content. +at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; } + +# row FILE ARTIST ALBUM YEAR DISC TRACK TITLE [MB] [PERFORMER] +# One row of the authority export scripts/media-census.py music writes. +row() { + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$1" "$2" "$3" "${4:-}" "${5:-}" "${6:-}" "$7" "${8:-}" "${9:-}" >> "$ROWS" +} + +# dst SRC — the destination the plan gives SRC, or nothing when it does not move. +dst() { + awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $3}' "$OUT" +} + +setup() { + MUSIC="$BATS_TEST_TMPDIR/Music" + PARK="$MUSIC/_unsorted" + ENUM="$BATS_TEST_TMPDIR/enum.tsv" + ROWS="$BATS_TEST_TMPDIR/rows.tsv" + OUT="$BATS_TEST_TMPDIR/plan.tsv" + EXC="$BATS_TEST_TMPDIR/exceptions.txt" + + # A discography pack: the artist's own albums buried under "01. Albums", + # with a lyric, a cover, an EAC log and a playlist beside the tracks. + A="$MUSIC/Artist One - Discography [FLAC]/01. Albums/Album X" + at "$A/01. First.flac" 9000 + at "$A/02. Second.flac" 9000 + at "$A/01. First.lrc" 10 + at "$A/cover.jpg" 20 + at "$A/rip.log" 10 + at "$A/Album X.m3u" 10 + + # A multi-disc compilation: one album (the album artist is the compilation + # tag), two discs, and a different performer on each track. + B="$MUSIC/Box (2019)" + at "$B/Disc 1/01 Track A.flac" 9000 + at "$B/Disc 2/01 Track B.flac" 9000 + +# The box cover lies above the two disc folders: the discs below it own it. +at "$B/box.jpg" 10 + + # One image plus its cuesheet: a whole album in one file. + C="$MUSIC/Image Cue" + at "$C/Image.flac" 9000 + at "$C/Image.cue" 10 + + # The hard cases: no row at all, a video container, two tracks claiming one + # number and one title, a track with no number, and a track whose album + # artist tag is empty so only the performer names it. + D="$MUSIC/Odd" + at "$D/loose.flac" 9000 + at "$D/concert.mkv" 9000 + at "$D/dup1.flac" 9000 + at "$D/dup2.flac" 9000 + at "$D/unnumbered.flac" 9000 + at "$D/lone.flac" 9000 + + # A sidecar in a folder that holds two different albums. + E="$MUSIC/Mixed" + at "$E/one.flac" 9000 + at "$E/two.flac" 9000 + at "$E/notes.pdf" 10 + + # An album title carrying a character Jellyfin says breaks playback. + F="$MUSIC/Question Mark" + at "$F/01. Bang.flac" 9000 + + # An album whose tags name no album artist at all, and two different + # performers; and one album title carried by two different album artists. + G="$MUSIC/Tagless" + at "$G/one.flac" 9000 + at "$G/two.flac" 9000 + H="$MUSIC/Same Title" + at "$H/a.flac" 9000 + at "$H/b.flac" 9000 + + # One album whose tracks disagree on the spelling of its name; the minority + # spelling sorts first, so it must not win the folder. + I="$MUSIC/Puppets" + at "$I/one.flac" 9000 + at "$I/two.flac" 9000 + at "$I/three.flac" 9000 + + # The authority export carries a header; the planner reads it by column name. + printf 'file\tartist\talbum\tyear\tdisc\ttrack\ttitle\tmb\tperformer\n' > "$ROWS" + row "$A/01. First.flac" "Artist One" "Album X" 2015 1 1 "First Song" + row "$A/02. Second.flac" "Artist One" "Album X" 2015 1 2 "Second Song: Reprise" + row "$B/Disc 1/01 Track A.flac" "Various Artists" "Best Of" 2019 1 1 "Track A" \ + "" "Aaa Singer" + row "$B/Disc 2/01 Track B.flac" "Various Artists" "Best Of" 2019 2 1 "Track B" \ + "" "Bbb Singer" + row "$C/Image.flac" "Solo" "Single Album" 2001 1 1 "Single Album" + row "$D/dup1.flac" "Dup Artist" "Dupes" "" "" 1 "Same" + row "$D/dup2.flac" "Dup Artist" "Dupes" "" "" 1 "Same" + row "$D/unnumbered.flac" "No Number" "No Number Album" "" "" "" "Untitled" + row "$D/lone.flac" "" "Session Works" "" "" 1 "Lone" "" "Session Singer" + row "$E/one.flac" "Mix Artist" "First Set" 2000 "" 1 "One" + row "$E/two.flac" "Mix Artist" "Second Set" 2001 "" 1 "Two" + row "$F/01. Bang.flac" "Billie" "WHEN WE ALL FALL ASLEEP, WHERE DO WE GO?" \ + 2019 "" 1 "!!!!!!" + row "$G/one.flac" "" "Two Seasons" 2005 "" 1 "One" "" "Alpha Band" + row "$G/two.flac" "" "Two Seasons" 2005 "" 2 "Two" "" "Beta Band" + row "$H/a.flac" "Aaa Singer" "Same Title" 2011 "" 1 "A Song" + row "$H/b.flac" "Bbb Singer" "Same Title" 2011 "" 1 "B Song" + row "$I/one.flac" "Metallica" "Master Of Puppets" 2015 "" 1 "One" + row "$I/two.flac" "Metallica" "Master of Puppets" 2015 "" 2 "Two" + row "$I/three.flac" "Metallica" "Master of Puppets" 2015 "" 3 "Three" + + # The enum TSV is what scripts/media-census.py enumerate writes: bytes, ext, + # path, no header. + python3 - "$MUSIC" "$ENUM" <<'PY' +import os, sys +root, out = sys.argv[1], sys.argv[2] +with open(out, "w") as fh: + for dirpath, _dirs, files in os.walk(root): + for fn in files: + p = os.path.join(dirpath, fn) + fh.write(f"{os.path.getsize(p)}\t{os.path.splitext(fn)[1].lower()}\t{p}\n") +PY +} + +plan() { + run python3 "$PLAN" --root "$MUSIC" --enum "$ENUM" --rows "$ROWS" \ + --out "$OUT" --exceptions "$EXC" + [ "$status" -eq 0 ] || echo "planner said: $output" +} + +@test "the census exports the tag and provider row Jellyfin matched for a track" { + DB="$BATS_TEST_TMPDIR/jellyfin.db" + python3 - "$DB" <<'PY' +import sqlite3, sys, uuid +c = sqlite3.connect(sys.argv[1]) +c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber, + ParentIndexNumber, ProductionYear, Album, + AlbumArtists, Artists, Data)""") +c.execute("""create table BaseItemProviders (ItemId, ProviderId, ProviderValue)""") +AU = "MediaBrowser.Controller.Entities.Audio.Audio" +AL = "MediaBrowser.Controller.Entities.Audio.MusicAlbum" +aid, trk = str(uuid.uuid4()), "/data/Music/Pack/Album X/01. First.flac" +c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?,?)", + (aid, None, AL, "Album X", "/data/Music/Pack/Album X", None, None, 2015, + None, "Artist One", None, '{}')) +c.execute("insert into BaseItemProviders values (?,?,?)", + (aid, "MusicBrainzReleaseGroup", "rg-1234")) +c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?,?)", + (str(uuid.uuid4()), aid, AU, "First Song", trk, 1, 1, 2015, + "album x (the tag)", "Artist One", "Artist One", '{}')) +c.commit() +PY + run python3 "$CENSUS" music --root "$BATS_TEST_TMPDIR/Music" --db "$DB" \ + --out "$BATS_TEST_TMPDIR/exp.tsv" + [ "$status" -eq 0 ] + run head -1 "$BATS_TEST_TMPDIR/exp.tsv" + [ "${lines[0]}" = "file artist album year disc track title mb performer" ] + tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \ + | grep -q "^$BATS_TEST_TMPDIR/Music/Pack/Album X/01\. First\.flac Artist One Album X 2015 1 1 First Song rg-1234 Artist One$" +} + +@test "a track whose folder is no album item still exports from its own tags" { + DB="$BATS_TEST_TMPDIR/jellyfin.db" + python3 - "$DB" <<'PY' +import sqlite3, sys, uuid +c = sqlite3.connect(sys.argv[1]) +c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber, + ParentIndexNumber, ProductionYear, Album, + AlbumArtists, Artists, Data)""") +c.execute("""create table BaseItemProviders (ItemId, ProviderId, ProviderValue)""") +c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?,?)", + (str(uuid.uuid4()), None, "MediaBrowser.Controller.Entities.Audio.Audio", + "Lone", "/data/Music/Pack/lone.flac", 1, None, None, + "Session Works", None, "Session Singer", '{}')) +c.commit() +PY + run python3 "$CENSUS" music --root "$BATS_TEST_TMPDIR/Music" --db "$DB" \ + --out "$BATS_TEST_TMPDIR/exp.tsv" + [ "$status" -eq 0 ] + tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \ + | grep -q "^$BATS_TEST_TMPDIR/Music/Pack/lone\.flac Session Singer Session Works 1 Lone Session Singer$" +} + +@test "a tagged track files under artist, album and year with a numbered title" { + plan + [ "$status" -eq 0 ] + [ "$(dst "$A/01. First.flac")" = "$MUSIC/Artist One/Album X (2015)/01 - First Song.flac" ] +} + +@test "the tag title wins over the name the ripper gave the file" { + plan + [ "$(dst "$A/02. Second.flac")" = "$MUSIC/Artist One/Album X (2015)/02 - Second Song Reprise.flac" ] +} + +@test "a character Jellyfin says breaks playback never reaches a destination" { + plan + [ "$(dst "$F/01. Bang.flac")" = "$MUSIC/Billie/WHEN WE ALL FALL ASLEEP, WHERE DO WE GO (2019)/01 - !!!!!!.flac" ] + run awk -F '\t' '$1 == "move" && $3 ~ /[<>:"|?*]/ {print $3}' "$OUT" + [ -z "$output" ] +} + +@test "every disc of a multi-disc album lands in one folder, numbered apart" { + plan + [ "$(dst "$B/Disc 1/01 Track A.flac")" = "$MUSIC/Various Artists/Best Of (2019)/1.01 - Track A.flac" ] + [ "$(dst "$B/Disc 2/01 Track B.flac")" = "$MUSIC/Various Artists/Best Of (2019)/2.01 - Track B.flac" ] +} + +@test "an album whose tracks name several artists files under Various Artists" { + plan + run awk -F '\t' '$1 == "move" && $3 ~ /\/Best Of \(2019\)\/.*\.flac$/ {print $3}' "$OUT" + [ "${#lines[@]}" -eq 2 ] + [[ "${lines[0]}" == "$MUSIC/Various Artists/"* ]] +} + +@test "an album with no album artist and two performers is a compilation" { + plan + [ "$(dst "$G/one.flac")" = "$MUSIC/Various Artists/Two Seasons (2005)/01 - One.flac" ] + [ "$(dst "$G/two.flac")" = "$MUSIC/Various Artists/Two Seasons (2005)/02 - Two.flac" ] +} + +@test "the same album title under two album artists is two albums" { + plan + [ "$(dst "$H/a.flac")" = "$MUSIC/Aaa Singer/Same Title (2011)/01 - A Song.flac" ] + [ "$(dst "$H/b.flac")" = "$MUSIC/Bbb Singer/Same Title (2011)/01 - B Song.flac" ] +} + +@test "the folder carries the album spelling most tracks carry" { + plan + [ "$(dst "$I/one.flac")" = "$MUSIC/Metallica/Master of Puppets (2015)/01 - One.flac" ] + [ "$(dst "$I/two.flac")" = "$MUSIC/Metallica/Master of Puppets (2015)/02 - Two.flac" ] + [ "$(dst "$I/three.flac")" = "$MUSIC/Metallica/Master of Puppets (2015)/03 - Three.flac" ] +} + +@test "a lyric file is renamed to match the track it belongs to" { + plan + [ "$(dst "$A/01. First.lrc")" = "$MUSIC/Artist One/Album X (2015)/01 - First Song.lrc" ] +} + +@test "the album image, the rip log and the playlist follow the album folder" { + plan + [ "$(dst "$A/cover.jpg")" = "$MUSIC/Artist One/Album X (2015)/cover.jpg" ] + [ "$(dst "$A/rip.log")" = "$MUSIC/Artist One/Album X (2015)/rip.log" ] + [ "$(dst "$A/Album X.m3u")" = "$MUSIC/Artist One/Album X (2015)/Album X.m3u" ] +} + +@test "a cover above the disc folders of one album follows that album" { + plan + [ "$(dst "$B/box.jpg")" = "$MUSIC/Various Artists/Best Of (2019)/box.jpg" ] +} + +@test "an image carrying a whole album keeps its own name beside its cuesheet" { + plan + [ "$(dst "$C/Image.flac")" = "$MUSIC/Solo/Single Album (2001)/Image.flac" ] + [ "$(dst "$C/Image.cue")" = "$MUSIC/Solo/Single Album (2001)/Image.cue" ] +} + +@test "an audio file no provider row names is parked, not filed under an artist" { + plan + [[ "$(dst "$D/loose.flac")" == "$PARK/Odd/loose.flac" ]] +} + +@test "a video container in the music library is parked, never renamed to an audio one" { + plan + [[ "$(dst "$D/concert.mkv")" == "$PARK/Odd/concert.mkv" ]] + run awk -F '\t' '$1 == "move" && $3 ~ /\.mka$|\.m4a$/ {print $3}' "$OUT" + [ -z "$output" ] +} + +@test "two tracks claiming one number and one title get two destinations" { + plan + [ "$(dst "$D/dup1.flac")" = "$MUSIC/Dup Artist/Dupes/01 - Same.flac" ] + [ "$(dst "$D/dup2.flac")" = "$MUSIC/Dup Artist/Dupes/01 - Same (2).flac" ] +} + +@test "an album with no year is named without one, a track with no number likewise" { + plan + [ "$(dst "$D/unnumbered.flac")" = "$MUSIC/No Number/No Number Album/Untitled.flac" ] +} + +@test "a track naming only a performer files under that performer" { + plan + [ "$(dst "$D/lone.flac")" = "$MUSIC/Session Singer/Session Works/01 - Lone.flac" ] +} + +@test "a sidecar in a folder holding two albums parks instead of guessing" { + plan + [[ "$(dst "$E/notes.pdf")" == "$PARK/Mixed/notes.pdf" ]] +} + +@test "the plan creates every folder before the move that needs it" { + plan + python3 - "$OUT" "$MUSIC" <<'PY' +import os, sys +made = set() +for line in open(sys.argv[1]).read().splitlines()[1:]: + action, _src, dest, _kind = line.split("\t") + if action == "mkdir": + made.add(dest) + elif action == "move": + parent = os.path.dirname(dest) + while parent and parent != sys.argv[2]: + assert parent in made, f"{dest} needs {parent}, which is never made" + parent = os.path.dirname(parent) +PY +} + +@test "no move lands outside the music library" { + plan + run awk -F '\t' -v l="$MUSIC" \ + '$1 == "move" && $3 !~ "^" l "/" {print $3}' "$OUT" + [ -z "$output" ] +} + +@test "the executor preflights the plan and no target is claimed twice" { + plan + run python3 "$APPLY" apply --plan "$OUT" --root "$BATS_TEST_TMPDIR" \ + --journal "$BATS_TEST_TMPDIR/journal.jsonl" + [ "$status" -eq 0 ] + dup=$(awk -F '\t' '$1 == "move" {print $3}' "$OUT" | sort | uniq -d) + [ -z "$dup" ] +} + +@test "re-planning a library the plan already satisfies moves nothing" { + plan + # Applying the plan and re-censusing is what the operator does next: both + # authority files are keyed by path, so both are replayed through the plan + # the way a fresh census of the moved tree would see them. (The enum has no + # header; the rows export does, and its header row is not a path.) + python3 - "$OUT" "$ENUM" "$ROWS" <<'PY' +import os, sys +plan = sys.argv[1] +moves = {} +for line in open(plan).read().splitlines()[1:]: + action, src, dest, _kind = line.split("\t") + if action == "move": + moves[src] = dest + os.makedirs(os.path.dirname(dest), exist_ok=True) + os.rename(src, dest) +for path, col in ((sys.argv[2], 2), (sys.argv[3], 0)): + rows = [line.rstrip("\n").split("\t") for line in open(path)] + with open(path, "w") as fh: + for f in rows: + f[col] = moves.get(f[col], f[col]) + fh.write("\t".join(f) + "\n") +PY + plan + run awk -F '\t' '$1 == "move" {print $2 " -> " $3}' "$OUT" + [ -z "$output" ] || { echo "moved again: $output"; false; } +}