Compare commits
67
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
75b04dc313 | ||
|
|
2625ad1083 | ||
|
|
9c25b1cccb | ||
|
|
7aa32d1b64 | ||
|
|
576c3421b8 | ||
|
|
e4656cfd7a | ||
|
|
acab4cef22 | ||
|
|
04802b3e48 | ||
|
|
44e55ad172 | ||
|
|
132805a25a | ||
|
|
2406641bc6 | ||
|
|
dbcf8a2c35 | ||
|
|
a5a9c04da2 | ||
|
|
c95ae20bfb | ||
|
|
a0dfa61623 | ||
|
|
4b67c3c4d8 | ||
|
|
7ee831adee | ||
|
|
afad21dda5 | ||
|
|
f430e2264d | ||
|
|
82a2183d40 | ||
|
|
c70f9b3f6d | ||
|
|
32b8901bff | ||
|
|
6d03bbc8e7 | ||
|
|
32b58dd787 | ||
|
|
0a365274b6 | ||
|
|
e531e90278 | ||
|
|
c3ab565d9b | ||
|
|
5dbbfcb8a0 | ||
|
|
02a568fb97 | ||
|
|
1e1527d101 | ||
|
|
9eb85ea1f9 | ||
|
|
96944f0b17 | ||
|
|
33f97cd3b3 | ||
|
|
254fb41fd0 | ||
|
|
3613f69593 | ||
|
|
6bf70c6a64 | ||
|
|
f1b8b60817 | ||
|
|
4ae2bde9ab | ||
|
|
ceb5617a42 | ||
|
|
a7ceda95fd | ||
|
|
a32361ee5e | ||
|
|
ce6b3b1d3b | ||
|
|
382e3eb0b8 | ||
|
|
54a393edab | ||
|
|
a5459b0b1e | ||
|
|
b03931f7ce | ||
|
|
82f6ec55c2 | ||
|
|
536949c18d | ||
|
|
337afac91e | ||
|
|
c0c77590b1 | ||
|
|
441b908937 | ||
|
|
1a1dbac268 | ||
|
|
949d3cb6f4 | ||
|
|
3ce8d62e70 | ||
|
|
38737ab84e | ||
|
|
4437519430 | ||
|
|
3b29e173c2 | ||
|
|
9706f5d2de | ||
|
|
206dc3baf1 | ||
|
|
f5a3badb56 | ||
|
|
3e4b4f24f7 | ||
|
|
86f5903e74 | ||
|
|
bd4ee78811 | ||
|
|
b7b39cdc92 | ||
|
|
09532e5216 | ||
|
|
e1786e7cf2 | ||
|
|
409a8d8b0c |
No files matched your search
@@ -808,6 +808,7 @@ per-attempt page and no per-swap ledger anywhere else.
|
||||
| Dankclaw kernel posture (`iommu=pt` since 2026-10-18) | `serverAppConfigs-docs/LLM/dankclaw-iommu-translation-off.md` |
|
||||
| Deployment flow | `serverAppConfigs-docs/Architecture/Deployment Flow.md` |
|
||||
| Movie-library relayout (Jellyfin format, Dankserver) | `serverAppConfigs-docs/Dankserver/Media Relayout.md` |
|
||||
| SSD-backed metadata view (Jellyfin writes, Plex reads, Dankserver) | `serverAppConfigs-docs/Dankserver/Media Metadata View.md` |
|
||||
|
||||
There is no page for a retired endpoint and no change diary. A payload swap
|
||||
appends one row to the live endpoint's **Eras** table; when the endpoint itself
|
||||
@@ -850,5 +851,6 @@ history.
|
||||
| Benchmarking (only when ordered) | **Stop and mask the service first**, bench, unmask, restart, verify health. |
|
||||
| Deleting a file | **Move it to trash** — `.AGENTTRASH/` (in-repo) or `~/.AGENTTRASH/YYYYMMDD-topic`. Never `rm`. |
|
||||
| Relayout the movie library | **Census → plan → read the exceptions → `scripts/media-apply.py`** — renames only, journal before every move; never hand-rename, never copy. |
|
||||
| Making media metadata visible to both media servers | **Bind `/var/lib/media-meta/view`** — the overlay `scripts/media-view.sh` installs (array below, NVMe above). Never write the array; never delete through the view. **Not yet deployed on Dankserver: the live quadlets still bind the array read-only — see [[Media Metadata View]].** |
|
||||
| Tests exist | **Run them** — verify tests pass after every change. Do not commit failing tests. |
|
||||
| Real secrets | **Never commit** — host `.env` files; `.env.example` holds placeholders only. |
|
||||
+6
-28
@@ -94,34 +94,12 @@ services:
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# Jellyfin — media server (movies, TV, music)
|
||||
# ═══════════════════════════════════════════════
|
||||
jellyfin:
|
||||
image: jellyfin/jellyfin:latest
|
||||
container_name: jellyfin
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8096:8096/tcp"
|
||||
- "8920:8920/tcp"
|
||||
- "7359:7359/udp"
|
||||
- "1900:1900/udp"
|
||||
environment:
|
||||
- JELLYFIN_PublishedServerUrl=http://192.168.0.235:8096
|
||||
- DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true
|
||||
volumes:
|
||||
- /opt/jellyfin/config:/config
|
||||
- /opt/jellyfin/cache:/cache
|
||||
- /mnt/entropy:/data:ro
|
||||
devices:
|
||||
- /dev/dri:/dev/dri
|
||||
group_add:
|
||||
- "992"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8096/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
# Jellyfin left this file when it became a Quadlet: the container is owned by
|
||||
# dankserver/jellyfin/quadlet/jellyfin.container, its data roots are under
|
||||
# /home/dank/Containers/jellyfin, and its library arrives through the SSD
|
||||
# overlay at /var/lib/media-meta/view (scripts/media-view.sh). The block that
|
||||
# was here named /opt/jellyfin/* and a read-only array bind — a second owner for
|
||||
# the container name `jellyfin`, wrong on every path. One owner per container.
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# Calibre-Web — e-book web frontend
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
entropy UUID=cd86c728-459e-4f3a-bcc2-41ecf84685ec /etc/entropy nofail,luks
|
||||
@@ -0,0 +1,18 @@
|
||||
# /etc/fstab: static file system information.
|
||||
#
|
||||
# Use 'blkid' to print the universally unique identifier for a
|
||||
# device; this may be used with UUID= as a more robust way to name devices
|
||||
# that works even if disks are added and removed. See fstab(5).
|
||||
#
|
||||
# <file system> <mount point> <type> <options> <dump> <pass>
|
||||
# / was on /dev/nvme0n1p2 during curtin installation
|
||||
/dev/disk/by-uuid/4f3731d2-fb7b-48e7-a79f-0e33cb8a4162 / ext4 defaults 0 1
|
||||
# /boot/efi was on /dev/nvme0n1p1 during curtin installation
|
||||
/dev/disk/by-uuid/183C-442D /boot/efi vfat defaults 0 1
|
||||
#/dev/disk/by-label/archive-one /mnt/archive-one auto nosuid,nodev,nofail,x-gvfs-show 0 0
|
||||
#/dev/disk/by-label/archive-two /mnt/archive-two auto nosuid,nodev,nofail,x-gvfs-show 0 0
|
||||
# /dev/md0 is a LUKS container, not a filesystem; its ext4 is mounted via
|
||||
# /dev/mapper/entropy on the next line. Mounting /dev/md0 directly fails.
|
||||
#/dev/md0 /mnt/md0 ext4 auto,nofail,nosuid,nodev 0 0
|
||||
/dev/mapper/entropy /mnt/entropy ext4 auto,nosuid,nofail,nodev,x-gvfs-show 0 0
|
||||
/swapfile none swap sw 0 0
|
||||
@@ -0,0 +1,30 @@
|
||||
# /etc/systemd/system/var-lib-media\x2dmeta-view.mount
|
||||
# (repo copy: dankserver/host/systemd/media-meta-view.mount — systemd names a
|
||||
# mount unit after its ESCAPED path, so the host file carries the \x2d spelling)
|
||||
#
|
||||
# Installed by `scripts/media-view.sh install` — the repo copy is the source of
|
||||
# truth (fleet law: never edit host state by hand).
|
||||
#
|
||||
# An OverlayFS merge that shows the media library as one tree while splitting
|
||||
# where its bytes live: lowerdir is the RAID6 array (the source of truth, never
|
||||
# written), upperdir is a directory on the NVMe that receives every write. The
|
||||
# two media servers bind the result at /var/lib/media-meta/view, so Jellyfin's
|
||||
# sidecars and artwork are SSD bytes — which is what makes them fast to browse
|
||||
# and what keeps Plex's NFO agent off the spinners — while the media itself stays
|
||||
# on the array untouched.
|
||||
#
|
||||
# RequiresMountsFor is load-bearing: without the array mounted, lowerdir is an
|
||||
# empty directory and the view would present an empty library as truth.
|
||||
|
||||
[Unit]
|
||||
Description=SSD-backed overlay view of the media library (array + NVMe metadata layer)
|
||||
RequiresMountsFor=/mnt/entropy
|
||||
|
||||
[Mount]
|
||||
What=overlay
|
||||
Where=/var/lib/media-meta/view
|
||||
Type=overlay
|
||||
Options=lowerdir=/mnt/entropy,upperdir=/var/lib/media-meta/upper,workdir=/var/lib/media-meta/work
|
||||
|
||||
[Install]
|
||||
WantedBy=local-fs.target
|
||||
@@ -4,8 +4,9 @@
|
||||
# Unit name: jellyfin.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# Data roots migrated from /opt/jellyfin to /home/dank/Containers/jellyfin at
|
||||
# cutover. All of /mnt/entropy is mounted read-only at /data: nothing automated
|
||||
# anywhere writes to the entropy volume. Every
|
||||
# cutover. The library is seen through /var/lib/media-meta/view, the SSD-backed
|
||||
# overlay view (scripts/media-view.sh): the array stays read-only and untouched,
|
||||
# while the sidecars this server writes land on the NVMe. Every
|
||||
# creation-time environment variable of the compose-era container is kept as
|
||||
# an Environment= line (zero drift); none of them are secrets, so this unit
|
||||
# needs no EnvironmentFile.
|
||||
@@ -41,7 +42,12 @@ PublishPort=7359:7359/udp
|
||||
PublishPort=1900:1900/udp
|
||||
Volume=/home/dank/Containers/jellyfin/config:/config
|
||||
Volume=/home/dank/Containers/jellyfin/cache:/cache
|
||||
Volume=/mnt/entropy:/data:ro
|
||||
# The view, writable: Jellyfin is the metadata writer (movie.nfo / tvshow.nfo /
|
||||
# episode.nfo + artwork) and those bytes belong on the SSD. The condition is the
|
||||
# fail-closed guard — starting against a plain, unmounted view directory would let
|
||||
# a scan conclude the library is empty.
|
||||
Volume=/var/lib/media-meta/view:/data
|
||||
UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view
|
||||
AddDevice=/dev/dri
|
||||
GroupAdd=keep-groups
|
||||
HealthCmd=curl -f http://localhost:8096/health
|
||||
|
||||
@@ -11,7 +11,12 @@ Environment=PUID=0
|
||||
Environment=PGID=0
|
||||
Environment=TZ=America/Los_Angeles
|
||||
Volume=/var/lib/plexmediaserver:/config
|
||||
Volume=/mnt/entropy:/mnt/entropy:ro
|
||||
# The container path stays /mnt/entropy on purpose: every path Plex recorded in
|
||||
# its own database is written against it, so swapping the HOST source of this bind
|
||||
# to the overlay view re-points Plex at the sidecars without touching its DB.
|
||||
# Read-only: Plex consumes the NFO files Jellyfin writes and writes no metadata.
|
||||
Volume=/var/lib/media-meta/view:/mnt/entropy:ro
|
||||
UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view
|
||||
PublishPort=32400:32400
|
||||
|
||||
[Service]
|
||||
|
||||
+16
-1
@@ -155,13 +155,14 @@ DANKCLAW_COMPOSE_SERVICES = set()
|
||||
# Quadlet unit now — the last compose services were the hermes trio, whose units
|
||||
# have owned their containers since 2026-09-12 and whose compose blocks were
|
||||
# dead weight that could only create a duplicate container.
|
||||
# jellyfin left the Dankserver file on 2026-10-30 for the same reason: its Quadlet
|
||||
# has owned the container since the cutover, so a compose edit must not restart it.
|
||||
|
||||
DANKSERVER_COMPOSE_SERVICES = {
|
||||
"mariadb",
|
||||
"redis",
|
||||
"romm",
|
||||
"copyparty",
|
||||
"jellyfin",
|
||||
"calibre-web",
|
||||
}
|
||||
COMPOSE_NAME_MAP = {"mariadb": "romm-mariadb", "redis": "romm-redis"}
|
||||
@@ -421,6 +422,11 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
"tautulli": {"tautulli"},
|
||||
}
|
||||
_add(services, *quadlet_services.get(parts[1], set()))
|
||||
if len(parts) >= 2 and parts[1] == "host":
|
||||
# dankserver/host/systemd/media-meta-view.mount is the overlay view the
|
||||
# media servers bind (scripts/media-view.sh installs it on the host),
|
||||
# so editing it restarts jellyfin and plex and nothing else.
|
||||
_add(services, "jellyfin", "plex")
|
||||
if len(parts) >= 2 and parts[1] == "observability":
|
||||
# Dankserver's node-exporter became a Quadlet unit on 2026-09-16
|
||||
# (serverAppConfigs-docs/Infrastructure/Observability.md): its
|
||||
@@ -466,6 +472,15 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
# that service (target_services then keeps it only on the host being
|
||||
# planned). Editing the collector restarts the running exporter.
|
||||
_add(services, "gpu-exporter")
|
||||
if name == "media-view.sh":
|
||||
# The installer is what creates, mounts and reaps the view both media
|
||||
# servers read, so a change to it is a restart of that pair.
|
||||
_add(services, "jellyfin", "plex")
|
||||
if name == "jellyfin-providers.sh":
|
||||
# The deployer runs this before jellyfin restarts, so a change to
|
||||
# the installer is a change to the provider state Jellyfin loads at
|
||||
# start-up: it selects jellyfin and nothing else.
|
||||
_add(services, "jellyfin")
|
||||
if name in {"frank-zealot.env", "frank-zealot.service", "serve-frank-zealot.sh"}:
|
||||
# llama.cpp frank-zealot (2026-09-16) keeps its unit, env and serve
|
||||
# files here; each one selects frank-zealot only. The ik_llama.cpp
|
||||
|
||||
@@ -1202,7 +1202,42 @@ deploy_romm_stack() {
|
||||
systemctl --user restart "$service.service"
|
||||
}
|
||||
|
||||
ensure_media_view() {
|
||||
# Both media servers read their library through the SSD-backed overlay view
|
||||
# (scripts/media-view.sh, dankserver/host/systemd/media-meta-view.mount): the
|
||||
# array below stays read-only and every sidecar Jellyfin writes lands on the
|
||||
# NVMe.
|
||||
# Raise it BEFORE either container starts — a start against a bare view
|
||||
# directory would let a scan clear the library database. The units also carry
|
||||
# ConditionPathIsMountPoint, so a missing view skips the start rather than
|
||||
# damaging anything. A relayout must be followed by `media-view.sh reap`
|
||||
# (see the Media Relayout page), which this is not.
|
||||
run_privileged bash scripts/media-view.sh install
|
||||
bash scripts/media-view.sh verify
|
||||
}
|
||||
|
||||
ensure_provider_keys() {
|
||||
# Jellyfin's metadata-provider keys live in the container-owned plugin
|
||||
# configuration under the data root, never in git; scripts/jellyfin-providers.sh
|
||||
# is what places them, reading the per-service host env file (the fleet
|
||||
# convention: grafana, caddy, pihole, copyparty, romm, seerr) that holds
|
||||
# TMDB_API_KEY= and TVDB_API_KEY=, mode 600.
|
||||
# Missing keys mean a keyless provider — a degraded library, not a broken
|
||||
# deploy — so this warns rather than failing the restart. But when the keys
|
||||
# ARE placed it must happen before the restart: the plugin and its keys are
|
||||
# read at start-up, so a restart first would boot the old state.
|
||||
local keys=/home/dank/Containers/jellyfin/provider-keys
|
||||
if [ -f "$keys" ]; then
|
||||
bash scripts/jellyfin-providers.sh install
|
||||
else
|
||||
echo "WARN: $keys is absent - provider keys NOT placed"
|
||||
echo "WARN: create it (TMDB_API_KEY=, TVDB_API_KEY=, chmod 600) and re-deploy jellyfin"
|
||||
fi
|
||||
}
|
||||
|
||||
deploy_jellyfin() {
|
||||
ensure_media_view
|
||||
ensure_provider_keys
|
||||
cp "dankserver/jellyfin/quadlet/jellyfin.container" "$quadlet_unit_dir/"
|
||||
systemctl --user daemon-reload
|
||||
if ! systemctl --user is-active -q jellyfin.service; then
|
||||
@@ -1290,6 +1325,7 @@ deploy_copyparty() {
|
||||
}
|
||||
|
||||
deploy_plex() {
|
||||
ensure_media_view
|
||||
cp "dankserver/plex/quadlet/plex.container" "$quadlet_unit_dir/"
|
||||
systemctl --user daemon-reload
|
||||
if ! systemctl --user is-active -q plex.service; then
|
||||
|
||||
Executable
+391
@@ -0,0 +1,391 @@
|
||||
#!/usr/bin/env bash
|
||||
# Metadata-provider keys for Jellyfin on Dankserver, placed from the service's
|
||||
# host env file with the plugin artifact pinned by content hash.
|
||||
#
|
||||
# Jellyfin 12 ships TMDb, OMDb, StudioImages and MusicBrainz as CORE plugins:
|
||||
# they have no DLL, only a configuration file under the data root, and that
|
||||
# file is created by the running container. So the fleet's provider state lives
|
||||
# in two places the repo does not track —
|
||||
#
|
||||
# plugins/configurations/Jellyfin.Plugin.Tmdb.xml <TmdbApiKey /> (empty = keyless)
|
||||
# plugins/configurations/Jellyfin.Plugin.Tvdb.xml <ApiKey /> (needs the plugin)
|
||||
#
|
||||
# and both are owned by the container user. This tool is the only thing that
|
||||
# writes them: it is declarative, idempotent, and it never prints a key, so it
|
||||
# is safe to run while someone watches.
|
||||
#
|
||||
# Keys never enter git and never enter argv (a value in argv is world-readable
|
||||
# through /proc/<pid>/cmdline). They live in the per-service host env file that
|
||||
# is the fleet's convention for container credentials (grafana, caddy, pihole,
|
||||
# copyparty, romm, seerr), mode 600:
|
||||
#
|
||||
# /home/dank/Containers/jellyfin/provider-keys
|
||||
# TMDB_API_KEY=<key>
|
||||
# TVDB_API_KEY=<key>
|
||||
# TVDB_PIN=<pin> # optional: only a subscriber-shaped TVDB key
|
||||
# # needs one; a licensed key does not
|
||||
#
|
||||
# TVDB itself is not core in 12 (the metadata provider was cut in 10.9), so it
|
||||
# is installed from the official catalog, pinned three ways: sha256 of the
|
||||
# artifact, the plugin's own guid, and its version. A matching sha proves only
|
||||
# that the bytes are the bytes fetched — the guid check is what says it is the
|
||||
# official plugin, the version check is what says it is the one this file was
|
||||
# reviewed against.
|
||||
#
|
||||
# Subcommands
|
||||
# plan print the resolved shape and every pending change, touch nothing
|
||||
# install fetch+verify the plugin, then write the keys (backing up first)
|
||||
# verify prove the plugin and both keys are actually in place
|
||||
#
|
||||
# Nothing here restarts Jellyfin: the deployer owns that
|
||||
# (`scripts/deploy.sh dankserver --service jellyfin`), and the plugin is read at
|
||||
# start-up, so a key written while the server runs is not live until then.
|
||||
set -euo pipefail
|
||||
|
||||
# Jellyfin's data root on Dankserver (the rootless quadlet's config volume).
|
||||
JF_ROOT="${JF_ROOT:-/home/dank/Containers/jellyfin/config}"
|
||||
PROV_KEYS="${PROV_KEYS:-/home/dank/Containers/jellyfin/provider-keys}"
|
||||
TRASH_DIR="${TRASH_DIR:-$HOME/.AGENTTRASH/$(date +%Y%m%d)-jellyfin-providers}"
|
||||
|
||||
PLUGINS="$JF_ROOT/plugins"
|
||||
CONF="$PLUGINS/configurations"
|
||||
TMDB_XML="$CONF/Jellyfin.Plugin.Tmdb.xml"
|
||||
TVDB_XML="$CONF/Jellyfin.Plugin.Tvdb.xml"
|
||||
|
||||
# repo.jellyfin.org catalog entry for TheTVDB (owner: jellyfin), version 24.0.0.0,
|
||||
# targetAbi 12.0.0.0 — published 2026-09-09, verified against Jellyfin 12.4.0.
|
||||
TVDB_PLUGIN_URL="${TVDB_PLUGIN_URL:-https://repo.jellyfin.org/files/plugin/thetvdb/thetvdb_24.0.0.0.zip}"
|
||||
TVDB_PLUGIN_SHA256="${TVDB_PLUGIN_SHA256:-6302fba3857bdde8d186dd099fac09ca43247041f531a528a2c788add70a5319}"
|
||||
TVDB_PLUGIN_GUID="${TVDB_PLUGIN_GUID:-a677c0da-fac5-4cde-941a-7134223f14c8}"
|
||||
TVDB_PLUGIN_VERSION="${TVDB_PLUGIN_VERSION:-24.0.0.0}"
|
||||
PLUGIN_DIR="thetvdb_${TVDB_PLUGIN_VERSION}"
|
||||
TVDB_DLL="Jellyfin.Plugin.Tvdb.dll"
|
||||
|
||||
die() {
|
||||
printf 'jellyfin-providers: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
usage() {
|
||||
printf 'usage: %s plan|install|verify\n' "$0" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
# check_keys — refuse anything that is not a readable private file. An open
|
||||
# mode is an error, not a warning: a provider key readable by every user on the
|
||||
# box is the failure this check exists to catch.
|
||||
check_keys() {
|
||||
[ -f "$PROV_KEYS" ] ||
|
||||
die "$PROV_KEYS not found (create it with TMDB_API_KEY= and TVDB_API_KEY=, then chmod 600)"
|
||||
local mode
|
||||
mode="$(stat -c '%a' "$PROV_KEYS")"
|
||||
case "$mode" in
|
||||
600 | 400) ;;
|
||||
*) die "$PROV_KEYS is mode $mode; provider keys must be 600" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# keys_value <VAR> — the value of one variable in the env file, or nothing.
|
||||
keys_value() {
|
||||
awk -v k="$1" -F= '$1 == k { sub(/^[^=]*=/, ""); print; exit }' "$PROV_KEYS"
|
||||
}
|
||||
|
||||
# keys_len <VAR> — the length, and a refusal naming the variable when absent.
|
||||
# The value itself is never printed.
|
||||
keys_len() {
|
||||
local v
|
||||
v="$(keys_value "$1")"
|
||||
[ -n "$v" ] || die "$PROV_KEYS has no $1= line"
|
||||
printf '%s' "${#v}"
|
||||
}
|
||||
|
||||
# xml_value <file> <element> — the element's text, empty when it is absent or
|
||||
# self-closing.
|
||||
xml_value() {
|
||||
python3 - "$1" "$2" <<'PYEOF'
|
||||
import re, sys
|
||||
src = open(sys.argv[1], encoding="utf-8").read()
|
||||
m = re.search(r"<%s(?:\s*/>|>(.*?)</%s>)" % (sys.argv[2], sys.argv[2]), src, re.S)
|
||||
print(m.group(1).strip() if (m and m.group(1)) else "")
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# xml_secret <file> <element> <var> — set <element> to that variable from the
|
||||
# env file. The python process reads the env file itself, so the value never
|
||||
# appears in argv.
|
||||
xml_secret() {
|
||||
python3 - "$1" "$2" "$3" "$PROV_KEYS" <<'PYEOF'
|
||||
import os, re, sys, tempfile
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
xml_path, elem, var, keys_path = sys.argv[1:5]
|
||||
val = ""
|
||||
for line in open(keys_path, encoding="utf-8"):
|
||||
key, sep, rest = line.partition("=")
|
||||
if sep and key.strip() == var:
|
||||
val = rest.strip()
|
||||
break
|
||||
if not val:
|
||||
raise SystemExit(f"{keys_path} has no {var}= line")
|
||||
|
||||
src = open(xml_path, encoding="utf-8").read()
|
||||
pat = re.compile(r"([ \t]*)<%s(?:\s*/>|>.*?</%s>)" % (elem, elem), re.S)
|
||||
text = escape(val)
|
||||
if pat.search(src):
|
||||
out = pat.sub(lambda m: f"{m.group(1)}<{elem}>{text}</{elem}>", src, count=1)
|
||||
else:
|
||||
out = src.replace("</Configuration>", f" <{elem}>{text}</{elem}>\n</Configuration>", 1)
|
||||
if out == src:
|
||||
raise SystemExit(f"{xml_path}: no place to write <{elem}>")
|
||||
|
||||
d, base = os.path.split(xml_path)
|
||||
fd, tmp = tempfile.mkstemp(prefix=base + ".", dir=d)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
fh.write(out)
|
||||
os.chmod(tmp, os.stat(xml_path).st_mode & 0o777)
|
||||
os.replace(tmp, xml_path)
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# plugin_meta <zip> — "guid version" out of the artifact's own meta.json.
|
||||
plugin_meta() {
|
||||
python3 - "$1" <<'PYEOF'
|
||||
import json, sys, zipfile
|
||||
meta = json.loads(zipfile.ZipFile(sys.argv[1]).read("meta.json").decode("utf-8"))
|
||||
print(meta.get("guid", ""), meta.get("version", ""))
|
||||
PYEOF
|
||||
}
|
||||
|
||||
check_root() {
|
||||
# Never fabricate a data root: creating plugins/ beside a typo'd path would
|
||||
# install a plugin Jellyfin never loads and hide the operator's mistake.
|
||||
[ -d "$CONF" ] || die "no Jellyfin configuration directory at $CONF (is JF_ROOT right?)"
|
||||
}
|
||||
|
||||
check_keys_report() {
|
||||
check_keys
|
||||
# A refusal inside a command substitution used as a printf argument would
|
||||
# be swallowed, so the guard runs as its own statement.
|
||||
keys_len TMDB_API_KEY >/dev/null
|
||||
keys_len TVDB_API_KEY >/dev/null
|
||||
printf '= keys read from %s (mode %s)\n' "$PROV_KEYS" "$(stat -c '%a' "$PROV_KEYS")"
|
||||
}
|
||||
|
||||
# The plugin's state, as one line in the shared vocabulary.
|
||||
plugin_state() {
|
||||
if [ -d "$PLUGINS/$PLUGIN_DIR" ]; then
|
||||
printf '= %s already installed\n' "$PLUGIN_DIR"
|
||||
return 0
|
||||
fi
|
||||
local other
|
||||
other="$(find "$PLUGINS" -maxdepth 1 -type d -name 'thetvdb_*' ! -name "$PLUGIN_DIR" -printf '%f\n' 2>/dev/null | sort | tr '\n' ' ')"
|
||||
if [ -n "$other" ]; then
|
||||
printf '+ install %s (a different version is present: %s)\n' "$PLUGIN_DIR" "$other"
|
||||
return 0
|
||||
fi
|
||||
printf '+ install %s from %s\n' "$PLUGIN_DIR" "${TVDB_PLUGIN_URL#*://}"
|
||||
}
|
||||
|
||||
tmdb_state() {
|
||||
if [ ! -f "$TMDB_XML" ]; then
|
||||
printf '! %s is missing — save the TMDb plugin settings once in the UI, then re-run\n' "$TMDB_XML" >&2
|
||||
return 1
|
||||
fi
|
||||
local now want
|
||||
now="$(xml_value "$TMDB_XML" TmdbApiKey)"
|
||||
want="$(keys_value TMDB_API_KEY)"
|
||||
if [ -n "$now" ] && [ "$now" = "$want" ]; then
|
||||
printf '= TmdbApiKey already set (%s chars)\n' "${#now}"
|
||||
elif [ -n "$now" ]; then
|
||||
printf '+ replace the TmdbApiKey in Jellyfin.Plugin.Tmdb.xml\n'
|
||||
else
|
||||
printf '+ set TmdbApiKey in Jellyfin.Plugin.Tmdb.xml\n'
|
||||
fi
|
||||
}
|
||||
|
||||
tvdb_state() {
|
||||
local now
|
||||
if [ -f "$TVDB_XML" ]; then
|
||||
now="$(xml_value "$TVDB_XML" ApiKey)"
|
||||
if [ -n "$now" ]; then
|
||||
printf '= Jellyfin.Plugin.Tvdb.xml already has an ApiKey (kept)\n'
|
||||
else
|
||||
printf '+ fill the empty ApiKey in Jellyfin.Plugin.Tvdb.xml\n'
|
||||
fi
|
||||
else
|
||||
printf '+ create Jellyfin.Plugin.Tvdb.xml with ApiKey\n'
|
||||
fi
|
||||
}
|
||||
|
||||
write_plugin() {
|
||||
local tmp guid version other
|
||||
other="$(find "$PLUGINS" -maxdepth 1 -type d -name 'thetvdb_*' ! -name "$PLUGIN_DIR" -printf '%f\n' 2>/dev/null | sort | tr '\n' ' ')"
|
||||
[ -z "$other" ] ||
|
||||
die "another TheTVDB version is already installed ($other); move it to the trash first — two copies of one plugin means two loads"
|
||||
|
||||
tmp="$(mktemp "${TMPDIR:-/tmp}/thetvdb.XXXXXX.zip")"
|
||||
# The pins are checked before anything is unpacked, so a bad artifact never
|
||||
# reaches the plugins directory.
|
||||
curl -fsSL --max-time 300 -o "$tmp" "$TVDB_PLUGIN_URL" ||
|
||||
{ rm -f "$tmp"; die "could not fetch $TVDB_PLUGIN_URL"; }
|
||||
local got
|
||||
got="$(sha256sum "$tmp" | cut -d' ' -f1)"
|
||||
if [ "$got" != "$TVDB_PLUGIN_SHA256" ]; then
|
||||
rm -f "$tmp"
|
||||
die "artifact sha256 is $got, pinned $TVDB_PLUGIN_SHA256 — refusing to install"
|
||||
fi
|
||||
read -r guid version <<<"$(plugin_meta "$tmp")"
|
||||
if [ "$guid" != "$TVDB_PLUGIN_GUID" ]; then
|
||||
rm -f "$tmp"
|
||||
die "artifact guid is $guid, expected the official $TVDB_PLUGIN_GUID — refusing to install"
|
||||
fi
|
||||
if [ "$version" != "$TVDB_PLUGIN_VERSION" ]; then
|
||||
rm -f "$tmp"
|
||||
die "artifact version is $version, pinned $TVDB_PLUGIN_VERSION — refusing to install"
|
||||
fi
|
||||
mkdir -p "$PLUGINS/$PLUGIN_DIR"
|
||||
python3 -m zipfile -e "$tmp" "$PLUGINS/$PLUGIN_DIR/"
|
||||
rm -f "$tmp"
|
||||
[ -f "$PLUGINS/$PLUGIN_DIR/$TVDB_DLL" ] ||
|
||||
die "$PLUGIN_DIR has no $TVDB_DLL — the artifact does not match this plugin"
|
||||
printf '+ installed %s\n' "$PLUGIN_DIR"
|
||||
}
|
||||
|
||||
backup_to_trash() {
|
||||
local f="$1"
|
||||
mkdir -p "$TRASH_DIR"
|
||||
local dest="$TRASH_DIR/$(basename "$f").$(date +%H%M%S)"
|
||||
cp -p "$f" "$dest"
|
||||
printf '+ previous copy kept at %s\n' "$dest"
|
||||
}
|
||||
|
||||
cmd_write_keys() {
|
||||
local now want
|
||||
KEYS_CHANGED=0
|
||||
# TMDb: the running container owns this file and every setting in it. Only
|
||||
# the key element is touched, so country, language and adult settings
|
||||
# survive the edit.
|
||||
[ -f "$TMDB_XML" ] ||
|
||||
die "$TMDB_XML is missing — save the TMDb plugin settings once in the UI, then re-run"
|
||||
now="$(xml_value "$TMDB_XML" TmdbApiKey)"
|
||||
keys_len TMDB_API_KEY >/dev/null
|
||||
want="$(keys_value TMDB_API_KEY)"
|
||||
if [ "$now" = "$want" ]; then
|
||||
printf '= TmdbApiKey already set (%s chars)\n' "${#now}"
|
||||
else
|
||||
backup_to_trash "$TMDB_XML"
|
||||
xml_secret "$TMDB_XML" TmdbApiKey TMDB_API_KEY
|
||||
printf '+ set TmdbApiKey (%s chars)\n' "${#want}"
|
||||
KEYS_CHANGED=1
|
||||
fi
|
||||
|
||||
# TVDB: never overwrite a config the operator saved through the UI.
|
||||
if [ -f "$TVDB_XML" ]; then
|
||||
now="$(xml_value "$TVDB_XML" ApiKey)"
|
||||
if [ -n "$now" ]; then
|
||||
printf '= Jellyfin.Plugin.Tvdb.xml already has an ApiKey (kept)\n'
|
||||
else
|
||||
keys_len TVDB_API_KEY >/dev/null
|
||||
backup_to_trash "$TVDB_XML"
|
||||
xml_secret "$TVDB_XML" ApiKey TVDB_API_KEY
|
||||
if [ -n "$(keys_value TVDB_PIN)" ] && [ -z "$(xml_value "$TVDB_XML" Pin)" ]; then
|
||||
xml_secret "$TVDB_XML" Pin TVDB_PIN
|
||||
fi
|
||||
printf '+ filled the ApiKey in Jellyfin.Plugin.Tvdb.xml\n'
|
||||
KEYS_CHANGED=1
|
||||
fi
|
||||
else
|
||||
keys_len TVDB_API_KEY >/dev/null
|
||||
cat >"$TVDB_XML" <<XML
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
|
||||
<ApiKey />
|
||||
</Configuration>
|
||||
XML
|
||||
xml_secret "$TVDB_XML" ApiKey TVDB_API_KEY
|
||||
if [ -n "$(keys_value TVDB_PIN)" ]; then
|
||||
xml_secret "$TVDB_XML" Pin TVDB_PIN
|
||||
fi
|
||||
printf '+ created Jellyfin.Plugin.Tvdb.xml with ApiKey\n'
|
||||
KEYS_CHANGED=1
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_plan() {
|
||||
check_root
|
||||
check_keys_report
|
||||
printf 'root %s\n' "$JF_ROOT"
|
||||
plugin_state
|
||||
tmdb_state
|
||||
tvdb_state
|
||||
}
|
||||
|
||||
cmd_install() {
|
||||
check_root
|
||||
check_keys_report
|
||||
KEYS_CHANGED=0
|
||||
if [ ! -d "$PLUGINS/$PLUGIN_DIR" ]; then
|
||||
write_plugin
|
||||
KEYS_CHANGED=1
|
||||
else
|
||||
printf '= %s already installed\n' "$PLUGIN_DIR"
|
||||
fi
|
||||
cmd_write_keys
|
||||
if [ "$KEYS_CHANGED" -eq 1 ]; then
|
||||
# Nothing is restarted here on purpose: this host may be playing media
|
||||
# right now, and both the plugin and its keys are read at start-up.
|
||||
printf 'next: scripts/deploy.sh dankserver --service jellyfin (the plugin and keys load at start-up)\n'
|
||||
else
|
||||
printf '= no change\n'
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_verify() {
|
||||
check_root
|
||||
local bad=0
|
||||
if [ ! -d "$PLUGINS/$PLUGIN_DIR" ]; then
|
||||
printf 'MISSING plugin directory %s (thetvdb not installed)\n' "$PLUGIN_DIR" >&2
|
||||
bad=1
|
||||
else
|
||||
local dlls abi
|
||||
dlls="$(find "$PLUGINS/$PLUGIN_DIR" -maxdepth 1 -name '*.dll' | wc -l)"
|
||||
[ "$dlls" -ge 1 ] || {
|
||||
printf 'MISSING %s inside %s\n' "$TVDB_DLL" "$PLUGIN_DIR" >&2
|
||||
bad=1
|
||||
}
|
||||
abi="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("targetAbi","?"))' \
|
||||
"$PLUGINS/$PLUGIN_DIR/meta.json" 2>/dev/null || echo '?')"
|
||||
printf 'plugin %s: %s dll(s), targetAbi %s\n' "$PLUGIN_DIR" "$dlls" "$abi"
|
||||
fi
|
||||
local pair elem n
|
||||
for pair in "$TMDB_XML|TmdbApiKey" "$TVDB_XML|ApiKey"; do
|
||||
elem="${pair##*|}"
|
||||
if [ ! -f "${pair%%|*}" ]; then
|
||||
printf 'MISSING %s\n' "${pair%%|*}" >&2
|
||||
bad=1
|
||||
continue
|
||||
fi
|
||||
if ! python3 -c 'import sys, xml.dom.minidom; xml.dom.minidom.parse(sys.argv[1])' "${pair%%|*}"; then
|
||||
printf 'UNPARSEABLE %s\n' "${pair%%|*}" >&2
|
||||
bad=1
|
||||
continue
|
||||
fi
|
||||
n="$(xml_value "${pair%%|*}" "$elem")"
|
||||
if [ -z "$n" ]; then
|
||||
printf 'MISSING key: %s has no %s value\n' "$(basename "${pair%%|*}")" "$elem" >&2
|
||||
bad=1
|
||||
else
|
||||
printf '%s %s chars\n' "$elem" "${#n}"
|
||||
fi
|
||||
done
|
||||
[ "$bad" -eq 0 ] || return 1
|
||||
printf 'providers configured\n'
|
||||
}
|
||||
|
||||
cmd="${1:-}"
|
||||
case "$cmd" in
|
||||
plan) cmd_plan ;;
|
||||
install) cmd_install ;;
|
||||
verify) cmd_verify ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -32,6 +32,12 @@ cross a filesystem (rename cannot do it and a silent copy at 24 TB would not).
|
||||
--force relaxes only the guards that are opinion (journal already rolled back,
|
||||
paths outside --root); it never allows overwriting a file.
|
||||
|
||||
--metadata-view names the overlay the media servers read their library through
|
||||
(default /var/lib/media-meta/view). The sidecar follow-up printed after an apply
|
||||
or rollback is conditioned on that path being a mountpoint: where no view is
|
||||
mounted the servers bind the array itself, the sidecars moved with the media,
|
||||
and a reap order would point at a directory that is not there.
|
||||
|
||||
The journal defaults to `media-relayout.jsonl` BESIDE the library root, never
|
||||
inside it: a log inside the library is a file Jellyfin will index.
|
||||
"""
|
||||
@@ -114,6 +120,24 @@ def read_plan(path, root, force):
|
||||
return mkdirs, keeps, moves
|
||||
|
||||
|
||||
# The media servers may read the library through the SSD overlay view
|
||||
# (scripts/media-view.sh). A relayout moves bytes on the raw array, so sidecars
|
||||
# the servers wrote through that view stay at the old coordinates — naming the
|
||||
# follow-up here is the point of use for that fact, but only where the view
|
||||
# exists: where it is not mounted the servers bind the array itself, the
|
||||
# sidecars moved with the media, and a reap order points at a directory that
|
||||
# is not there. See the Media Metadata View page for the host's state.
|
||||
METADATA_VIEW = "/var/lib/media-meta/view"
|
||||
METADATA_HINT = ("note: sidecars written through the metadata view did not follow"
|
||||
" these moves — next: scripts/media-view.sh reap")
|
||||
|
||||
|
||||
def metadata_note(args):
|
||||
"""Print the sidecar follow-up only when the overlay is really mounted."""
|
||||
if os.path.ismount(args.metadata_view):
|
||||
print(METADATA_HINT)
|
||||
|
||||
|
||||
def do_apply(args, root, journal, mkdirs, keeps, moves):
|
||||
"""Preflight everything, then folders, then renames. Never a partial start."""
|
||||
problems = []
|
||||
@@ -195,6 +219,7 @@ def do_apply(args, root, journal, mkdirs, keeps, moves):
|
||||
f" {dirs_new} folders created, {len(keeps)} files stayed put")
|
||||
print(f"journal: {journal.path} — undo with: media-apply.py rollback"
|
||||
f" --journal {journal.path} --root {root} --apply")
|
||||
metadata_note(args)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -378,6 +403,7 @@ def do_rollback(args, root, journal):
|
||||
t=datetime.datetime.now(datetime.timezone.utc).isoformat())
|
||||
print(f"rolled back: {undone} files back where the census found them,"
|
||||
f" {dirs_gone} empty folders removed, {recreated} swept folders recreated")
|
||||
metadata_note(args)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -389,6 +415,9 @@ def main():
|
||||
ap.add_argument("--journal")
|
||||
ap.add_argument("--apply", action="store_true", help="actually change the library")
|
||||
ap.add_argument("--force", action="store_true", help="relax the opinion guards")
|
||||
ap.add_argument("--metadata-view", default=METADATA_VIEW,
|
||||
help="overlay the media servers read through; the sidecar"
|
||||
" note prints only when this path is a mountpoint")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = os.path.abspath(args.root)
|
||||
|
||||
Executable
+425
@@ -0,0 +1,425 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""media-audiobooks.py — a plan that re-files the audiobook library into one folder per book.
|
||||
|
||||
media-audiobooks.py --root DIR --enum TSV --rows TSV [--out TSV] [--exceptions FILE]
|
||||
|
||||
Plex reads this library and Jellyfin does not: its music scanner has no audiobook
|
||||
concept, and the tree is indexed entirely by Plex, which matched 58,975 of the
|
||||
62,879 files here into its Artist/Album/Track shape. So the authority is
|
||||
`scripts/media-census.py audiobooks` — the BOOK (album row), its NARRATOR (that
|
||||
book's single artist row) and its CHAPTERS (track rows) — and never the folder a
|
||||
release pack made. The plan is:
|
||||
|
||||
Audiobooks/<Narrator>/<Book> (<year>)/<the file's own name>.<ext>
|
||||
... the same folder, " (2)" on the name, when two chapters
|
||||
of one book carry the same name
|
||||
|
||||
What that buys is the one property both media servers' book scanners assume: a
|
||||
folder holds exactly one book. Today 212 top-level folders are a mix of
|
||||
author-named shelves and collection packs, and the books inside the packs are
|
||||
split across `… [Part 1]`, `… [Part 2]` folders three or four deep.
|
||||
|
||||
Three laws here are the opposite of what a music shelf does, and each is forced
|
||||
by what the rows actually hold:
|
||||
|
||||
* A chapter is NEVER renumbered. 7,556 of the 10,485 books contain at least one
|
||||
chapter whose index Plex does not know, so a number written into a name would
|
||||
be invented — and it would buy nothing, because both servers order playback by
|
||||
the embedded track tag, not by the file name. A wrong number in a name is worse
|
||||
than none: it is the number a human later trusts. A file already carrying its
|
||||
own number keeps it, untouched.
|
||||
* The shelf is the NARRATOR, not the author. Every book has exactly one artist row
|
||||
and it is the narrator; the author survives only inside 10% of the album titles
|
||||
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving the
|
||||
author would mean parsing titles for one book in ten and inventing it for the
|
||||
rest. One rule that always resolves beats a shelf that is half author and half
|
||||
narrator; the author stays visible in the book folder's own name, which is the
|
||||
title Plex shows.
|
||||
* Nothing is guessed at. A chapter row with no book or no narrator parks — a
|
||||
tag-scan placeholder (`[Unknown Album]`) names neither, since unrelated books
|
||||
share it and filing on one merges them, and so does a chapter number the ripper
|
||||
mistagged as the artist (`0.4`). An audio file Plex never matched parks, a
|
||||
video container parks, and a loose file in a folder holding two books parks.
|
||||
|
||||
Nothing is deleted and nothing is overwritten. The output is a list of rename(2)
|
||||
intents for `scripts/media-apply.py`, whose preflight refuses any target that is
|
||||
already claimed and whose journal is the rollback.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
# What a chapter file can be. .m4b is the audiobook container; the rest are the
|
||||
# audio a pack ships when it was ripped as music.
|
||||
AUDIO_EXT = {".mp3", ".m4a", ".m4b", ".ogg", ".oga", ".opus", ".wav", ".flac",
|
||||
".wma", ".aiff", ".aif", ".m4v", ".aac", ".mka"}
|
||||
# Never an audiobook, whatever the library it sits in says: Plex reads these as
|
||||
# video, and renaming one into an audio extension would claim it is a chapter.
|
||||
VIDEO_EXT = {".mkv", ".mp4", ".webm", ".avi", ".mov", ".mpg", ".mpeg", ".ts",
|
||||
".m2ts", ".vob", ".ifo", ".wmv", ".iso", ".img", ".bdmv", ".ogm",
|
||||
".divx"}
|
||||
|
||||
# Tag-scan placeholders. These strings are not identities: 1,106 chapters from
|
||||
# 14 unrelated top-level folders carry the album `[Unknown Album]`, so filing on
|
||||
# them would merge books that have nothing to do with each other — the exact
|
||||
# shape this relayout exists to make impossible. Such a chapter parks with the
|
||||
# path it has today and the fix belongs in its tags. `Various Artists` is NOT a
|
||||
# placeholder: that is what a multi-cast read carries, and the one book here
|
||||
# with 1,189 chapters under it is genuinely one book.
|
||||
PLACEHOLDER_BOOK = {"[unknown album]", "unknown album", "no title", "untitled",
|
||||
"[untitled]", "unknown title", "n/a", "-"}
|
||||
PLACEHOLDER_NARRATOR = {"[unknown artist]", "unknown artist", "unknown author",
|
||||
"<unknown>", "[unknown]", "unknown"}
|
||||
|
||||
|
||||
def identity(value, placeholders, needs_letter=False):
|
||||
"""The tag value Plex matched, when it names something; '' when it does not.
|
||||
|
||||
A trailing parenthesized group is stripped for the test only, because rippers
|
||||
glue timestamps to the placeholder (`Unknown album (03/03/2015 21:39:55)`).
|
||||
|
||||
`needs_letter` is for a NARRATOR alone: a shelf is a person's name, and 272
|
||||
chapters here carry a chapter number in the artist tag (`0.4`, `31`). The same
|
||||
test must never judge a BOOK title, which may honestly be `1984`.
|
||||
"""
|
||||
v = (value or "").strip()
|
||||
bare = re.sub(r"\s*\([^()]*\)\s*$", "", v).strip().casefold()
|
||||
if bare in placeholders or (needs_letter and not any(c.isalpha() for c in v)):
|
||||
return ""
|
||||
return v
|
||||
|
||||
|
||||
# The characters Plex documents as breaking playback. They are dropped, not
|
||||
# substituted: a replacement character is a guess at what the title meant.
|
||||
BAD_CHARS = set('<>:"/\\|?*')
|
||||
TRASH_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir"
|
||||
r"|lost\+found)(/|$)")
|
||||
|
||||
ARGS = None
|
||||
HELD = {} # path -> (bytes, ext)
|
||||
ROWS = {} # path -> the chapter row Plex matched
|
||||
PLACED = {} # chapter path -> the destination the plan gave it
|
||||
CLAIMED = set() # every destination path the plan has handed out
|
||||
MOVES, KEEPS = [], []
|
||||
PARKED = defaultdict(list) # reason -> [path]
|
||||
NOTES = defaultdict(list) # section -> [line]
|
||||
AUDIO_BY_DIR = defaultdict(set) # source dir -> chapter files a row names
|
||||
CHILD_OF = defaultdict(set) # dir -> the dirs directly below it holding chapters
|
||||
FOLDERS_SEEN = {} # (dir, depth) -> book folders placed below it
|
||||
SUBTREE_DEPTH = 2 # how far below a folder its chapters may sit
|
||||
BOOKS = {} # (narrator, book, year) -> the book group
|
||||
|
||||
AMBIGUOUS = object()
|
||||
|
||||
|
||||
def dominant(values):
|
||||
"""The spelling the rows mostly agree on: the most frequent value, ties broken
|
||||
by sort so the answer never depends on the order rows arrived in."""
|
||||
counts = Counter(values)
|
||||
return min(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0]
|
||||
|
||||
|
||||
def rel(path):
|
||||
"""The path as seen from the library root."""
|
||||
return os.path.relpath(path, ARGS.root)
|
||||
|
||||
|
||||
def safe(name):
|
||||
"""A name every client can carry: no reserved character, nothing unprintable,
|
||||
no trailing dot or space, and short enough to rename."""
|
||||
out = "".join(c for c in (name or "")
|
||||
if c not in BAD_CHARS and (c.isprintable() or c == " "))
|
||||
out = re.sub(r"\s+", " ", out).strip().rstrip(". ")
|
||||
while out and len(out.encode("utf-8", "ignore")) > 200:
|
||||
out = out[:-4].rstrip()
|
||||
return out
|
||||
|
||||
|
||||
def assign(src, folder, name, ext, kind, note):
|
||||
"""Book one file into `folder` under `name`, renamed if that exact destination
|
||||
is already taken — by another file in the plan, or by a file standing there
|
||||
today. Two files claiming one path is the fault that costs a re-run of a 60k
|
||||
plan, and so is a plan that only works if the renames happen in the right
|
||||
order: the executor refuses any target that is occupied, so an occupied path
|
||||
is never a destination, even when its occupant is scheduled to leave.
|
||||
A file already standing where the contract puts it is a keep, not a move."""
|
||||
stem = safe(name) or "file"
|
||||
dst = os.path.join(folder, stem + ext)
|
||||
if dst == src:
|
||||
CLAIMED.add(dst.lower()) # it is standing on its own target
|
||||
KEEPS.append((src, dst, kind))
|
||||
return dst
|
||||
# A file already in this folder under this name — or under the numbered
|
||||
# variant the collision rule gives it — keeps the name it has. Renumbering it
|
||||
# every run would turn the plan into a shuffle instead of a fixed point.
|
||||
base = os.path.splitext(os.path.basename(src))[0]
|
||||
if os.path.dirname(src) == folder and (base == stem
|
||||
or base.startswith(stem + " (")):
|
||||
CLAIMED.add(src.lower())
|
||||
KEEPS.append((src, src, kind))
|
||||
return src
|
||||
n = 1
|
||||
while dst.lower() in CLAIMED:
|
||||
n += 1
|
||||
dst = os.path.join(folder, f"{stem} ({n}){ext}")
|
||||
if n > 1:
|
||||
NOTES[note].append(rel(dst))
|
||||
CLAIMED.add(dst.lower())
|
||||
MOVES.append((src, dst, kind))
|
||||
return dst
|
||||
|
||||
|
||||
def park(path, reason):
|
||||
"""Hold a file where it is, under the park folder, keeping its relative path."""
|
||||
PARKED[reason].append(path)
|
||||
dst = os.path.join(ARGS.park_dir, rel(path))
|
||||
if dst != path:
|
||||
CLAIMED.add(dst.lower())
|
||||
MOVES.append((path, dst, "park"))
|
||||
|
||||
|
||||
def under_park(path):
|
||||
"""True for a file already parked: that pile is for a human to review, and
|
||||
re-parking it every run would only push it deeper."""
|
||||
return path.startswith(ARGS.park_dir + os.sep)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------ books
|
||||
def group_books():
|
||||
"""Every matched chapter into one group per (narrator, book, year).
|
||||
|
||||
The key is the identity Plex asserts in its own database, not the folder the
|
||||
file sits in: one book arrives as four part folders inside a collection pack,
|
||||
and two copies of one book in one pack are one book to Plex, whose album item
|
||||
IS that narrator, title and year. The year is part of the key so a 1965
|
||||
recording and a 2020 recording that share a title stay two books.
|
||||
"""
|
||||
for path, r in sorted(ROWS.items()):
|
||||
if path not in HELD or under_park(path) or TRASH_DIR.search(path):
|
||||
continue
|
||||
d = os.path.dirname(path)
|
||||
AUDIO_BY_DIR[d].add(path)
|
||||
CHILD_OF[os.path.dirname(d)].add(d) # so a cover above Part 1/Part 2 sees both
|
||||
narrator = identity(r["narrator"], PLACEHOLDER_NARRATOR, needs_letter=True)
|
||||
book = identity(r["book"], PLACEHOLDER_BOOK)
|
||||
if not book:
|
||||
park(path, "a chapter row that names no book — Plex's album title is a "
|
||||
"placeholder like [Unknown Album], so filing on it would "
|
||||
"merge unrelated books onto one shelf")
|
||||
continue
|
||||
if not narrator:
|
||||
park(path, "a chapter row that names no narrator — the artist row is a "
|
||||
"placeholder like [Unknown Artist] or a chapter number a "
|
||||
"ripper mistagged as the artist, so no shelf owns it")
|
||||
continue
|
||||
key = (narrator.casefold(), book.casefold(), (r["year"] or "").strip())
|
||||
g = BOOKS.setdefault(key, {"narrators": [], "books": [], "year": key[2],
|
||||
"members": [], "indexed": 0})
|
||||
g["members"].append(path)
|
||||
g["narrators"].append(narrator)
|
||||
g["books"].append(book)
|
||||
if str(r["track"] or "").isdigit():
|
||||
g["indexed"] += 1
|
||||
|
||||
|
||||
def book_folder(g):
|
||||
"""The shelf Plex's own rows name: <narrator>/<book> (<year>). None when a
|
||||
name is made of characters no path can hold — inventing a shelf is filing it
|
||||
wrong, so those chapters park instead."""
|
||||
narrator = safe(dominant(g["narrators"]))
|
||||
book = safe(dominant(g["books"]))
|
||||
if not narrator or not book:
|
||||
return None
|
||||
return os.path.join(ARGS.root, narrator,
|
||||
f"{book} ({g['year']})" if g["year"] else book)
|
||||
|
||||
|
||||
def place_books():
|
||||
"""One folder per book; every chapter keeps the name it has today."""
|
||||
for key in sorted(BOOKS):
|
||||
g = BOOKS[key]
|
||||
folder = book_folder(g)
|
||||
if folder is None:
|
||||
for path in sorted(g["members"]):
|
||||
park(path, "a book or narrator Plex spells with characters no "
|
||||
"folder name can hold")
|
||||
continue
|
||||
for path in sorted(g["members"]):
|
||||
ext = HELD[path][1]
|
||||
base = os.path.splitext(os.path.basename(path))[0]
|
||||
# The name the ripper wrote, unchanged: the track tag inside the file
|
||||
# is what orders playback, and 72% of the books here have chapters no
|
||||
# row numbers, so any number added here would be invented.
|
||||
PLACED[path] = assign(path, folder, base, ext, "chapter",
|
||||
"two chapters of one book carried the same name")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- sidecars
|
||||
def folders_below(d, depth):
|
||||
"""The book folders the chapters in `d`, or within `depth` directories below
|
||||
it, were placed in. Memoised, because every loose file in a big collection
|
||||
folder asks the same question about it."""
|
||||
if (d, depth) not in FOLDERS_SEEN:
|
||||
seen = {os.path.dirname(PLACED[a]) for a in AUDIO_BY_DIR.get(d, ())
|
||||
if a in PLACED}
|
||||
if depth:
|
||||
for child in CHILD_OF.get(d, ()):
|
||||
seen |= folders_below(child, depth - 1)
|
||||
FOLDERS_SEEN[(d, depth)] = seen
|
||||
return FOLDERS_SEEN[(d, depth)]
|
||||
|
||||
|
||||
def owner_folder(d):
|
||||
"""The book folder that owns this directory — the first place up the tree
|
||||
where chapters sit, in the directory itself or within SUBTREE_DEPTH levels
|
||||
below it, which is how a pack whose parts are `… [Part 1]`/`… [Part 2]` owns
|
||||
the cover art lying above them. None when no chapter is near enough to say,
|
||||
AMBIGUOUS when the chapters there belong to more than one book."""
|
||||
while d.startswith(ARGS.root + os.sep) and d != ARGS.root:
|
||||
folders = folders_below(d, SUBTREE_DEPTH)
|
||||
if folders:
|
||||
return next(iter(folders)) if len(folders) == 1 else AMBIGUOUS
|
||||
d = os.path.dirname(d)
|
||||
return None
|
||||
|
||||
|
||||
def place_other():
|
||||
"""Everything that is not a chapter Plex matched: covers, booklets, EPUBs,
|
||||
logs, checksums, playlists — and the clutter that is none of those."""
|
||||
for path, (_size, ext) in sorted(HELD.items()):
|
||||
if path in ROWS or under_park(path) or TRASH_DIR.search(path):
|
||||
continue
|
||||
d = os.path.dirname(path)
|
||||
base = os.path.splitext(os.path.basename(path))[0]
|
||||
if ext in VIDEO_EXT:
|
||||
park(path, "a video container inside the audiobook library: Plex reads "
|
||||
"it as video, and renaming it would claim it is a chapter")
|
||||
elif ext in AUDIO_EXT:
|
||||
park(path, "an audio file no chapter row names — it is not in the "
|
||||
"library Plex matched, so nothing says which book it is")
|
||||
else:
|
||||
owner = owner_folder(d)
|
||||
if owner is AMBIGUOUS:
|
||||
park(path, "a file whose folder holds more than one book, so "
|
||||
"nothing says which one it belongs to")
|
||||
elif owner:
|
||||
assign(path, owner, base, ext,
|
||||
"playlist" if ext in (".m3u", ".m3u8") else "sidecar",
|
||||
"one book folder would have carried two files of the same"
|
||||
" sidecar name")
|
||||
else:
|
||||
park(path, "not an audiobook file, and no book's folder lies with"
|
||||
" it to carry it")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- outputs
|
||||
def write_plan():
|
||||
"""mkdir rows for every destination folder and its whole chain, then the moves.
|
||||
Parents first: the executor creates directories in the order they are listed."""
|
||||
chain = set()
|
||||
for _s, dst, _k in MOVES:
|
||||
d = os.path.dirname(dst)
|
||||
while d and d != ARGS.root and d != os.path.dirname(d):
|
||||
chain.add(d)
|
||||
d = os.path.dirname(d)
|
||||
dirs = sorted(chain, key=lambda p: (p.count("/"), p))
|
||||
with open(ARGS.out, "w") as fh:
|
||||
fh.write("action\tsrc\tdst\tkind\n")
|
||||
for d in dirs:
|
||||
fh.write(f"mkdir\t\t{d}\tdir\n")
|
||||
for src, dst, kind in sorted(MOVES, key=lambda m: (m[1], m[0])):
|
||||
fh.write(f"move\t{src}\t{dst}\t{kind}\n")
|
||||
for src, dst, kind in sorted(KEEPS, key=lambda m: (m[1], m[0])):
|
||||
fh.write(f"keep\t{src}\t{dst}\t{kind}\n")
|
||||
parked = [m for m in MOVES if m[2] == "park"]
|
||||
sized = sum(HELD[s][0] for s, _d, _k in MOVES if s in HELD)
|
||||
print(f"plan: {len(MOVES) - len(parked)} moves, {len(parked)} parked"
|
||||
f" ({sized / 2**40:.2f} TiB touched), {len(KEEPS)} already placed,"
|
||||
f" {len(dirs)} folders -> {ARGS.out}")
|
||||
|
||||
|
||||
def write_exceptions():
|
||||
with open(ARGS.exceptions, "w") as fh:
|
||||
fh.write(f"Audiobook relayout exceptions for {ARGS.root}\n\n")
|
||||
fh.write(f"Everything below is parked under {ARGS.park_dir}, keeping the\n"
|
||||
"relative path it has today. Nothing is deleted and nothing is\n"
|
||||
"overwritten: a parked file is one rename from wherever the plan is\n"
|
||||
"pointed next.\n\n")
|
||||
for reason in sorted(PARKED, key=lambda r: -len(PARKED[r])):
|
||||
rs = PARKED[reason]
|
||||
fh.write(f"{len(rs):>6} files {reason}\n")
|
||||
for p in sorted(rs, key=rel)[:400]:
|
||||
fh.write(f" {rel(p)}\n")
|
||||
if len(rs) > 400:
|
||||
fh.write(f" … and {len(rs) - 400} more\n")
|
||||
fh.write("\n")
|
||||
for section in sorted(NOTES, key=lambda s: -len(NOTES[s])):
|
||||
lines = sorted(set(NOTES[section]))
|
||||
fh.write(f"{len(lines):>6} {section}\n")
|
||||
for line in lines[:200]:
|
||||
fh.write(f" {line}\n")
|
||||
if len(lines) > 200:
|
||||
fh.write(f" … and {len(lines) - 200} more\n")
|
||||
fh.write("\n")
|
||||
unindexed = sorted(f"{safe(dominant(g['narrators']))}/{safe(dominant(g['books']))}"
|
||||
for g in BOOKS.values()
|
||||
if 0 < g["indexed"] < len(g["members"]))
|
||||
if unindexed:
|
||||
fh.write(f"{len(unindexed):>6} books have chapters Plex does not number. Their\n")
|
||||
fh.write(" files keep the names they carry: playback order comes from the\n"
|
||||
" track tag inside each file, and a number written here would be\n"
|
||||
" invented for the chapters with no index. Fix it in the tags, not\n"
|
||||
" in the names — Plex re-reads tags on a scan and never names files.\n\n")
|
||||
fh.write(" Playlists move with their book and their contents are never\n"
|
||||
" rewritten: the paths inside an .m3u go stale.\n")
|
||||
print(f"exceptions -> {ARGS.exceptions}")
|
||||
|
||||
|
||||
def main():
|
||||
global ARGS
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--root", default=os.environ.get("MEDIA_AUDIOBOOKS_ROOT",
|
||||
"/mnt/entropy/Audiobooks"))
|
||||
ap.add_argument("--enum", default="/tmp/audiobooks-enum.tsv",
|
||||
help="the enumerate TSV (bytes, ext, path)")
|
||||
ap.add_argument("--rows", default="/tmp/plex-audiobooks.tsv",
|
||||
help="the chapter rows scripts/media-census.py audiobooks wrote")
|
||||
ap.add_argument("--out", default="/tmp/audiobooks-plan.tsv")
|
||||
ap.add_argument("--exceptions", default="/tmp/audiobooks-exceptions.txt")
|
||||
ap.add_argument("--park", default="_unsorted",
|
||||
help="the folder inside the root that holds what cannot be filed")
|
||||
ARGS = ap.parse_args()
|
||||
ARGS.root = os.path.normpath(ARGS.root)
|
||||
ARGS.park_dir = os.path.join(ARGS.root, ARGS.park)
|
||||
read_inputs()
|
||||
# Every path a file occupies today is claimed before a single destination is
|
||||
# handed out: the executor refuses an occupied target, so a plan that aimed at
|
||||
# one would only fail after the renames that were supposed to clear it.
|
||||
CLAIMED.update(p.lower() for p in HELD)
|
||||
group_books()
|
||||
place_books()
|
||||
place_other()
|
||||
write_plan()
|
||||
write_exceptions()
|
||||
|
||||
|
||||
def read_inputs():
|
||||
with open(ARGS.enum) as fh:
|
||||
for line in fh:
|
||||
parts = line.rstrip("\n").split("\t")
|
||||
if len(parts) >= 3:
|
||||
HELD[parts[2]] = (int(parts[0]), parts[1].lower())
|
||||
with open(ARGS.rows, newline="") as fh:
|
||||
for row in csv.DictReader(fh, delimiter="\t"):
|
||||
if row.get("file"):
|
||||
ROWS[row["file"]] = row
|
||||
gone = [p for p in ROWS if p not in HELD]
|
||||
if gone:
|
||||
NOTES["a chapter row whose file the census no longer sees"].extend(gone)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+336
-13
@@ -1,13 +1,15 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""Read-only census of a movie library — the inputs a relayout plan is built from.
|
||||
r"""Read-only census of a media library — the inputs a relayout plan is built from.
|
||||
|
||||
media-census.py enumerate [--root DIR] [--out TSV]
|
||||
media-census.py probe [--enum TSV] [--out TSV] [--jobs N]
|
||||
media-census.py plex [--root DIR] [--db PATH] [--out TSV]
|
||||
media-census.py jellyfin [--root DIR] [--db PATH] [--container PREFIX] [--out TSV]
|
||||
media-census.py audiobooks [--root DIR] [--db PATH] [--out TSV]
|
||||
|
||||
Run it on the machine that mounts the library. Nothing here writes inside the
|
||||
library and nothing is deleted; the only outputs are the three TSVs below, and
|
||||
scripts/media-plan.py reads them instead of touching media.
|
||||
library and nothing is deleted; the only outputs are the TSVs below, and the
|
||||
scripts/media-plan.py and scripts/media-shows.py read them instead of touching media.
|
||||
|
||||
enum bytes <TAB> ext <TAB> path every file under the root
|
||||
probed bytes <TAB> path <TAB> dur <TAB> vcodec <TAB> WxH <TAB> fps <TAB> vbit
|
||||
@@ -18,10 +20,34 @@ scripts/media-plan.py reads them instead of touching media.
|
||||
plex file <TAB> leaf_title <TAB> … (see PLEX_COLS)
|
||||
the titles Plex already matched — the authority over any filename guess
|
||||
|
||||
jellyfin file <TAB> series <TAB> year <TAB> season <TAB> episode <TAB> extra
|
||||
<TAB> tmdb <TAB> title <TAB> tvdb (see JF_COLS)
|
||||
one row per episode Jellyfin has matched and numbered, its path mapped
|
||||
from the container's view back to the host; `year`, `tmdb` and `tvdb`
|
||||
are the SERIES' values, `title` is the episode title the row carries,
|
||||
`extra` is Jellyfin's own extra type when the row is bonus material
|
||||
rather than an episode
|
||||
|
||||
music file <TAB> artist <TAB> album <TAB> year <TAB> disc <TAB> track <TAB> title
|
||||
<TAB> mb <TAB> performer (see MUSIC_COLS)
|
||||
one row per track Jellyfin has tagged: the tag read is what the row IS,
|
||||
so `artist` is the album artist (the track artist when that is empty) and
|
||||
`album`/`year` are the album item's provider values when the folder is an
|
||||
album item and the tags' otherwise; `mb` is the album's MusicBrainz
|
||||
release group, `performer` the track artist the tags carry
|
||||
|
||||
audiobooks file <TAB> narrator <TAB> book <TAB> year <TAB> track <TAB> chapter
|
||||
(see AB_COLS)
|
||||
one row per chapter Plex matched under the audiobook root: `book` is the
|
||||
album row Plex built from the files' own tags, `narrator` that book's
|
||||
artist row — Plex keeps exactly one artist per book, which is why it, and
|
||||
not an author parsed out of a title, names the shelf. `track` is empty
|
||||
wherever Plex has no chapter number, which is most of this library.
|
||||
|
||||
`probe` is the slow step (every container is opened for its header, ~10 min for
|
||||
4,800 files on a spindle array at 6 workers) so it is `nice`d: a census beside a
|
||||
live service must not starve it. `enumerate` is a stat pass, so re-censusing
|
||||
before a plan is cheap — and a plan is only ever as good as these three files.
|
||||
before a plan is cheap — and a plan is only ever as good as these files.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
@@ -57,6 +83,72 @@ join metadata_items mi on mi.id = m.metadata_item_id
|
||||
left join metadata_items par on par.id = mi.parent_id
|
||||
where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
||||
"""
|
||||
JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db"
|
||||
JF_COLS = ["file", "series", "year", "season", "episode", "extra",
|
||||
"tmdb", "title", "tvdb"]
|
||||
# Jellyfin stores the entity's .NET class as its type; the TV tree is these three.
|
||||
JF_TV = ("MediaBrowser.Controller.Entities.TV.Episode",
|
||||
"MediaBrowser.Controller.Entities.TV.Season",
|
||||
"MediaBrowser.Controller.Entities.TV.Series")
|
||||
JF_SQL = ("""select Id, Path, IndexNumber, ParentIndexNumber, ExtraType,
|
||||
IsVirtualItem, Name from BaseItems where Type = ? and Path like ?""")
|
||||
|
||||
# The music tree is these two entity classes. The track row carries the tags
|
||||
# Jellyfin read out of the file; the album row carries what the provider matched,
|
||||
# which is the better folder name of the two.
|
||||
JF_AUDIO = "MediaBrowser.Controller.Entities.Audio.Audio"
|
||||
JF_ALBUM = "MediaBrowser.Controller.Entities.Audio.MusicAlbum"
|
||||
MUSIC_COLS = ["file", "artist", "album", "year", "disc", "track", "title", "mb",
|
||||
"performer"]
|
||||
MUSIC_SQL = ("""select b.Path, b.AlbumArtists, b.Artists, b.Album, b.ProductionYear,
|
||||
b.ParentIndexNumber, b.IndexNumber, b.Name, b.Id,
|
||||
a.Name, a.ProductionYear, a.Id
|
||||
from BaseItems b
|
||||
left join BaseItems a on a.Id = b.ParentId and a.Type = ?
|
||||
where b.Type = ? and b.Path like ?""")
|
||||
# The album's identity, in preference order. A release group is the album; a bare
|
||||
# release id is one reissue of it.
|
||||
MUSIC_PROVIDERS = ("MusicBrainzReleaseGroup", "MusicBrainzAlbum")
|
||||
|
||||
# Plex's audiobook tree is the same Artist/Album/Track shape as its music tree:
|
||||
# the leaf (10) is a CHAPTER, its parent (9) is the BOOK, and that book's artist
|
||||
# (8) is the NARRATOR. The track's own `index` is exported because the exceptions
|
||||
# report counts how many books have chapters Plex does not number — the reason the
|
||||
# relayout refuses to invent numbers into file names.
|
||||
PLEX_ARTIST, PLEX_ALBUM, PLEX_TRACK = 8, 9, 10
|
||||
AB_COLS = ["file", "narrator", "book", "year", "track", "chapter"]
|
||||
AB_SQL = f"""
|
||||
select p.file, coalesce(ar.title,''), coalesce(al.title,''), coalesce(al.year,0),
|
||||
coalesce(mi."index",-1), coalesce(mi.title,'')
|
||||
from media_parts p
|
||||
join media_items m on m.id = p.media_item_id
|
||||
join metadata_items mi on mi.id = m.metadata_item_id
|
||||
join metadata_items al on al.id = mi.parent_id and al.metadata_type = {PLEX_ALBUM}
|
||||
left join metadata_items ar on ar.id = al.parent_id
|
||||
and ar.metadata_type = {PLEX_ARTIST}
|
||||
where p.file like ? and p.deleted_at is null and mi.deleted_at is null
|
||||
and al.deleted_at is null
|
||||
"""
|
||||
|
||||
def open_plex_ro(db):
|
||||
"""Plex's library, read the way Plex writes it.
|
||||
|
||||
`immutable=1` tells sqlite the file never changes, so it skips the
|
||||
write-ahead log: every row a scan has committed but not yet checkpointed
|
||||
is invisible, and reading a live database that way can fail outright as
|
||||
"database disk image is malformed". Plain ro reads the WAL through its
|
||||
-shm; immutable is only the fallback for a database whose -shm cannot be
|
||||
read. The same law scripts/media-verify.py states.
|
||||
"""
|
||||
last = None
|
||||
for opt in ("mode=ro", "mode=ro&immutable=1"):
|
||||
try:
|
||||
return sqlite3.connect(f"file:{db}?{opt}", uri=True)
|
||||
except sqlite3.Error as e:
|
||||
last = e
|
||||
sys.exit(f"cannot open the Plex database read-only ({db}): {last}\n"
|
||||
"Plex runs rootless here: run this as the container's owner, never "
|
||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||
|
||||
|
||||
def enumerate_lib(root, out):
|
||||
@@ -157,15 +249,9 @@ def probe(enum, out, jobs):
|
||||
|
||||
def plex(root, db, out):
|
||||
"""Plex's matched titles for files under root, from its DB, read-only."""
|
||||
uri = f"file:{db}?mode=ro&immutable=1"
|
||||
try:
|
||||
con = sqlite3.connect(uri, uri=True)
|
||||
except sqlite3.Error as e:
|
||||
sys.exit(f"cannot open the Plex database read-only ({db}): {e}\n"
|
||||
"Plex runs rootless here: run this as the container's owner, never "
|
||||
"with sudo — rootless storage is per-user, so root sees nothing.")
|
||||
con = open_plex_ro(db)
|
||||
with open(out, "w", newline="") as fh:
|
||||
w = csv.writer(fh, delimiter="\t")
|
||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||
w.writerow(PLEX_COLS)
|
||||
n = 0
|
||||
for r in con.execute(PLEX_SQL, (root + "/%",)):
|
||||
@@ -174,6 +260,210 @@ def plex(root, db, out):
|
||||
print(f"plex-matched={n} -> {out}")
|
||||
|
||||
|
||||
def jellyfin(root, db, container, out):
|
||||
"""The episodes Jellyfin has already numbered, for the library at root.
|
||||
|
||||
Jellyfin stores each path as the CONTAINER sees it (its quadlet binds the
|
||||
share at /data), so `container` is the in-container prefix that stands for
|
||||
root and every exported path is mapped back to the host before it is written
|
||||
— the planner reads host paths, because it plans moves on the array.
|
||||
|
||||
The numbering is what matters, not the name: a release group's file name says
|
||||
nothing about which episode a file is, and Jellyfin has already answered that
|
||||
question against a provider. Season/episode and the episode's own title come
|
||||
from the episode row; the series name, its year and its TMDb and Tvdb ids come
|
||||
from the Series row the episode climbs to, because those are the only place
|
||||
Jellyfin keeps them.
|
||||
"""
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||
except sqlite3.Error as e:
|
||||
sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n"
|
||||
"It is a WAL database: copy it with its -wal and -shm beside it and"
|
||||
" point --db at the copy. Never sudo it — the data dir belongs to"
|
||||
" the container's owner, and rootless storage is per-user.")
|
||||
have = {r[1] for r in con.execute("pragma table_info(BaseItems)")}
|
||||
need = {"Id", "ParentId", "Type", "Path", "IndexNumber", "ParentIndexNumber",
|
||||
"ExtraType", "IsVirtualItem", "ProductionYear", "Data"}
|
||||
if not need <= have:
|
||||
sys.exit(f"{db} is not a Jellyfin library database (BaseItems has no"
|
||||
f" {','.join(sorted(need - have))})")
|
||||
node = {}
|
||||
for i, pid, typ, name, year, data in con.execute(
|
||||
"select Id, ParentId, Type, Name, ProductionYear, Data from BaseItems"
|
||||
" where Type in (?,?,?)", JF_TV):
|
||||
node[i] = (pid, typ, name, year, data)
|
||||
|
||||
# Jellyfin 12 keeps provider ids in their own table; older builds kept them in
|
||||
# the row's Data blob. Both shapes are read, because the tvdb id the series
|
||||
# naming depends on is wherever this build happened to write it.
|
||||
prov = {}
|
||||
if "BaseItemProviders" in {r[0] for r in con.execute(
|
||||
"select name from sqlite_master where type='table'")}:
|
||||
for iid, pid, val in con.execute(
|
||||
"select ItemId, ProviderId, ProviderValue from BaseItemProviders"
|
||||
" where ProviderId in ('Tmdb', 'Tvdb')"):
|
||||
prov[(iid, pid)] = val
|
||||
|
||||
def provider_of(item_id, want):
|
||||
got = prov.get((item_id, want))
|
||||
if got:
|
||||
return str(got)
|
||||
try:
|
||||
blob = json.loads(node[item_id][4] or "{}").get("ProviderIds") or {}
|
||||
except (KeyError, ValueError, TypeError):
|
||||
return ""
|
||||
return str(blob.get(want, "") or "")
|
||||
|
||||
def series_of(item_id):
|
||||
"""Climb episode -> season -> series; empty when the chain is broken."""
|
||||
for _ in range(4):
|
||||
if item_id not in node:
|
||||
return "", "", "", ""
|
||||
pid, typ = node[item_id][0], node[item_id][1]
|
||||
if typ.endswith(".Series"):
|
||||
return (node[item_id][2] or "", node[item_id][3] or "",
|
||||
provider_of(item_id, "Tmdb"), provider_of(item_id, "Tvdb"))
|
||||
item_id = pid
|
||||
return "", "", "", ""
|
||||
|
||||
# The fleet's quadlets bind the share at /data, so the container's view of
|
||||
# this library is /data/<the library's own folder name> unless told otherwise.
|
||||
if not container:
|
||||
container = "/data/" + os.path.basename(os.path.normpath(root))
|
||||
prefix = container.rstrip("/") + "/"
|
||||
written = skipped = 0
|
||||
with open(out, "w", newline="") as fh:
|
||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||
w.writerow(JF_COLS)
|
||||
for iid, path, idx, pidx, extra, virt, epname in con.execute(
|
||||
JF_SQL, (JF_TV[0], prefix + "%")):
|
||||
if virt or not path or not path.startswith(prefix):
|
||||
skipped += 1 # a virtual item is a variant, not a file
|
||||
continue
|
||||
sname, syear, tmdb, tvdb = series_of(iid)
|
||||
w.writerow([root + "/" + path[len(prefix):],
|
||||
sname.replace("\t", " "), syear,
|
||||
"" if pidx is None else pidx,
|
||||
"" if idx is None else idx,
|
||||
(extra or "").replace("\t", " "), tmdb,
|
||||
(epname or "").replace("\t", " "), tvdb])
|
||||
written += 1
|
||||
print(f"episodes={written} virtual-or-blank={skipped} -> {out}")
|
||||
|
||||
|
||||
def music(root, db, container, out):
|
||||
"""The tracks Jellyfin has already tagged, for the library at root.
|
||||
|
||||
Jellyfin reads music metadata out of the file itself, so an Audio row IS the
|
||||
tag read, done once, by the vendor's own parser: its album-artist, album,
|
||||
year, disc and track columns are what the embedded tags say and its Name is
|
||||
the track title. That makes this export the whole authority for a music
|
||||
relayout — the path a ripper chose is not evidence of anything, and the
|
||||
folder Jellyfin put a track under is often a release pack, not an album.
|
||||
|
||||
Two gaps this export answers from the row rather than the tree. Many tracks
|
||||
sit under a folder that is not an album item at all (a disc folder, or a
|
||||
mixed one), so the album name and year fall back to the tags when the album
|
||||
row is absent. And an album's MusicBrainz ids are not in the row's own Data
|
||||
— the current schema leaves that empty for music — they are rows of
|
||||
BaseItemProviders keyed by the item's Id.
|
||||
"""
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||
except sqlite3.Error as e:
|
||||
sys.exit(f"cannot open the Jellyfin database read-only ({db}): {e}\n"
|
||||
"It is a WAL database: copy it with its -wal and -shm beside it and"
|
||||
" point --db at the copy. Never sudo it — the data dir belongs to"
|
||||
" the container's owner, and rootless storage is per-user.")
|
||||
have = {r[1] for r in con.execute("pragma table_info(BaseItems)")}
|
||||
need = {"Id", "ParentId", "Type", "Path", "Name", "IndexNumber",
|
||||
"ParentIndexNumber", "ProductionYear", "Album", "AlbumArtists",
|
||||
"Artists"}
|
||||
if not need <= have:
|
||||
sys.exit(f"{db} is not a Jellyfin library database with music columns"
|
||||
f" (BaseItems has no {','.join(sorted(need - have))})")
|
||||
# Provider ids, best identity first: the release group is the album, a bare
|
||||
# release id is one reissue of it and never overrides a group.
|
||||
mb = {}
|
||||
if "BaseItemProviders" in {r[0] for r in con.execute(
|
||||
"select name from sqlite_master where type='table'")}:
|
||||
rank = {p: i for i, p in enumerate(MUSIC_PROVIDERS)}
|
||||
best = {}
|
||||
for iid, pid, val in con.execute(
|
||||
"select ItemId, ProviderId, ProviderValue from BaseItemProviders"
|
||||
" where ProviderId in (?, ?)", MUSIC_PROVIDERS):
|
||||
if iid not in best or rank[pid] < best[iid][0]:
|
||||
best[iid] = (rank[pid], val)
|
||||
mb = {k: v[1] for k, v in best.items()}
|
||||
|
||||
# The fleet's quadlets bind the share at /data, so the container's view of
|
||||
# this library is /data/<the library's own folder name> unless told otherwise.
|
||||
if not container:
|
||||
container = "/data/" + os.path.basename(os.path.normpath(root))
|
||||
prefix = container.rstrip("/") + "/"
|
||||
written = skipped = 0
|
||||
with open(out, "w", newline="") as fh:
|
||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||
w.writerow(MUSIC_COLS)
|
||||
for (path, aal, arts, tag_album, tag_year, disc, track, name, iid,
|
||||
alb_name, alb_year, alb_id) in con.execute(
|
||||
MUSIC_SQL, (JF_ALBUM, JF_AUDIO, prefix + "%")):
|
||||
if not path or not path.startswith(prefix):
|
||||
skipped += 1 # a row for a file outside this library
|
||||
continue
|
||||
# The album artist owns the folder; the track artist is the fallback
|
||||
# for the compilations whose album-artist tag was left empty.
|
||||
w.writerow([root + "/" + path[len(prefix):],
|
||||
((aal or "").strip() or (arts or "").strip()),
|
||||
(alb_name or tag_album or "").replace("\t", " "),
|
||||
alb_year or tag_year or "",
|
||||
"" if disc is None else disc,
|
||||
"" if track is None else track,
|
||||
(name or "").replace("\t", " "),
|
||||
mb.get(alb_id or "") or mb.get(iid or "") or "",
|
||||
(arts or "").replace("\t", " ")])
|
||||
written += 1
|
||||
print(f"tracks={written} outside-root-or-blank={skipped} -> {out}")
|
||||
|
||||
|
||||
def audiobooks(root, db, out):
|
||||
"""The chapters Plex matched inside the audiobook library, read-only.
|
||||
|
||||
Plex stores HOST paths here (unlike Jellyfin, which stores the container's
|
||||
view), so no prefix mapping is needed: the part's `file` is the path the
|
||||
operator has today. Rows for files outside `root` are skipped, because one
|
||||
Plex library DB carries every library on the box.
|
||||
|
||||
The chapter's `index` is exported as an empty field when Plex has none. That
|
||||
is not a gap to fill in here: three quarters of the books on this box contain
|
||||
at least one un-indexed chapter, so the relayout that reads this file must
|
||||
not renumber anything from it.
|
||||
"""
|
||||
con = open_plex_ro(db)
|
||||
tables = {r[0] for r in con.execute(
|
||||
"select name from sqlite_master where type='table'")}
|
||||
missing = {"media_parts", "media_items", "metadata_items"} - tables
|
||||
if missing:
|
||||
sys.exit(f"{db} is not a Plex library database (no "
|
||||
f"{', '.join(sorted(missing))})")
|
||||
written = skipped = 0
|
||||
with open(out, "w", newline="") as fh:
|
||||
w = csv.writer(fh, delimiter="\t", lineterminator="\n")
|
||||
w.writerow(AB_COLS)
|
||||
for file, narrator, book, year, track, chapter in con.execute(
|
||||
AB_SQL, (root.rstrip("/") + "/%",)):
|
||||
if not file or not file.startswith(root.rstrip("/") + "/"):
|
||||
skipped += 1 # a row for a file outside this library
|
||||
continue
|
||||
w.writerow([file.replace("\t", " "), narrator.replace("\t", " "),
|
||||
book.replace("\t", " "), "" if not year else year,
|
||||
"" if track is None or int(track) < 0 else track,
|
||||
chapter.replace("\t", " ")])
|
||||
written += 1
|
||||
print(f"chapters={written} outside-root={skipped} -> {out}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
@@ -190,13 +480,46 @@ def main():
|
||||
x.add_argument("--root", **root)
|
||||
x.add_argument("--db", default=PLEX_DB)
|
||||
x.add_argument("--out", default="/tmp/plex-movies.tsv")
|
||||
j = sub.add_parser("jellyfin",
|
||||
help="export the episodes Jellyfin has numbered for the root")
|
||||
j.add_argument("--root", default=os.environ.get("MEDIA_ROOT",
|
||||
"/mnt/entropy/TVShows"),
|
||||
help="TV library root on the host")
|
||||
j.add_argument("--db", default=JF_DB)
|
||||
j.add_argument("--container", default=None,
|
||||
help="the in-container prefix that stands for --root"
|
||||
" (default /data/<root's folder name>)")
|
||||
j.add_argument("--out", default="/tmp/jellyfin-tv.tsv")
|
||||
m = sub.add_parser("music",
|
||||
help="export the tracks Jellyfin has tagged for the root")
|
||||
m.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT",
|
||||
"/mnt/entropy/Music"),
|
||||
help="music library root on the host")
|
||||
m.add_argument("--db", default=JF_DB)
|
||||
m.add_argument("--container", default=None,
|
||||
help="the in-container prefix that stands for --root"
|
||||
" (default /data/<root's folder name>)")
|
||||
m.add_argument("--out", default="/tmp/jellyfin-music.tsv")
|
||||
b = sub.add_parser("audiobooks",
|
||||
help="export the chapters Plex matched for the audiobook root")
|
||||
b.add_argument("--root", default=os.environ.get("MEDIA_AUDIOBOOKS_ROOT",
|
||||
"/mnt/entropy/Audiobooks"),
|
||||
help="audiobook library root on the host")
|
||||
b.add_argument("--db", default=PLEX_DB)
|
||||
b.add_argument("--out", default="/tmp/plex-audiobooks.tsv")
|
||||
a = ap.parse_args()
|
||||
if a.cmd == "enumerate":
|
||||
enumerate_lib(a.root, a.out)
|
||||
elif a.cmd == "probe":
|
||||
probe(a.enum, a.out, a.jobs)
|
||||
else:
|
||||
elif a.cmd == "plex":
|
||||
plex(a.root, a.db, a.out)
|
||||
elif a.cmd == "music":
|
||||
music(a.root, a.db, a.container, a.out)
|
||||
elif a.cmd == "audiobooks":
|
||||
audiobooks(a.root, a.db, a.out)
|
||||
else:
|
||||
jellyfin(a.root, a.db, a.container, a.out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Executable
+445
@@ -0,0 +1,445 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""media-music.py — a plan that re-files the music library into Jellyfin's own shape.
|
||||
|
||||
media-music.py --root DIR --enum TSV --rows TSV [--out TSV] [--exceptions FILE]
|
||||
|
||||
Jellyfin's music contract (https://jellyfin.org/docs/general/server/media/music/)
|
||||
is not the movie one. Jellyfin reads the metadata out of the file, so a folder is
|
||||
an ALBUM and nothing else, every disc of an album belongs in that one folder, the
|
||||
tags decide the track order, and a file name only matters when it carries a
|
||||
character known to break playback (`< > : " / \ | ? *`) or when a lyric file has
|
||||
to match the basename of the track it belongs to. The plan is therefore:
|
||||
|
||||
Music/<album artist>/<Album> (<year>)/NN - <Title>.<ext>
|
||||
... the same folder, D.NN - <Title> when the album has discs
|
||||
Music/Various Artists/... when its tracks name
|
||||
more than one artist
|
||||
|
||||
The authority is the tag row Jellyfin already read — `scripts/media-census.py
|
||||
music` — never the release-pack folder a ripper made. A file with no row is not
|
||||
guessed at: it parks under `<root>/_unsorted` with the relative path it has today.
|
||||
|
||||
The rules below exist because of what a downloaded discography pack actually
|
||||
contains:
|
||||
|
||||
* A cuesheet beside exactly one audio file means the album is one image with a
|
||||
track list inside it. That file keeps its own name: numbering it `01` would
|
||||
claim it is one track, and Jellyfin shows one track either way.
|
||||
* Sidecars — images, cuesheets, EAC logs, checksums, booklets, playlists —
|
||||
follow the album the audio in their own folder belongs to. A folder holding
|
||||
two albums gives a sidecar no single owner, so it parks instead of being
|
||||
guessed into one.
|
||||
* A video container (.mkv, .vob, .ifo, …) inside a music library is not music.
|
||||
Renaming it to .mka/.m4a would tell Jellyfin to index video as audio, so it
|
||||
parks for a human decision.
|
||||
* Playlists move with their album and their CONTENTS are never rewritten: the
|
||||
paths inside an .m3u go stale. Editing a file the operator owns to keep a
|
||||
convenience file tidy is not this tool's job.
|
||||
|
||||
Nothing is deleted and nothing is overwritten. The output is a list of rename(2)
|
||||
intents for `scripts/media-apply.py`, whose preflight refuses any target that is
|
||||
already claimed and whose journal is the rollback.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
# What Jellyfin will index as music. .mp4/.mkv/.webm are deliberately NOT here:
|
||||
# the vendor doc says an audio-only stream in those containers is never read as
|
||||
# music, and this tool renames for a reason it can prove, not to guess at one.
|
||||
AUDIO_EXT = {".flac", ".mp3", ".m4a", ".m4b", ".ogg", ".oga", ".opus", ".wav",
|
||||
".wma", ".aiff", ".aif", ".ape", ".dsf", ".dff", ".wv", ".mka",
|
||||
".mpc", ".aac"}
|
||||
VIDEO_EXT = {".mkv", ".vob", ".ifo", ".bup", ".mpg", ".mpeg", ".avi", ".mov",
|
||||
".ts", ".m2ts", ".wmv", ".webm", ".mp4", ".m4v", ".iso", ".img",
|
||||
".bdmv", ".divx", ".ogm"}
|
||||
# A lyric has to carry the basename of its track, so it is named after the audio
|
||||
# file and not after itself. .txt counts only when its stem matches an audio stem:
|
||||
# a loose .txt in an album folder is a note.
|
||||
LYRIC_EXT = {".lrc", ".elrc", ".txt"}
|
||||
IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tiff"}
|
||||
# The album-artist strings that mean "compilation", so such an album is not filed
|
||||
# under one artist who does not own it.
|
||||
VARIOUS = {"various", "various artists", "unknown", "unknown artist", "unknow"}
|
||||
# The characters Jellyfin documents as breaking playback. They are dropped, not
|
||||
# substituted: a replacement character is a guess at what the title meant.
|
||||
BAD_CHARS = set('<>:"/\\|?*')
|
||||
TRASH_DIR = re.compile(r"(^|/)(\.AGENTTRASH|\.thumbnails|\.Trash-\d+|@eaDir"
|
||||
r"|lost\+found)(/|$)")
|
||||
|
||||
ARGS = None
|
||||
HELD = {} # path -> (bytes, ext)
|
||||
ROWS = {} # path -> the tag row
|
||||
PLACED = {} # audio path -> the destination the plan gave it
|
||||
CLAIMED = set() # every destination path the plan has handed out
|
||||
MOVES, KEEPS = [], []
|
||||
PARKED = defaultdict(list) # reason -> [path]
|
||||
NOTES = defaultdict(list) # section -> [line]
|
||||
CUE_DIRS = set() # source dirs holding a .cue
|
||||
AUDIO_BY_DIR = defaultdict(set) # source dir -> audio files a row names
|
||||
CHILD_OF = defaultdict(set) # dir -> the dirs directly below it holding audio
|
||||
FOLDERS_SEEN = {} # (dir, depth) -> album folders placed below it
|
||||
SUBTREE_DEPTH = 2 # how far below a folder its audio may sit
|
||||
ALBUMS = {} # (artist, album, year) -> the album group
|
||||
|
||||
AMBIGUOUS = object()
|
||||
|
||||
|
||||
def dominant(values):
|
||||
"""The spelling the tags mostly agree on: the most frequent value, ties broken
|
||||
by sort so the answer never depends on the order rows arrived in."""
|
||||
counts = Counter(values)
|
||||
return min(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0]
|
||||
|
||||
|
||||
def rel(path):
|
||||
"""The path as seen from the library root."""
|
||||
return os.path.relpath(path, ARGS.root)
|
||||
|
||||
|
||||
def safe(name):
|
||||
"""A name every client can carry: no reserved character, nothing
|
||||
unprintable, no trailing dot or space, and short enough to rename."""
|
||||
out = "".join(c for c in (name or "")
|
||||
if c not in BAD_CHARS and (c.isprintable() or c == " "))
|
||||
out = re.sub(r"\s+", " ", out).strip().rstrip(". ")
|
||||
while out and len(out.encode("utf-8", "ignore")) > 200:
|
||||
out = out[:-4].rstrip()
|
||||
return out
|
||||
|
||||
|
||||
def assign(src, folder, name, ext, kind, note):
|
||||
"""Book one file into `folder` under `name`, renamed if that exact destination
|
||||
is already taken — by another file in the plan, or by a file standing there
|
||||
today. Two files claiming one path is the fault that costs a re-run of a 90k
|
||||
plan, and so is a plan that only works if the renames happen in the right
|
||||
order: the executor refuses any target that is occupied, so an occupied path
|
||||
is never a destination, even when its occupant is scheduled to leave.
|
||||
A file already standing where the contract puts it is a keep, not a move."""
|
||||
stem = safe(name) or "file"
|
||||
dst = os.path.join(folder, stem + ext)
|
||||
if dst == src:
|
||||
CLAIMED.add(dst.lower()) # it is standing on its own target
|
||||
KEEPS.append((src, dst, kind))
|
||||
return dst
|
||||
# A file already in this folder under this name — or under the numbered
|
||||
# variant the collision rule gives it — keeps the name it has. Renumbering it
|
||||
# every run would turn the plan into a shuffle instead of a fixed point.
|
||||
base = os.path.splitext(os.path.basename(src))[0]
|
||||
if os.path.dirname(src) == folder and (base == stem
|
||||
or base.startswith(stem + " (")):
|
||||
CLAIMED.add(src.lower())
|
||||
KEEPS.append((src, src, kind))
|
||||
return src
|
||||
n = 1
|
||||
while dst.lower() in CLAIMED:
|
||||
n += 1
|
||||
dst = os.path.join(folder, f"{stem} ({n}){ext}")
|
||||
if n > 1:
|
||||
NOTES[note].append(rel(dst))
|
||||
CLAIMED.add(dst.lower())
|
||||
MOVES.append((src, dst, kind))
|
||||
return dst
|
||||
|
||||
|
||||
def park(path, reason):
|
||||
"""Hold a file where it is, under the park folder, keeping its relative path."""
|
||||
PARKED[reason].append(path)
|
||||
dst = os.path.join(ARGS.park_dir, rel(path))
|
||||
if dst != path:
|
||||
CLAIMED.add(dst.lower())
|
||||
MOVES.append((path, dst, "park"))
|
||||
|
||||
|
||||
def under_park(path):
|
||||
"""True for a file already parked: that pile is for a human to review, and
|
||||
re-parking it every run would only push it deeper."""
|
||||
return path.startswith(ARGS.park_dir + os.sep)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- albums
|
||||
def group_albums():
|
||||
"""Every tagged track into one group per (album-artist tag, album, year).
|
||||
|
||||
The key is the identity the tags assert, not the folder the file sits in: one
|
||||
album arrives as three discs in three folders, and two copies of an album in
|
||||
one pack are one album to Jellyfin, whose album item is that name, artist and
|
||||
year. The artist in the key is the ALBUM-artist tag and never the performer,
|
||||
because a compilation is one album whose tracks name many performers — keying
|
||||
on the performer would file one album once per artist.
|
||||
"""
|
||||
for path, r in sorted(ROWS.items()):
|
||||
if path not in HELD or under_park(path):
|
||||
continue
|
||||
d = os.path.dirname(path)
|
||||
AUDIO_BY_DIR[d].add(path)
|
||||
CHILD_OF[os.path.dirname(d)].add(d) # so a cover above CD1/CD2 sees both
|
||||
tag = (r["artist"] or "").strip()
|
||||
album = (r["album"] or "").strip()
|
||||
performer = (r["performer"] or "").strip()
|
||||
if not album:
|
||||
park(path, "a track no album names")
|
||||
continue
|
||||
if not tag and not performer:
|
||||
park(path, "a track no artist names")
|
||||
continue
|
||||
key = (tag.casefold(), album.casefold(), (r["year"] or "").strip())
|
||||
g = ALBUMS.setdefault(key, {"tags": [], "albums": [], "year": key[2],
|
||||
"members": [], "performers": set(),
|
||||
"mb": set()})
|
||||
g["members"].append(path)
|
||||
g["tags"].append(tag)
|
||||
g["albums"].append(album)
|
||||
g["performers"].add(performer or tag)
|
||||
if r.get("mb"):
|
||||
g["mb"].add(r["mb"])
|
||||
|
||||
|
||||
def folder_artist(g):
|
||||
"""Whose shelf the album sits on. The album-artist tag owns it when it names
|
||||
one; when the tags name only performers, a single performer names the album
|
||||
and several make it a compilation, which belongs to no one of them. The key is
|
||||
case-folded, so the spelling that reaches the folder is the one the tracks
|
||||
mostly carry — see dominant()."""
|
||||
tags = [t for t in g["tags"] if t]
|
||||
if tags:
|
||||
name = dominant(tags)
|
||||
return "Various Artists" if name.casefold() in VARIOUS else name
|
||||
if len({p.casefold() for p in g["performers"]}) == 1:
|
||||
return dominant(g["performers"])
|
||||
return "Various Artists"
|
||||
|
||||
|
||||
def album_folder(g):
|
||||
name = safe(dominant(g["albums"])) or "Untitled"
|
||||
return os.path.join(ARGS.root, safe(folder_artist(g)) or "Unknown Artist",
|
||||
f"{name} ({g['year']})" if g["year"] else name)
|
||||
|
||||
|
||||
def is_cue_image(path):
|
||||
"""A folder holding this one audio file and a cuesheet is an album image."""
|
||||
d = os.path.dirname(path)
|
||||
return len(AUDIO_BY_DIR.get(d, ())) == 1 and d in CUE_DIRS
|
||||
|
||||
|
||||
def number_of(value, width):
|
||||
return f"{int(value):0{width}d}" if str(value or "").isdigit() else ""
|
||||
|
||||
|
||||
def place_albums():
|
||||
for key in sorted(ALBUMS):
|
||||
g = ALBUMS[key]
|
||||
folder = album_folder(g)
|
||||
discs = {g0["disc"] for g0 in (ROWS[p] for p in g["members"]) if g0["disc"]}
|
||||
tracks = [int(ROWS[p]["track"]) for p in g["members"]
|
||||
if str(ROWS[p]["track"] or "").isdigit()]
|
||||
# More than one disc, or a disc numbered past the first: the track number
|
||||
# alone cannot name a file uniquely, so the disc goes in front of it.
|
||||
multi = len(discs) > 1 or any(str(d).isdigit() and int(d) > 1 for d in discs)
|
||||
width = 3 if multi or (max(tracks, default=0) > 99) else 2
|
||||
for path in sorted(g["members"]):
|
||||
r = ROWS[path]
|
||||
ext = HELD[path][1]
|
||||
base = os.path.splitext(os.path.basename(path))[0]
|
||||
if is_cue_image(path):
|
||||
name = safe(base) or "image"
|
||||
else:
|
||||
title = safe(r["title"] or base) or "untitled"
|
||||
num = number_of(r["track"], 2 if multi else width)
|
||||
if multi and num:
|
||||
num = f"{r['disc'] or '1'}.{num}"
|
||||
name = f"{num} - {title}" if num else title
|
||||
PLACED[path] = assign(path, folder, name, ext, "track",
|
||||
"two tracks claimed one name inside one album")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- sidecars
|
||||
def folders_below(d, depth):
|
||||
"""The album folders the audio in `d`, or within `depth` directories below it,
|
||||
was placed in. Memoised, because every loose file in a big discography folder
|
||||
asks the same question about it."""
|
||||
if (d, depth) not in FOLDERS_SEEN:
|
||||
seen = {os.path.dirname(PLACED[a]) for a in AUDIO_BY_DIR.get(d, ())
|
||||
if a in PLACED}
|
||||
if depth:
|
||||
for child in CHILD_OF.get(d, ()):
|
||||
seen |= folders_below(child, depth - 1)
|
||||
FOLDERS_SEEN[(d, depth)] = seen
|
||||
return FOLDERS_SEEN[(d, depth)]
|
||||
|
||||
|
||||
def owner_folder(d):
|
||||
"""The album folder that owns this directory — the first place up the tree
|
||||
where audio sits, in the directory itself or within SUBTREE_DEPTH levels below
|
||||
it, which is how a box with its discs in CD1/CD2 owns the cover lying above
|
||||
them. None when no audio is near enough to say, AMBIGUOUS when the audio there
|
||||
belongs to more than one album."""
|
||||
while d.startswith(ARGS.root + os.sep) and d != ARGS.root:
|
||||
folders = folders_below(d, SUBTREE_DEPTH)
|
||||
if folders:
|
||||
return next(iter(folders)) if len(folders) == 1 else AMBIGUOUS
|
||||
d = os.path.dirname(d)
|
||||
return None
|
||||
|
||||
|
||||
def place_sidecars():
|
||||
"""Everything that is not a tagged track: lyrics, images, cuesheets, logs,
|
||||
booklets, playlists — and the clutter that is none of those."""
|
||||
# The lyric rule needs the audio's stem as it is named TODAY, before the plan
|
||||
# renames it, because that is what the sidecar's own name was copied from.
|
||||
stem = {(d, os.path.splitext(os.path.basename(a))[0]): a
|
||||
for d, audios in AUDIO_BY_DIR.items() for a in audios}
|
||||
for path, (_size, ext) in sorted(HELD.items()):
|
||||
if path in ROWS or under_park(path) or TRASH_DIR.search(path):
|
||||
continue
|
||||
d, base = os.path.dirname(path), os.path.splitext(os.path.basename(path))[0]
|
||||
if ext in VIDEO_EXT:
|
||||
park(path, "a video container inside the music library: Jellyfin will "
|
||||
"not read it as music, and renaming it would claim it is")
|
||||
elif ext in AUDIO_EXT:
|
||||
park(path, "an audio file no provider row names — its tags are not in "
|
||||
"the library Jellyfin has matched")
|
||||
elif ext in (LYRIC_EXT | IMAGE_EXT) and (d, base) in stem:
|
||||
# A lyric, or a picture named after its track: it follows that track
|
||||
# and takes the name the track now has, or Jellyfin cannot find it.
|
||||
audio = stem[(d, base)]
|
||||
dst = PLACED.get(audio)
|
||||
if dst:
|
||||
folder, name = os.path.dirname(dst), os.path.splitext(
|
||||
os.path.basename(dst))[0]
|
||||
assign(path, folder, name, ext, "lyric",
|
||||
"a sidecar whose name collided with another in its album")
|
||||
else:
|
||||
park(path, "a sidecar of a track the plan could not place")
|
||||
else:
|
||||
owner = owner_folder(d)
|
||||
if owner is AMBIGUOUS:
|
||||
park(path, "a sidecar whose folder holds more than one album, so "
|
||||
"nothing says which one it belongs to")
|
||||
elif owner:
|
||||
assign(path, owner, base, ext,
|
||||
"playlist" if ext in (".m3u", ".m3u8") else "sidecar",
|
||||
"one album folder would have carried two files of the same "
|
||||
"sidecar name")
|
||||
else:
|
||||
park(path, "not music, or music with nothing beside it to belong to")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- outputs
|
||||
def write_plan():
|
||||
"""mkdir rows for every destination folder and its whole chain, then the moves.
|
||||
Parents first: the executor creates directories in the order they are listed."""
|
||||
chain = set()
|
||||
for _s, dst, _k in MOVES:
|
||||
d = os.path.dirname(dst)
|
||||
while d and d != ARGS.root and d != os.path.dirname(d):
|
||||
chain.add(d)
|
||||
d = os.path.dirname(d)
|
||||
dirs = sorted(chain, key=lambda p: (p.count("/"), p))
|
||||
with open(ARGS.out, "w") as fh:
|
||||
fh.write("action\tsrc\tdst\tkind\n")
|
||||
for d in dirs:
|
||||
fh.write(f"mkdir\t\t{d}\tdir\n")
|
||||
for src, dst, kind in sorted(MOVES, key=lambda m: (m[1], m[0])):
|
||||
fh.write(f"move\t{src}\t{dst}\t{kind}\n")
|
||||
for src, dst, kind in sorted(KEEPS, key=lambda m: (m[1], m[0])):
|
||||
fh.write(f"keep\t{src}\t{dst}\t{kind}\n")
|
||||
parked = [m for m in MOVES if m[2] == "park"]
|
||||
sized = sum(HELD[s][0] for s, _d, _k in MOVES if s in HELD)
|
||||
print(f"plan: {len(MOVES) - len(parked)} moves, {len(parked)} parked"
|
||||
f" ({sized / 2**40:.2f} TiB touched), {len(KEEPS)} already placed,"
|
||||
f" {len(dirs)} folders -> {ARGS.out}")
|
||||
|
||||
|
||||
def write_exceptions():
|
||||
with open(ARGS.exceptions, "w") as fh:
|
||||
fh.write(f"Music relayout exceptions for {ARGS.root}\n\n")
|
||||
fh.write(f"Everything below is parked under {ARGS.park_dir}, keeping the\n"
|
||||
"relative path it has today. Nothing is deleted and nothing is\n"
|
||||
"overwritten: a parked file is one rename from wherever the plan is\n"
|
||||
"pointed next.\n\n")
|
||||
for reason in sorted(PARKED, key=lambda r: -len(PARKED[r])):
|
||||
rs = PARKED[reason]
|
||||
fh.write(f"{len(rs):>6} files {reason}\n")
|
||||
for p in sorted(rs, key=rel)[:400]:
|
||||
fh.write(f" {rel(p)}\n")
|
||||
if len(rs) > 400:
|
||||
fh.write(f" … and {len(rs) - 400} more\n")
|
||||
fh.write("\n")
|
||||
for section in sorted(NOTES, key=lambda s: -len(NOTES[s])):
|
||||
lines = sorted(set(NOTES[section]))
|
||||
fh.write(f"{len(lines):>6} {section}\n")
|
||||
for line in lines[:200]:
|
||||
fh.write(f" {line}\n")
|
||||
if len(lines) > 200:
|
||||
fh.write(f" … and {len(lines) - 200} more\n")
|
||||
fh.write("\n")
|
||||
no_year = sorted(f"{folder_artist(g)}/{dominant(g['albums'])}"
|
||||
for g in ALBUMS.values() if not g["year"])
|
||||
if no_year:
|
||||
fh.write(f"{len(no_year):>6} albums carry no year in any tag, so their folders\n"
|
||||
" are named without one; Jellyfin still matches them.\n")
|
||||
fh.write(" " + ", ".join(no_year[:300]) + "\n\n")
|
||||
clash = sorted(f"{folder_artist(g)}/{dominant(g['albums'])}"
|
||||
for g in ALBUMS.values() if len(g["mb"]) > 1)
|
||||
if clash:
|
||||
fh.write(f"{len(clash):>6} albums put one folder name on more than one\n"
|
||||
" MusicBrainz release group — different releases, one name.\n"
|
||||
" They share the folder Jellyfin's own album item names.\n")
|
||||
fh.write(" " + ", ".join(clash[:200]) + "\n\n")
|
||||
fh.write(" Playlists move with their album and their contents are never\n"
|
||||
" rewritten: the paths inside an .m3u go stale.\n")
|
||||
print(f"exceptions -> {ARGS.exceptions}")
|
||||
|
||||
|
||||
def main():
|
||||
global ARGS
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--root", default=os.environ.get("MEDIA_MUSIC_ROOT",
|
||||
"/mnt/entropy/Music"))
|
||||
ap.add_argument("--enum", default="/tmp/music-enum.tsv",
|
||||
help="the enumerate TSV (bytes, ext, path)")
|
||||
ap.add_argument("--rows", default="/tmp/jellyfin-music.tsv",
|
||||
help="the tag rows scripts/media-census.py music wrote")
|
||||
ap.add_argument("--out", default="/tmp/music-plan.tsv")
|
||||
ap.add_argument("--exceptions", default="/tmp/music-exceptions.txt")
|
||||
ap.add_argument("--park", default="_unsorted",
|
||||
help="the folder inside the root that holds what cannot be filed")
|
||||
ARGS = ap.parse_args()
|
||||
ARGS.root = os.path.normpath(ARGS.root)
|
||||
ARGS.park_dir = os.path.join(ARGS.root, ARGS.park)
|
||||
read_inputs()
|
||||
# Every path a file occupies today is claimed before a single destination is
|
||||
# handed out: the executor refuses an occupied target, so a plan that aimed at
|
||||
# one would only fail after the renames that were supposed to clear it.
|
||||
CLAIMED.update(p.lower() for p in HELD)
|
||||
for path, (_size, ext) in HELD.items():
|
||||
if ext == ".cue":
|
||||
CUE_DIRS.add(os.path.dirname(path))
|
||||
group_albums()
|
||||
place_albums()
|
||||
place_sidecars()
|
||||
write_plan()
|
||||
write_exceptions()
|
||||
|
||||
|
||||
def read_inputs():
|
||||
with open(ARGS.enum) as fh:
|
||||
for line in fh:
|
||||
parts = line.rstrip("\n").split("\t")
|
||||
if len(parts) >= 3:
|
||||
HELD[parts[2]] = (int(parts[0]), parts[1].lower())
|
||||
with open(ARGS.rows, newline="") as fh:
|
||||
for row in csv.DictReader(fh, delimiter="\t"):
|
||||
if row.get("file"):
|
||||
ROWS[row["file"]] = row
|
||||
gone = [p for p in ROWS if p not in HELD]
|
||||
if gone:
|
||||
NOTES["a tag row whose file the census no longer sees"].extend(gone)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,35 @@
|
||||
# The operator's tvdb id map: `folder <TAB> tvdb id <TAB> note`.
|
||||
#
|
||||
# These are the pack folders whose show tvdb's NAME SEARCH cannot reach: the
|
||||
# record exists, but tvdb titles it in the show's own language and the English
|
||||
# name rides in aliases the search never reads. Without a row here the folder
|
||||
# keeps the release folder's words and no episode title; with one, both matchers
|
||||
# read `[tvdbid-NNNN]` and the files rename to tvdb's own name and titles.
|
||||
#
|
||||
# Every id was read from tvdb's own record (`/series/<id>/extended`) on
|
||||
# 2026-10-31, and the note states what that record says plus the neighbouring
|
||||
# record a name search would have picked instead — the row is checkable without
|
||||
# re-searching, because one wrong id renames a whole pack to the wrong show. The
|
||||
# plan refuses a line whose id is not a number and lists every folder the map
|
||||
# touched in the exceptions file.
|
||||
#
|
||||
# Folders the name search DOES reach (the EXACT and PUNCT classes of the proposal
|
||||
# table) stay off this file on purpose: a map row would hide a live lookup behind
|
||||
# a number nobody re-checks.
|
||||
[Anime Time] Pokemon Complete Series (Season 01-25+Movies+Specials+Shorts+OST) [DVD+BD][1080p][HEVC 10bit x265][AAC] 76703 tvdb: ポケットモンスター (1997), the umbrella record for every era this pack spans; 1268 files
|
||||
[Anime Time] One Piece (0001-1071+Movies+Specials) [BD+CR] [Dual Audio] [1080p][HEVC 10bit x265][AAC][Multi Sub] 81797 tvdb: ワンピース (1999), alias ONE PIECE; the bare alias "One Piece" belongs to 392276, the 2023 live action; 1072 files
|
||||
[Judas] Naruto (Complete Series) [BD 1080p][HEVC x265 10bit][Multi-Audio][Multi-Subs] 78857 tvdb: NARUTO-ナルト- (2002), alias "Naruto" carries the full-width dashes; 220 files
|
||||
Star Trek Deep Space 9 Deluxe DVD Boxset + Extras DVDRip TSV 72073 tvdb: Star Trek: Deep Space Nine (1993), alias "Star Trek Deep Space 9"; the box set's own words match nothing; 279 files
|
||||
Top Gear UK 1-17 74608 tvdb: Top Gear (2002), no "UK" alias — the US version is 199051; 131 files
|
||||
Black Clover 001 331753 tvdb: ブラッククローバー (2017), which holds NO alias at all; 378762 is Mugyutto! Black Clover (2019), a different show; 108 files
|
||||
Shin Chan - Complete Eng Dub Series 2160p Ai Upscaled -Mesc 437356 tvdb: Shin Chan (2006) — the English dub record this pack is; 79654 is the 1992 Japanese original; 78 files
|
||||
Fairy Tail (2009-2014) (Seasons 1-8 + OVAs) (BD 1080p)(HEVC x265 10bit)(Dual-Audio)(Eng-Subs)-Judas[TGx] 114801 tvdb: フェアリーテイル (2009), alias FAIRY TAIL; 410031 is FAIRY TAIL 100 NEN QUEST (2024), the sequel; 59 files
|
||||
(CBB) Tokyo Ghoul - The Complete Series (BD 1080p)(Dual-Audio)(HEVC-10bit-x265) 281249 tvdb: 東京喰種トーキョーグール (2014) — :re and √A are seasons of THIS record, confirmed by search; 58 files
|
||||
[Anime Time] Demon Slayer - Kimetsu no Yaiba (S01+S02+S03+S04+Mugen Train Arc+Movie) [BD] [Dual Audio][1080p][HEVC 10bit x265][AAC][Eng Sub] 348545 tvdb: 鬼滅の刃 (2019), alias "Demon Slayer: Kimetsu no Yaiba" — the folder's hyphen splits it differently; 56 files
|
||||
(CBB) Tengen Toppa Gurren Lagann - The Complete Series (BD 1080p)(Dual-Audio)(HEVC-x265-10bit) 80096 tvdb: 天元突破グレンラガン (2007), alias Tengen Toppa Gurren Lagann; 51 files
|
||||
Fullmetal Alchemist - Brotherhood 85249 tvdb: 鋼の錬金術師 FULLMETAL ALCHEMIST (2009), alias "Full Metal Alchemist Brotherhood" — spaced, unlike this folder; 64 files
|
||||
[NewbSubs] Attack on Titan~Shingeki no Kyojin Series (1080p Blu-ray x265 Dual Audio) 267440 tvdb: 進撃の巨人 (2013), aliases carry "Attack on Titan" and "Shingeki no Kyojin", joined here by a tilde; 27 files
|
||||
Kimetsu no Yaiba 348545 the same record as the Anime Time pack above: two release folders, ONE show; 12 files
|
||||
Rick and Morty The Anime 420279 tvdb: リック・アンド・モーティ:ジ・アニメ (2024), no English alias on the record; 10 files
|
||||
[Anime Time] Dorohedoro (Season 1+OVA+Extra) [Dual Audio][BD][1080p][HEVC 10bit x265][AAC][Eng Sub] 370761 tvdb: ドロヘドロ (2020), alias Dorohedoro; 9 files
|
||||
Ken Il Guerriero HOKUTO NO KEN 79156 tvdb: 北斗の拳 (1984), alias "Hokuto no Ken"; the Italian title matches nothing and 465216 is the 2026 remake; 5 files
|
||||
|
Can't render this file because it contains an unexpected character in line 20 and column 182.
|
Executable
+306
@@ -0,0 +1,306 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""Ask tvdb what it calls the shows a plan renamed WITHOUT a tvdb id.
|
||||
|
||||
media-shows-tvdb.py [--plan TSV] [--root DIR] [--park DIR] [--map TSV]
|
||||
[--out TSV] [--credentials FILE]
|
||||
|
||||
`media-shows.py` renames a series to tvdb's own words only when an id backs the
|
||||
name; everything else keeps the metadata provider's words and no episode title,
|
||||
and the plan's mkdir rows are where those folders show up. This script turns that
|
||||
list into a decision sheet: one row per destination series folder that carries no
|
||||
`[tvdbid-NNNN]`, with what tvdb's search answers for it and a verdict that says
|
||||
HOW MUCH was guessed to get there.
|
||||
|
||||
It changes nothing. The sheet is for the operator; the file that changes
|
||||
behaviour is `scripts/media-shows-tvdb-ids.tsv`, whose rows he approves. That is
|
||||
the loop: plan -> this sheet -> approved map rows -> re-plan.
|
||||
|
||||
Verdicts, weakest last:
|
||||
|
||||
EXACT tvdb's own name equals the folder's words as they stand.
|
||||
PUNCT it equals them only after release junk is stripped (pack numbers,
|
||||
codecs, group names, `Complete Series`); the difference is spelling,
|
||||
not identity.
|
||||
NEAR tvdb answered something whose name is NOT the query. The best hit is
|
||||
printed so a human can judge it, and it is never applied by anything.
|
||||
NONE nothing left to ask with, or no answer at all.
|
||||
|
||||
Acceptance is deliberately unforgiving — a hit counts only when tvdb's name
|
||||
equals the query with every non-alphanumeric dropped from both sides, and the
|
||||
series' year still has to fall inside any year span the folder names. The
|
||||
cleaner is the only thing that got smarter over the passes this tool replaces;
|
||||
the acceptance test never moved, because a wrong id renames a whole pack to the
|
||||
wrong show. A query that needs the cleaner is reported as PUNCT for exactly that
|
||||
reason: the machine stripped words, so a human reads the row.
|
||||
|
||||
Read-only against tvdb. The credential file is the planner's own
|
||||
(`api_key=`, optional `pin=`), read here and never printed.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
ARGS = argparse.ArgumentParser(description=__doc__.splitlines()[0],
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ARGS.add_argument("--plan", default="/tmp/tv-plan.tsv",
|
||||
help="the plan media-shows.py just wrote (its mkdir rows are "
|
||||
"the folders this sheet asks about)")
|
||||
ARGS.add_argument("--root", default=os.environ.get("TV_ROOT", "/mnt/entropy/TVShows"),
|
||||
help="the TV library the plan renames")
|
||||
ARGS.add_argument("--park", default="",
|
||||
help="the review pile inside it (its folders are not shows)")
|
||||
ARGS.add_argument("--map",
|
||||
default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"media-shows-tvdb-ids.tsv"),
|
||||
help="the operator's approved map; its folders are reported as "
|
||||
"MAPPED and never asked again")
|
||||
ARGS.add_argument("--credentials",
|
||||
default=os.path.expanduser("~/.config/secrets/tvdb"),
|
||||
help="file with an api_key= line (and a pin= for a user-issued "
|
||||
"key); mode 600, and never printed")
|
||||
ARGS.add_argument("--out", default="/tmp/tvdb-proposal.tsv",
|
||||
help="the decision sheet")
|
||||
ARGS = ARGS.parse_args()
|
||||
|
||||
ROOT = os.path.abspath(ARGS.root)
|
||||
PARK = os.path.abspath(ARGS.park or os.path.join(ROOT, "_unsorted"))
|
||||
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"eng", os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"media-titles.py"))
|
||||
eng = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(eng)
|
||||
|
||||
# Release metadata, not the show: everything a scene folder name says about the
|
||||
# ENCODING, the packaging or the languages, which tvdb has never heard of.
|
||||
JUNK = re.compile(
|
||||
r"complete|series|dual|multi|audio|sub|subs|multisub|dualaudio|"
|
||||
r"\d{3,4}p|1080p|720p|2160p|480p|4k|hd|hdtv|cam|hdcam|telesync|x264|x265|hevc|avc|"
|
||||
r"\d+bit|bit|bluray|blu-?ray|bdrip|web-?rip|web-?dl|webrip|hdtvrip|"
|
||||
r"dvdrip|dvdr|dvdscr|tsr|tsv|mkv|avi|mp4|aac|ac3|dts|eac3|flac|ddp?5[.1]1|"
|
||||
r"bd|dvd|rip|boxset|box|season|seasons|episodes?|specials?|shorts|ost|"
|
||||
r"ova|ovas|oad|repack|proper|internal|mi?xed|budgetbits|deluxe|remux|"
|
||||
r"remastered|uncut|collector|edition|extras?|volume|volumes|part|"
|
||||
r"disc|disk|cd|ntsc|pal|region|fixed|patched|proof|english|esp|"
|
||||
r"espagnol|french|vostfr|subtitul\w*|spanish|german|italian|ita|eng|fre|spa|lat|"
|
||||
r"upscaled?|ai|dd|movie|movies",
|
||||
re.I)
|
||||
GROUP = re.compile(r"KONTRAST|TGx|Judas|Silence|FreetheFilms|NTb|DEADPOOL|Xploit|"
|
||||
r"SM737|TSA|3OISL3R|PLAYWEB|EPiC|FLUX|HONEY|Zaxton|Cca3rW|"
|
||||
r"monikeres|frank|hdh|psi|rtfpemaker|cbm|web|asser|astro|"
|
||||
r"yfn|uindex|www|org|com|net|mesc|ataraxiaprime|seph\d*|"
|
||||
r"[a-z]+subs?|edge", re.I)
|
||||
BRACKET = re.compile(r"[\[({]([^\])}]*)[\])}]")
|
||||
NUM_RANGE = re.compile(r"(?<!\w)\d{1,4}\s*(?:-|–|to|through)\s*\d{1,4}(?!\w)", re.I)
|
||||
INSIDE = re.compile(r"all|complete|full|dual|multi|audio|sub|subs|ova|ovas|"
|
||||
r"movies?|episodes?|extras?|bonus|pack|remux|region\s*\d*|"
|
||||
r"bd|dvd|cd|ntsc|pal|hr-?dvd", re.I)
|
||||
|
||||
|
||||
def drop_bracket(m):
|
||||
"""A bracket says nothing when its words are junk; keep it when they aren't."""
|
||||
toks = [t for t in re.split(r"[\s_.,+~&]+", m.group(1)) if t]
|
||||
if not toks or all(INSIDE.fullmatch(t) or JUNK.fullmatch(t)
|
||||
or GROUP.fullmatch(t)
|
||||
or re.fullmatch(r"\d+[a-z]?", t, re.I)
|
||||
or re.fullmatch(r"(19|20)\d\d", t) for t in toks):
|
||||
return " "
|
||||
return " " + m.group(1) + " "
|
||||
|
||||
|
||||
def clean(name):
|
||||
"""The folder's words with everything about the RELEASE taken out, so what is
|
||||
left is what a person would type into tvdb. `Star Trek Deep Space 9 Deluxe
|
||||
DVD Boxset + Extras DVDRip TSV` becomes `Star Trek Deep Space 9`; `SNL 16-20`
|
||||
becomes `SNL`."""
|
||||
name = re.sub(r"^\s*www\b.*?\b(org|com|net)\b\s*[-–]\s*", " ", name, flags=re.I)
|
||||
name = re.sub(r"\bS\d{1,2}([ .Ee]\d+)?[-–Ee\d.]*", " ", name)
|
||||
name = re.sub(r"\b\d{1,2}[xX]\d{1,3}\b", " ", name)
|
||||
name = NUM_RANGE.sub(" ", name)
|
||||
name = re.sub(r"(19|20)\d\d(?:\s*[-/–]\s*(?:19|20)\d\d)?", " ", name)
|
||||
name = re.sub(r"\bby\s+[A-Za-z0-9_.-]+$", " ", name)
|
||||
name = BRACKET.sub(drop_bracket, name)
|
||||
name = re.sub(r"[\[()\]:,;!?*]", " ", name)
|
||||
toks = re.split(r"[\s_.\-+~&]+", name)
|
||||
out = []
|
||||
for i, t in enumerate(toks):
|
||||
if not t or JUNK.fullmatch(t) or GROUP.fullmatch(t):
|
||||
continue
|
||||
if re.fullmatch(r"\d+", t):
|
||||
after = sum(1 for x in toks[i + 1:] if x
|
||||
and not JUNK.fullmatch(x) and not GROUP.fullmatch(x)
|
||||
and not re.fullmatch(r"\d+", x))
|
||||
# A pack index (`001`) and a lead-in number over a real title
|
||||
# (`01 Charles Barkley`) are noise; `30 Rock`, `24`, `Deep Space 9`
|
||||
# are the title itself.
|
||||
if len(t) >= 3 or (not out and after >= 2):
|
||||
continue
|
||||
out.append(t)
|
||||
while out and out[-1].lower() in ("and", "with", "the", "a", "an", "to", "of"):
|
||||
out.pop()
|
||||
return " ".join(out).strip()
|
||||
|
||||
|
||||
def years(name):
|
||||
return [int(y) for y in re.findall(r"\b(19\d\d|20\d\d)\b", name)]
|
||||
|
||||
|
||||
def bare(s):
|
||||
return re.sub(r"[^a-z0-9]", "", s.lower())
|
||||
|
||||
|
||||
CRED = {}
|
||||
try:
|
||||
for _line in open(ARGS.credentials):
|
||||
if "=" in _line:
|
||||
k, v = _line.strip().split("=", 1)
|
||||
CRED[k.strip()] = v.strip()
|
||||
except OSError:
|
||||
print(f"tvdb: no credentials at {ARGS.credentials} — the sheet would be all "
|
||||
"NEAR and NONE, which is not a decision sheet", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if not CRED.get("api_key"):
|
||||
print(f"tvdb: no api_key in {ARGS.credentials}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
TOKEN = {"jwt": ""}
|
||||
|
||||
|
||||
def call(path):
|
||||
"""One tvdb v4 GET, logging in again once when the JWT expires mid-run."""
|
||||
for attempt in (1, 2):
|
||||
if not TOKEN["jwt"]:
|
||||
body = {"apiKey": CRED["api_key"]}
|
||||
if CRED.get("pin"):
|
||||
body["token"] = CRED["pin"]
|
||||
req = urllib.request.Request("https://api4.thetvdb.com/v4/login",
|
||||
data=json.dumps(body).encode(),
|
||||
headers={"Content-Type":
|
||||
"application/json"})
|
||||
TOKEN["jwt"] = json.load(
|
||||
urllib.request.urlopen(req, timeout=30))["data"]["token"]
|
||||
try:
|
||||
req = urllib.request.Request("https://api4.thetvdb.com/v4" + path,
|
||||
headers={"Authorization":
|
||||
"Bearer " + TOKEN["jwt"]})
|
||||
return json.load(urllib.request.urlopen(req, timeout=30))["data"]
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 401 and attempt == 1:
|
||||
TOKEN["jwt"] = ""
|
||||
continue
|
||||
raise
|
||||
|
||||
|
||||
MAPPED = {}
|
||||
if ARGS.map and os.path.exists(ARGS.map):
|
||||
for _line in open(ARGS.map):
|
||||
_line = _line.rstrip("\n")
|
||||
if not _line.strip() or _line.startswith("#"):
|
||||
continue
|
||||
cols = _line.split("\t")
|
||||
if len(cols) > 1 and cols[1].strip().isdigit():
|
||||
MAPPED[cols[0].strip()] = cols[1].strip()
|
||||
|
||||
# The folders the plan created inside the library, with the file count each one
|
||||
# will hold. The review pile is not a show: its files belong to no identity, and
|
||||
# naming the pile would put one junk series on the sheet for all of them.
|
||||
shows = collections.Counter()
|
||||
for line in open(ARGS.plan).read().splitlines()[1:]:
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 4 or parts[0] != "mkdir":
|
||||
continue
|
||||
dst = os.path.abspath(parts[2])
|
||||
if not dst.startswith(ROOT + os.sep) or dst.startswith(PARK + os.sep):
|
||||
continue
|
||||
top = os.path.relpath(dst, ROOT).split(os.sep)[0]
|
||||
if top.startswith((".", "_")):
|
||||
continue
|
||||
if "[tvdbid-" in top:
|
||||
continue # an id already backs it
|
||||
shows.setdefault(top, 0)
|
||||
for line in open(ARGS.plan).read().splitlines()[1:]:
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 4 or parts[0] != "move":
|
||||
continue
|
||||
dst = os.path.abspath(parts[2])
|
||||
if not dst.startswith(ROOT + os.sep) or dst.startswith(PARK + os.sep):
|
||||
continue
|
||||
top = os.path.relpath(dst, ROOT).split(os.sep)[0]
|
||||
if "[tvdbid-" in top:
|
||||
continue
|
||||
shows[top] += 1
|
||||
|
||||
tally = collections.Counter()
|
||||
rows = []
|
||||
for folder in sorted(shows, key=lambda f: (-shows[f], f)):
|
||||
files, tag, note = shows[folder], "NONE", ""
|
||||
if folder in MAPPED:
|
||||
rows.append([files, "MAPPED", folder, "", "", "", MAPPED[folder],
|
||||
"the operator's map already states this id"])
|
||||
tally["MAPPED"] += 1
|
||||
continue
|
||||
query, untouched = clean(folder), clean(folder) == folder
|
||||
ys = years(folder)
|
||||
if not query:
|
||||
rows.append([files, "NONE", folder, "", "", "", "",
|
||||
"nothing is left of the folder's name once release junk"
|
||||
" is gone"])
|
||||
else:
|
||||
try:
|
||||
hits = call("/search?query=" + urllib.parse.quote(query)
|
||||
+ "&type=series") or []
|
||||
except Exception as e: # noqa: BLE001
|
||||
hits = []
|
||||
note = f"tvdb did not answer: {e}"
|
||||
want = bare(query)
|
||||
pick = None
|
||||
for h in hits:
|
||||
if bare(h.get("name") or "") != want:
|
||||
continue
|
||||
hy = str(h.get("year") or "")
|
||||
if ys and hy.isdigit() and not (min(ys) <= int(hy) <= max(ys)):
|
||||
continue
|
||||
pick = h
|
||||
break
|
||||
if pick:
|
||||
tag = "EXACT" if untouched else "PUNCT"
|
||||
rows.append([files, tag, folder, query, pick.get("name", ""),
|
||||
str(pick.get("year", "") or ""),
|
||||
str(pick.get("tvdb_id", "") or ""), ""])
|
||||
elif hits:
|
||||
best = hits[0]
|
||||
rows.append([files, "NEAR", folder, query, best.get("name", ""),
|
||||
str(best.get("year", "") or ""),
|
||||
str(best.get("tvdb_id", "") or ""),
|
||||
f"{len(hits)} answer(s); tvdb's own name is not the "
|
||||
"query, so nothing would take it"])
|
||||
else:
|
||||
rows.append([files, "NONE", folder, query, "", "", "",
|
||||
note or "tvdb has no series by that name"])
|
||||
tally[rows[-1][1]] += 1
|
||||
time.sleep(0.12)
|
||||
|
||||
# Two folders mapping to one record is not two shows; it is one show the release
|
||||
# folders split. Say it on every row involved, or the operator approves the same
|
||||
# id twice and reads the second as a mistake.
|
||||
by_id = collections.Counter(r[6] for r in rows if r[6] and r[1] != "MAPPED")
|
||||
for r in rows:
|
||||
if r[6] and r[1] != "MAPPED" and by_id[r[6]] > 1:
|
||||
r[7] = (f"MERGE: {by_id[r[6]]} folders map to tvdb {r[6]} — one show, "
|
||||
"the releases split it; " + r[7]).strip("; ")
|
||||
|
||||
with open(ARGS.out, "w") as fh:
|
||||
fh.write("files\tverdict\tfolder\tquery\ttvdb_name\ttvdb_year\ttvdb_id\tnote\n")
|
||||
for r in rows:
|
||||
fh.write("\t".join(str(c) for c in r) + "\n")
|
||||
print(f"sheet: {len(rows)} folders"
|
||||
+ "".join(f" {t}={n}" for t, n in sorted(tally.items()))
|
||||
+ f" -> {ARGS.out}")
|
||||
Executable
+1258
File diff suppressed because it is too large.
Load diff
@@ -275,7 +275,14 @@ def is_extra(path, title="", dur=0.0, root=ROOT, matched=False):
|
||||
bonus_folder = False
|
||||
for folder in parts[head:-1]:
|
||||
if folder_is_bonus(folder):
|
||||
bonus_folder = not mentions(name_words(name), name_words(folder))
|
||||
# The folder's words are often the film's own name ("Angel's Egg
|
||||
# Extras/"), and then they are not bonus evidence. Only WORDS can name
|
||||
# a film: "extras" is release junk that name_words drops, so a folder
|
||||
# like "01. Extras" reduces to its index — and an index matching an
|
||||
# index ("NCOP 01") is a coincidence, not a title.
|
||||
nw = [w for w in name_words(name) if not w.isdigit()]
|
||||
fw = [w for w in name_words(folder) if not w.isdigit()]
|
||||
bonus_folder = not fw or not mentions(nw, fw)
|
||||
break
|
||||
if dur and dur > SHORT_MAX:
|
||||
# Runtime vetoes vocabulary, with ONE exception: an explicit bonus folder
|
||||
|
||||
Executable
+221
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove a relayout landed, using the media server's own tables.
|
||||
|
||||
A scan answers `204` whether or not it found anything, and a rename that the
|
||||
scanner has not followed looks exactly like a rename that never happened from
|
||||
the outside. The only honest proof is the run's journal replayed against the
|
||||
database the server actually serves from: every `move` row must have a row at
|
||||
its destination, and NO row may still name its source — a source row is a scan
|
||||
that has not caught up, and it is the one signature of a library pointing at
|
||||
files that no longer exist.
|
||||
|
||||
Two doors, both read-only, both without a lock while the server runs (there is
|
||||
no `sqlite3` binary on these hosts). Open `mode=ro` PLAIN: both servers keep
|
||||
their libraries in WAL mode, and `immutable=1` — which claims the file never
|
||||
changes — makes sqlite skip the write-ahead log entirely, so every row the
|
||||
server has committed but not yet checkpointed reads as MISSING, and a read
|
||||
beside a running scanner can even fail as "database disk image is malformed".
|
||||
`immutable=1` is only the fallback, for a database whose `-shm` cannot be read.
|
||||
|
||||
jellyfin `BaseItems.Path`. The container binds the share at `/data`, so the
|
||||
database holds `/data/<library>/…` where the journal holds
|
||||
`/mnt/entropy/<library>/…`. Translate, or every row reads as missing.
|
||||
plex `media_parts.file`, joined up to `metadata_items`. Plex is bound at
|
||||
the host path, so its rows match the journal verbatim. Filter
|
||||
`deleted_at is null` on BOTH tables — a deleted scan leaves its parts
|
||||
behind, and one of those rows is not proof anything landed.
|
||||
|
||||
Neither server holds a row for everything the journal renames: a `.log`, `.cue`
|
||||
or cover image has no row to land in, so its absence is not a fault. The report
|
||||
therefore breaks the missing rows down by extension, and `--ignore-ext` names the
|
||||
extensions this server does not index, which are then counted separately and kept
|
||||
out of the verdict. What is left is split by what the filesystem says, because
|
||||
the database alone cannot: a file still at its `src` is `REAL` (this pass's own
|
||||
damage), a file at neither path is `gone` (a later pass or the server renamed it),
|
||||
and a file sitting at its `dst` with no row naming it is simply something the
|
||||
server does not index. A row that still names `src` is none of these: it is the
|
||||
server's own stale pointer — a phantom — and it costs a re-scan, never a restore.
|
||||
|
||||
Exit codes: 0 every move landed; 1 something is stale or missing (the details are
|
||||
printed); 2 the journal or the database is not usable at all.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
JF_DB = "/home/dank/Containers/jellyfin/config/data/jellyfin.db"
|
||||
PLEX_DB = ("/var/lib/plexmediaserver/Library/Application Support/Plex Media Server"
|
||||
"/Plug-in Support/Databases/com.plexapp.plugins.library.db")
|
||||
|
||||
# Per kind: the table whose columns are required, those columns, and the query
|
||||
# that returns every path the server believes it has.
|
||||
KINDS = {
|
||||
"jellyfin": {
|
||||
"table": "BaseItems",
|
||||
"columns": {"Path"},
|
||||
"sql": "select Path from BaseItems",
|
||||
"default_db": JF_DB,
|
||||
},
|
||||
"plex": {
|
||||
"table": "media_parts",
|
||||
"columns": {"file", "media_item_id", "deleted_at"},
|
||||
"sql": ("select p.file from media_parts p"
|
||||
" join media_items m on m.id = p.media_item_id"
|
||||
" join metadata_items mi on mi.id = m.metadata_item_id"
|
||||
" where p.deleted_at is null and mi.deleted_at is null"),
|
||||
"default_db": PLEX_DB,
|
||||
},
|
||||
}
|
||||
|
||||
def die(msg):
|
||||
print(f"media-verify: {msg}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def read_moves(path):
|
||||
"""The (src, dst) pairs of a journal, in the order the run performed them."""
|
||||
moves = []
|
||||
try:
|
||||
fh = open(path, encoding="utf-8")
|
||||
except OSError as e:
|
||||
die(f"cannot read the journal {path}: {e.strerror}")
|
||||
with fh:
|
||||
for n, line in enumerate(fh, 1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
die(f"{path} line {n} is not a journal record")
|
||||
if rec.get("op") == "move":
|
||||
if not rec.get("src") or not rec.get("dst"):
|
||||
die(f"{path} line {n} is a move without both src and dst")
|
||||
moves.append((rec["src"], rec["dst"]))
|
||||
return moves
|
||||
|
||||
|
||||
def server_paths(db, kind):
|
||||
"""Every path the server holds, or a refusal that names what is missing."""
|
||||
spec = KINDS[kind]
|
||||
con = None
|
||||
err = None
|
||||
# Plain ro reads the WAL the server commits through; immutable is the last
|
||||
# resort, because it cannot see a row the server has not checkpointed.
|
||||
for opt in ("mode=ro", "mode=ro&immutable=1"):
|
||||
try:
|
||||
con = sqlite3.connect(f"file:{db}?{opt}", uri=True)
|
||||
break
|
||||
except sqlite3.Error as e:
|
||||
err = e
|
||||
if con is None:
|
||||
die(f"cannot open {db}: {err}")
|
||||
with con:
|
||||
have = {r[1] for r in con.execute(f"pragma table_info({spec['table']})")}
|
||||
if not spec["columns"] <= have:
|
||||
die(f"{db} is not a {kind} library database ({spec['table']} has no "
|
||||
f"{','.join(sorted(spec['columns'] - have))})")
|
||||
try:
|
||||
return {r[0] for r in con.execute(spec["sql"]) if r[0]}
|
||||
except sqlite3.Error as e:
|
||||
die(f"{db} refused the {kind} query: {e}")
|
||||
|
||||
|
||||
def to_host(path, container, share):
|
||||
"""The database's path in the journal's terms; identity when it is not a
|
||||
container-side path."""
|
||||
if not container:
|
||||
return path
|
||||
prefix = container.rstrip("/") + "/"
|
||||
if path.startswith(prefix):
|
||||
return share.rstrip("/") + "/" + path[len(prefix):]
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--journal", required=True,
|
||||
help="the run's journal (scripts/media-apply.py writes it)")
|
||||
ap.add_argument("--db-kind", choices=sorted(KINDS), required=True)
|
||||
ap.add_argument("--db", default=None, help="the server's database on this host")
|
||||
ap.add_argument("--share", default="/mnt/entropy",
|
||||
help="the share every journal path lives under")
|
||||
ap.add_argument("--container", default=None,
|
||||
help="what the share is called inside the container; empty"
|
||||
" compares verbatim (default /data for jellyfin, nothing"
|
||||
" for plex)")
|
||||
ap.add_argument("--show", type=int, default=20,
|
||||
help="how many offenders to name before saying 'and N more'")
|
||||
ap.add_argument("--ignore-ext", default="",
|
||||
help="comma-separated extensions this server does not index,"
|
||||
" kept out of the verdict (e.g. log,cue,nfo,srt)")
|
||||
args = ap.parse_args()
|
||||
|
||||
db = args.db or KINDS[args.db_kind]["default_db"]
|
||||
container = ("/data" if args.db_kind == "jellyfin" else "") \
|
||||
if args.container is None else args.container
|
||||
ignore = set()
|
||||
for ext in args.ignore_ext.split(","):
|
||||
ext = ext.strip().lower()
|
||||
if ext:
|
||||
ignore.add(ext if ext.startswith(".") else "." + ext)
|
||||
|
||||
moves = read_moves(args.journal)
|
||||
stored = {to_host(p, container, args.share) for p in server_paths(db, args.db_kind)}
|
||||
|
||||
landed, stale, missing, ignored = 0, [], [], 0
|
||||
for src, dst in moves:
|
||||
if os.path.splitext(dst)[1].lower() in ignore:
|
||||
ignored += 1
|
||||
continue
|
||||
at_dst, at_src = dst in stored, src in stored
|
||||
if at_dst and not at_src:
|
||||
landed += 1
|
||||
continue
|
||||
if at_src:
|
||||
stale.append(src)
|
||||
if not at_dst:
|
||||
missing.append((src, dst))
|
||||
|
||||
# The database cannot tell a lost file from one it never indexes: both read
|
||||
# as "no row at the destination". Stat both paths, so the report separates
|
||||
# the classes the runbook names — `REAL` is the only one that is this pass's
|
||||
# own damage, and `gone` is a later pass or the server's own rename.
|
||||
on_disk, real, gone = [], [], []
|
||||
for src, dst in missing:
|
||||
if os.path.exists(dst):
|
||||
on_disk.append(dst)
|
||||
elif os.path.exists(src):
|
||||
real.append(src)
|
||||
else:
|
||||
gone.append(dst)
|
||||
|
||||
print(f"{args.db_kind}: {len(moves)} moves checked")
|
||||
print(f"landed: {landed}")
|
||||
print(f"still at src: {len(stale)}")
|
||||
print(f"missing from the DB: {len(missing)}")
|
||||
print(f" on disk at dst, no row names it: {len(on_disk)}")
|
||||
print(f" REAL — file still at src: {len(real)}")
|
||||
print(f" gone — at neither path: {len(gone)}")
|
||||
if ignored:
|
||||
print(f"not indexed (extension ignored): {ignored}")
|
||||
if missing:
|
||||
exts = Counter(os.path.splitext(d)[1].lower() or "(no extension)"
|
||||
for _, d in missing)
|
||||
print("missing by extension: "
|
||||
+ ", ".join(f"{e} {n}" for e, n in sorted(exts.items())))
|
||||
for label, items in (("real", real), ("gone", gone), ("missing", on_disk),
|
||||
("stale", stale)):
|
||||
for path in items[:args.show]:
|
||||
print(f" {label}: {path}")
|
||||
if len(items) > args.show:
|
||||
print(f" {label}: … and {len(items) - args.show} more")
|
||||
return 1 if (stale or missing) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+241
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env bash
|
||||
# The SSD-backed metadata view that lets Jellyfin and Plex share one metadata set.
|
||||
#
|
||||
# Both servers read a Kodi-format sidecar (movie.nfo / tvshow.nfo / episode.nfo)
|
||||
# ONLY when it sits next to the media file. The media lives on the RAID6 array at
|
||||
# /mnt/entropy — five spinning disks, deliberately mounted read-only for every
|
||||
# container, and slow to seek for a thousand tiny XML reads. So neither media
|
||||
# server binds the array any more: they bind an OverlayFS merge of it with a
|
||||
# writable layer on the NVMe.
|
||||
#
|
||||
# lowerdir /mnt/entropy RAID6 (ROTA=1) — source of truth, never written
|
||||
# upperdir /var/lib/media-meta/upper NVMe — every sidecar and image byte
|
||||
# workdir /var/lib/media-meta/work NVMe — kernel bookkeeping
|
||||
# Where /var/lib/media-meta/view the merged tree jellyfin and plex bind
|
||||
#
|
||||
# Everything the view needs — layers, bookkeeping, trash, the merged door — lives
|
||||
# under the one directory /var/lib/media-meta, so nothing new appears under /mnt.
|
||||
#
|
||||
# Consequences worth knowing:
|
||||
# * A sidecar write is a few KB onto the SSD; the media bytes never move and the
|
||||
# array is still never written by anything automated.
|
||||
# * Deleting through the view writes a whiteout into the upper layer instead of
|
||||
# deleting from the array. Nothing should delete through the view: deletions
|
||||
# go to .AGENTTRASH on the RAW path, and `reap` clears the resulting ghosts.
|
||||
# * inotify events on the lower layer do not cross the merge, so a file that
|
||||
# arrives on the raw array is invisible to a file-system watcher in the view:
|
||||
# trigger a library refresh (see the Media Relayout page).
|
||||
#
|
||||
# Subcommands
|
||||
# plan [--upper-base D] [--lower D] print the resolved shape, touch nothing
|
||||
# verify prove the live mount IS that shape
|
||||
# install [--upper-base D] create the layers, install+enable+start the unit
|
||||
# reap [--apply] [--upper-base D] [--lower D]
|
||||
# list upper-layer entries whose media is gone
|
||||
# status one line per fact a human wants
|
||||
#
|
||||
# install must run as root (it writes /etc/systemd/system and mounts); the others
|
||||
# are read-only and safe unprivileged.
|
||||
set -euo pipefail
|
||||
|
||||
LOWER=/mnt/entropy
|
||||
UPPER_BASE=/var/lib/media-meta
|
||||
VIEW=$UPPER_BASE/view
|
||||
# systemd names a mount unit after its ESCAPED path ('/' -> '-', a literal '-'
|
||||
# -> '\x2d'), so the unit on the host is var-lib-media\x2dmeta-view.mount. The
|
||||
# repo copy keeps a readable name; install writes it under the escaped one.
|
||||
UNIT='var-lib-media\x2dmeta-view.mount'
|
||||
UNIT_DIR=/etc/systemd/system
|
||||
# The rootless containers write as this host uid (the deploy user). The upper
|
||||
# layer has to accept those writes, so it is owned by them.
|
||||
MEDIA_UID="${MEDIA_VIEW_UID:-1000}"
|
||||
MEDIA_GID="${MEDIA_VIEW_GID:-1000}"
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
UNIT_SRC="$REPO_ROOT/dankserver/host/systemd/media-meta-view.mount"
|
||||
|
||||
die() {
|
||||
printf 'media-view: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
usage() {
|
||||
printf 'usage: %s plan|verify|install|reap|status [options]\n' "$0" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
cmd="${1:-}"
|
||||
[ -n "$cmd" ] || usage
|
||||
shift || true
|
||||
|
||||
upper_base="$UPPER_BASE"
|
||||
lower="$LOWER"
|
||||
apply=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--upper-base) [ $# -ge 2 ] || usage; upper_base="$2"; shift 2 ;;
|
||||
--lower) [ $# -ge 2 ] || usage; lower="$2"; shift 2 ;;
|
||||
--apply) apply=1; shift ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
upper="$upper_base/upper"
|
||||
work="$upper_base/work"
|
||||
|
||||
# The device backing a path, and whether the kernel calls it rotational.
|
||||
rotational_of() {
|
||||
local src rota
|
||||
src="$(findmnt -no SOURCE --target "$1" 2>/dev/null || true)"
|
||||
[ -n "$src" ] || die "cannot find a block device behind $1"
|
||||
rota="$(lsblk -no ROTA "$src" 2>/dev/null | head -1 || true)"
|
||||
[ -n "$rota" ] || die "cannot read the rotational flag of $src"
|
||||
printf '%s %s\n' "$src" "$rota"
|
||||
}
|
||||
|
||||
require_upper_base() {
|
||||
[ -d "$upper_base" ] \
|
||||
|| die "refused: upper base $upper_base does not exist (put it on the SSD first)"
|
||||
}
|
||||
|
||||
do_plan() {
|
||||
printf 'lowerdir=%s\nupperdir=%s\nworkdir=%s\nmountpoint=%s\nunit=%s/%s\n' \
|
||||
"$lower" "$upper" "$work" "$VIEW" "$UNIT_DIR" "$UNIT"
|
||||
}
|
||||
|
||||
do_verify() {
|
||||
local fail=0 fst opts src rota
|
||||
if ! mountpoint -q "$VIEW"; then
|
||||
printf 'refused: %s is not a mountpoint — a plain directory here is a\n' "$VIEW" >&2
|
||||
printf ' library that scans itself empty, not a metadata view.\n' >&2
|
||||
return 1
|
||||
fi
|
||||
fst="$(findmnt -no FSTYPE --mountpoint "$VIEW" 2>/dev/null || true)"
|
||||
if [ "$fst" != overlay ]; then
|
||||
printf 'refused: %s is mounted %s, not overlay\n' "$VIEW" "${fst:-nothing}" >&2
|
||||
fail=1
|
||||
fi
|
||||
opts="$(findmnt -no OPTIONS --mountpoint "$VIEW" 2>/dev/null || true)"
|
||||
local want
|
||||
for want in "lowerdir=$LOWER" "upperdir=$UPPER_BASE/upper" "workdir=$UPPER_BASE/work"; do
|
||||
case ",$opts," in
|
||||
*",${want},"*) ;;
|
||||
*) printf 'refused: %s options lack %s (got: %s)\n' "$VIEW" "$want" "$opts" >&2
|
||||
fail=1 ;;
|
||||
esac
|
||||
done
|
||||
read -r src rota <<<"$(rotational_of "$UPPER_BASE")"
|
||||
if [ "$rota" != 0 ]; then
|
||||
printf 'refused: the upper layer sits on %s which is rotational — metadata\n' "$src" >&2
|
||||
printf ' belongs on the SSD, not on the array.\n' >&2
|
||||
fail=1
|
||||
fi
|
||||
if [ ! -w "$UPPER_BASE/upper" ]; then
|
||||
printf 'refused: %s is not writable\n' "$upper" >&2
|
||||
fail=1
|
||||
fi
|
||||
[ "$fail" -eq 0 ] || return 1
|
||||
printf 'view ok: %s <- overlay(%s,%s)\n' "$VIEW" "$LOWER" "$UPPER_BASE/upper"
|
||||
}
|
||||
|
||||
do_install() {
|
||||
[ "$(id -u)" -eq 0 ] || die "install must run as root (writes $UNIT_DIR and mounts)"
|
||||
require_upper_base
|
||||
mountpoint -q "$LOWER" \
|
||||
|| die "refused: $LOWER is not mounted; overlaying an empty directory would present an empty library"
|
||||
local src rota
|
||||
read -r src rota <<<"$(rotational_of "$upper_base")"
|
||||
[ "$rota" = 0 ] || die "refused: $upper_base is on $src, a rotational device — the metadata layer must be on the SSD"
|
||||
[ -f "$UNIT_SRC" ] || die "refused: $UNIT_SRC is missing from $REPO_ROOT"
|
||||
# systemd will not create a mount point; a missing directory fails the start.
|
||||
install -d -m 0755 -o "$MEDIA_UID" -g "$MEDIA_GID" "$upper" "$work" "$VIEW"
|
||||
install -d -m 0700 "$upper_base/.AGENTTRASH"
|
||||
install -m 0644 "$UNIT_SRC" "$UNIT_DIR/$UNIT"
|
||||
systemctl daemon-reload
|
||||
systemctl enable "$UNIT" >/dev/null 2>&1 || true
|
||||
systemctl start "$UNIT"
|
||||
do_verify
|
||||
}
|
||||
|
||||
# An upper-layer DIRECTORY with no counterpart below is metadata for a media
|
||||
# folder that has been moved or renamed on the RAW path — a ghost the media
|
||||
# servers would still browse. Files are never candidates on their own: a sidecar
|
||||
# (movie.nfo, an .srt) exists ONLY above, by design, and is exactly what the view
|
||||
# is for. A copy-up whose lower file was deleted by hand is the one case this
|
||||
# leaves in place; the raw path is where deletions belong, so it stays visible.
|
||||
do_reap() {
|
||||
[ -d "$upper" ] || die "refused: $upper does not exist (no upper layer at $upper_base)"
|
||||
stamp="$upper_base/.AGENTTRASH/$(date -u +%Y%m%d-%H%M%S)"
|
||||
count=0
|
||||
ghosts=0
|
||||
reap_walk ""
|
||||
if [ "$count" -eq 0 ] && [ "$ghosts" -eq 0 ]; then
|
||||
printf 'reap: no ghosts under %s\n' "$upper"
|
||||
return 0
|
||||
fi
|
||||
if [ "$ghosts" -gt 0 ]; then
|
||||
printf 'reap: %d whiteout%s above — someone deleted THROUGH the view.\n' \
|
||||
"$ghosts" "$([ "$ghosts" -eq 1 ] || printf s)" >&2
|
||||
printf ' Leaving them: removing a whiteout un-hides the file it hides.\n' >&2
|
||||
printf ' Deletions belong in .AGENTTRASH on the raw path.\n' >&2
|
||||
fi
|
||||
if [ "$count" -eq 0 ]; then return 0; fi
|
||||
if [ "$apply" -eq 1 ]; then
|
||||
printf 'reap: moved %d entr%s to %s\n' "$count" \
|
||||
"$([ "$count" -eq 1 ] && printf y || printf ies)" "$stamp"
|
||||
else
|
||||
printf 'reap: %d ghost entr%s above; re-run with --apply to move them to %s\n' \
|
||||
"$count" "$([ "$count" -eq 1 ] && printf y || printf ies)" "$stamp"
|
||||
fi
|
||||
}
|
||||
|
||||
# Descend only where the lower layer still has the directory: a directory that is
|
||||
# missing below is itself the ghost, and its whole subtree goes with it.
|
||||
reap_walk() {
|
||||
local rel="$1" entries abs r name
|
||||
entries="$(find "$upper${rel:+/$rel}" -mindepth 1 -maxdepth 1 -print 2>/dev/null || true)"
|
||||
[ -n "$entries" ] || return 0
|
||||
while IFS= read -r abs; do
|
||||
r="${abs#"$upper"/}"
|
||||
name="${abs##*/}"
|
||||
if [ -c "$abs" ] || printf '%s' "$name" | grep -q '^\.wh\.'; then
|
||||
ghosts=$((ghosts + 1))
|
||||
continue
|
||||
fi
|
||||
if [ -e "$lower/$r" ]; then
|
||||
[ -d "$abs" ] && reap_walk "$r"
|
||||
continue
|
||||
fi
|
||||
[ -d "$abs" ] || continue
|
||||
count=$((count + 1))
|
||||
printf '%s\n' "$r"
|
||||
if [ "$apply" -eq 1 ]; then
|
||||
mkdir -p "$stamp/$(dirname "$r")"
|
||||
mv "$abs" "$stamp/$r"
|
||||
fi
|
||||
done <<<"$entries"
|
||||
}
|
||||
|
||||
do_status() {
|
||||
if mountpoint -q "$VIEW"; then
|
||||
printf 'view: %s (%s)\n' "$VIEW" "$(findmnt -no OPTIONS --mountpoint "$VIEW")"
|
||||
else
|
||||
printf 'view: %s NOT MOUNTED\n' "$VIEW"
|
||||
fi
|
||||
if [ -d "$upper" ]; then
|
||||
printf 'upper: %s %s files %s\n' "$upper" \
|
||||
"$(find "$upper" -type f 2>/dev/null | wc -l)" "$(du -sh "$upper" 2>/dev/null | cut -f1)"
|
||||
else
|
||||
printf 'upper: %s missing\n' "$upper"
|
||||
fi
|
||||
systemctl is-active "$UNIT" 2>/dev/null | sed 's/^/unit: /' || printf 'unit: %s unknown\n' "$UNIT"
|
||||
}
|
||||
|
||||
case "$cmd" in
|
||||
plan) do_plan ;;
|
||||
verify) do_verify ;;
|
||||
install) do_install ;;
|
||||
reap) do_reap ;;
|
||||
status) do_status ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -94,6 +94,16 @@ for path in "${CHANGED[@]}"; do
|
||||
dankserver/.env.example|dankclaw/.env.example)
|
||||
add_test tests/env_consistency.bats
|
||||
;;
|
||||
# The SSD-backed metadata view: the installer, the host mount unit, and the
|
||||
# two media units that bind the view rather than the raw array.
|
||||
scripts/media-view.sh|dankserver/host/*)
|
||||
add_test tests/media_overlay.bats
|
||||
add_test tests/dankserver_quadlet.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
;;
|
||||
dankserver/jellyfin/quadlet/*|dankserver/plex/quadlet/*)
|
||||
add_test tests/media_overlay.bats
|
||||
;;
|
||||
shared/homepage/*)
|
||||
add_test tests/homepage_caddy.bats
|
||||
add_test tests/homepage_config.bats
|
||||
|
||||
@@ -48,6 +48,14 @@ Use the `Authorization: token` header for everything.
|
||||
- **Read an issue**: `curl -s -K "$cfg" $API/issues/<number>` (fields: `number`, `title`, `body`, `labels[].name`, `state`)
|
||||
- **List issues**: `curl -s -K "$cfg" "$API/issues?state=open&limit=50"` — add `&labels=<id>` to filter
|
||||
- **Comment**: `curl -s -K "$cfg" -X POST -d "$(jq -n --rawfile b "$BODY" '{body:$b}')" $API/issues/<number>/comments`
|
||||
- **Comment on a PR**: the conversation thread is `POST $API/issues/<PR-number>/comments`
|
||||
(201, and the answer's `html_url` points at `pulls/<n>`; Gitea's issue index is
|
||||
unified, so the PR's number is the same number). `$API/pulls/<n>/comments` is the
|
||||
**review**-comment store and answers `405` to a plain body.
|
||||
- **A body built by `jq --rawfile b /dev/stdin` inside `$(…)` is empty**: the command
|
||||
substitution does not inherit the heredoc, so Gitea receives `{}` and answers 422 —
|
||||
and `jq -r '.id'` on the error prints nothing, which reads as success. Write the
|
||||
body to a file and pass `--rawfile` that file.
|
||||
- **Apply labels**: `POST $API/issues/<number>/labels` with `{"labels":[<label-ID numbers>]}` — Gitea wants numeric label **IDs**, not names; resolve them from `GET $API/labels` (see [[Triage Labels]] for which labels exist)
|
||||
- **Remove a label**: `DELETE $API/issues/<number>/labels/<label-id>`
|
||||
- **Close**: `PATCH $API/issues/<number>` with `{"state":"closed"}`
|
||||
|
||||
@@ -195,8 +195,9 @@ deploy.sh, never a live env poke.
|
||||
check every namespace (`sudo podman ps -a`) and native units
|
||||
(`systemctl list-units`) before declaring a service absent. Compare
|
||||
compose mount sources against the real filesystem — compose files name
|
||||
phantom paths (e.g. `/mnt/media/*` vs the real LUKS array at
|
||||
`/mnt/entropy`). Fix compose through git first, deploy second.
|
||||
phantom paths (the legacy `dankserver/compose.yaml` still names `/opt/jellyfin/*`
|
||||
for a service whose roots moved to `/home/dank/Containers/jellyfin`). Fix
|
||||
compose through git first, deploy second.
|
||||
2. **Pre-create bind-mount sources** (`sudo mkdir -p` + `chown 1000:1000`)
|
||||
or creation fails `statfs …: no such file or directory`.
|
||||
3. **Staged bring-up:** infra first (`up -d mariadb redis`), poll
|
||||
|
||||
@@ -1,51 +1,68 @@
|
||||
---
|
||||
type: reference
|
||||
description: "The Dankserver entropy volume is a RAID-backed LUKS filesystem mounted at"
|
||||
description: "Dankserver's /mnt/entropy volume: the LUKS/array chain, the crypttab + fstab boot path, and the containers bound to it"
|
||||
tags:
|
||||
- dankserver
|
||||
timestamp: 2026-08-28
|
||||
timestamp: 2026-10-31
|
||||
---
|
||||
|
||||
# Entropy Volume
|
||||
|
||||
The Dankserver entropy volume is a RAID-backed LUKS filesystem mounted at
|
||||
`/mnt/entropy`. The command-line helper reuses the host's desktop-integrated
|
||||
`udisks2`, `crypttab`, and `fstab` configuration; it does not store or handle
|
||||
the passphrase itself.
|
||||
`/mnt/entropy` is the ext4 filesystem on the LUKS container holding `/dev/md0`
|
||||
(RAID6, seven disks). The media stack's data lives there, so the mount is a boot
|
||||
dependency for those containers rather than a convenience.
|
||||
|
||||
## Command-line mount
|
||||
|
||||
```bash
|
||||
/home/dank/serverAppConfigs/scripts/mount-entropy.sh
|
||||
```
|
||||
|
||||
Run it as the normal desktop user (`dank`), not from a root shell. `udisksctl`
|
||||
will securely prompt for the LUKS passphrase. `sudo mount /mnt/entropy` then
|
||||
uses the existing `/etc/fstab` entry, preserving its options:
|
||||
## Chain
|
||||
|
||||
```text
|
||||
nosuid,nodev
|
||||
/dev/md0 <- mdadm (its mdadm.conf ARRAY is what makes md0 exist at boot)
|
||||
-> LUKS UUID=cd86c728-459e-4f3a-bcc2-41ecf84685ec keyfile: /etc/entropy
|
||||
-> /dev/mapper/entropy <- systemd-cryptsetup@entropy.service
|
||||
-> ext4 at /mnt/entropy <- mnt-entropy.mount, from /etc/fstab
|
||||
```
|
||||
|
||||
The existing mountpoint is owned by `dank:dankserver` with mode `0755`; the
|
||||
helper deliberately does not run `chmod` or `chown`, so it cannot silently
|
||||
change those permissions.
|
||||
## Files that own it
|
||||
|
||||
## Device path
|
||||
| Repo file | Installs to | Produces at boot |
|
||||
|---|---|---|
|
||||
| `dankserver/host/crypttab` | `/etc/crypttab` | `systemd-cryptsetup@entropy.service` → `/dev/mapper/entropy` |
|
||||
| `dankserver/host/fstab` | `/etc/fstab` | `mnt-entropy.mount` (mountpoint `dank:dankserver`, `0755`) |
|
||||
|
||||
The current storage chain is:
|
||||
Deploy a change by copying the file onto the host, then
|
||||
`systemctl daemon-reload && update-initramfs -u`. The initramfs step is not
|
||||
optional: the cryptsetup hook bakes `/etc/crypttab` into the initrd, and the
|
||||
array is unlocked there, before the real root mounts. `scripts/mount-entropy.sh`
|
||||
(`udisksctl unlock` + `mount`) stays as the manual path for a volume that was
|
||||
never unlocked; on a correct boot it is redundant.
|
||||
|
||||
```text
|
||||
/dev/md0 -> LUKS UUID cd86c728-459e-4f3a-bcc2-41ecf84685ec
|
||||
-> ext4 label entropy
|
||||
-> /mnt/entropy
|
||||
```
|
||||
## Law
|
||||
|
||||
The helper checks that `/dev/md0` exists and exits successfully if the volume
|
||||
is already mounted. It is intentionally specific to this host and volume:
|
||||
that avoids accidentally unlocking or mounting a different disk.
|
||||
**`crypttab`'s device field takes the LUKS UUID, never the md array UUID.**
|
||||
`blkid` prints both on adjacent lines for a RAID-backed container, and the file
|
||||
shipped since 2025 carried the array UUID (`61bd6eaf-…`) where the LUKS container
|
||||
is `cd86c728-…`. The generator produced a perfectly valid unit pointing at a
|
||||
device that never appears, so the mapper came into existence only through a
|
||||
desktop `udisks2` session — the volume looked mounted on every login while every
|
||||
boot failed to mount it. The matching `fstab` line named the same wrong UUID
|
||||
directly instead of the mapper.
|
||||
|
||||
## Related
|
||||
**A LUKS keyfile is hashed whole, so a trailing newline is a different
|
||||
passphrase.** The shipped `/etc/entropy` was 27 bytes — 26 characters plus `\n` —
|
||||
and `cryptsetup` rejected it even after the UUID was corrected. Keyfiles stay
|
||||
`0600` with no trailing newline; prove it with
|
||||
`cryptsetup open --test-passphrase --key-file /etc/entropy UUID=…` *before*
|
||||
relying on it for a boot.
|
||||
|
||||
- [[Storage Policy]]
|
||||
- [[Dankserver Stack]]
|
||||
**Prove a mount fix with the artifacts systemd generates, not with a live
|
||||
mount.** The regenerated `systemd-cryptsetup@entropy.service` naming the right
|
||||
device, `findmnt /mnt/entropy` sourcing `/dev/mapper/entropy`, and a matching
|
||||
checksum between the repo copy and the host copy are the evidence. A mount that
|
||||
exists because someone logged in proves nothing about the boot path — that
|
||||
exactly describes the year this bug survived.
|
||||
|
||||
## Consumers
|
||||
|
||||
`jellyfin`, `copyparty`, and `plex` bind paths under `/mnt/entropy`; with the
|
||||
volume absent they start `active (healthy)` against empty directories, which is
|
||||
how a missing 91 TiB volume hides. See [[Dankserver Stack]] for the stack and
|
||||
[[Storage Policy]] for what may be written here.
|
||||
@@ -1,9 +1,9 @@
|
||||
---
|
||||
type: reference
|
||||
title: Jellyfin
|
||||
description: "Jellyfin on Dankserver: quadlet unit, entropy-volume data layout, ports, client access, hardware-transcode status."
|
||||
description: "Jellyfin on Dankserver: quadlet unit, overlay metadata view, provider keys, ports, access, transcode status."
|
||||
tags: [jellyfin, dankserver, media-server]
|
||||
timestamp: 2026-09-03T00:00:00Z
|
||||
timestamp: 2026-10-30T00:00:00Z
|
||||
---
|
||||
|
||||
# Jellyfin
|
||||
@@ -18,7 +18,7 @@ clients.
|
||||
public vhost `jelly.franksdank.cloud` behind the [[Anubis]] guard chain →
|
||||
[[Authentik]] outpost (post-auth route proxies `localhost:8096`).
|
||||
|
||||
## Facts (verified from the live unit, 2026-09-03)
|
||||
## Facts (the repo quadlet; the live unit is still the pre-view generation — see Data layout)
|
||||
|
||||
| Item | Value |
|
||||
|------|-------|
|
||||
@@ -26,7 +26,7 @@ public vhost `jelly.franksdank.cloud` behind the [[Anubis]] guard chain →
|
||||
| Image | `ghcr.io/jellyfin/jellyfin:latest` |
|
||||
| Network | `dankserver_appnet`; publishes 8096/tcp, 8920/tcp, 7359/udp, 1900/udp |
|
||||
| Data roots | `/home/dank/Containers/jellyfin/config:/config`, `/home/dank/Containers/jellyfin/cache:/cache` |
|
||||
| Media | `Volume=/mnt/entropy:/data:ro` — the whole [[Entropy Volume]] mounted read-only; nothing automated writes to it |
|
||||
| Media | `Volume=/var/lib/media-meta/view:/data` in git — the [[Media Metadata View]] overlay, **writable**, sidecars land on the NVMe and the [[Entropy Volume]] array underneath stays untouched. **The live unit is still pre-view: `/mnt/entropy:/data:ro`** — nothing is written into the library until the view is deployed |
|
||||
| GPU | none — no `AddDevice`; CPU decode only (jellyfin-ffmpeg path) |
|
||||
| Env quirks | `DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true`, `LD_PRELOAD=libjemalloc.so.2`, `MALLOC_TRIM_THRESHOLD_=131072` kept verbatim from the compose-era container (zero-drift cutover) |
|
||||
| Health | `curl -f http://localhost:8096/health` every 30 s |
|
||||
@@ -36,11 +36,27 @@ public vhost `jelly.franksdank.cloud` behind the [[Anubis]] guard chain →
|
||||
```
|
||||
/home/dank/Containers/jellyfin/config -> /config (JELLYFIN_DATA_DIR)
|
||||
/home/dank/Containers/jellyfin/cache -> /cache (JELLYFIN_CACHE_DIR, XDG_CACHE_HOME)
|
||||
/mnt/entropy -> /data (read-only media volume)
|
||||
/var/lib/media-meta/view -> /data (git: the overlay view)
|
||||
/mnt/entropy -> /data (live today: the array, read-only)
|
||||
```
|
||||
|
||||
Library folders point at `/data` paths; because the mount uses the same
|
||||
absolute path as on the host, library paths survive container recreation.
|
||||
Library folders point at `/data` paths: stable across container recreation and
|
||||
identical to the raw array layout below. Which side of that path the container
|
||||
is on is the open question on this host — the installed unit binds the read-only
|
||||
array, and `scripts/media-view.sh status` reports `NOT MOUNTED` (2026-10-31).
|
||||
|
||||
`Local metadata` — save metadata to media folders — is enabled, and it is the
|
||||
reason the view exists: `movie.nfo`, `tvshow.nfo`, `episode.nfo` and artwork
|
||||
written once on the NVMe upper layer are files [[Plex]] can read too. While
|
||||
`/data` is bound read-only every item update logs the error below and no sidecar
|
||||
is written; Jellyfin becomes the fleet's only metadata writer with the deploy
|
||||
that raises the view.
|
||||
|
||||
**Failure signature:** `System.IO.IOException: Read-only file system :
|
||||
'/data/Movies/<film>/movie.nfo'` means the container is binding the raw array
|
||||
instead of the view. On Dankserver that is today's expected state — the view has
|
||||
never been installed (`scripts/media-view.sh status`). Once it is deployed, the
|
||||
same line is a fault: the quadlet drifted or the mount died.
|
||||
|
||||
## Ports & access
|
||||
|
||||
@@ -105,12 +121,109 @@ where the LDAP plugin runs. Verified 2026-09-25: `alex` authenticates over
|
||||
`AuthenticateByName` (HTTP 200; full plugin flow logged — service bind,
|
||||
search, `ldapUid`, user bind, token).
|
||||
|
||||
## Metadata providers and their keys (2026-10-30)
|
||||
|
||||
TMDb, OMDb, StudioImages and MusicBrainz are **core plugins** in Jellyfin 12 —
|
||||
no DLL, just a configuration file the running container owns under the data
|
||||
root. TVDB is not core (its metadata provider was removed in 10.9), so it is an
|
||||
official-catalog plugin.
|
||||
|
||||
| Provider | Shape on disk | Key lives in |
|
||||
|---|---|---|
|
||||
| TMDb | `config/plugins/configurations/Jellyfin.Plugin.Tmdb.xml` → `<TmdbApiKey>` | `/home/dank/Containers/jellyfin/provider-keys` → `TMDB_API_KEY=` (mode 600) |
|
||||
| TVDB | `config/plugins/thetvdb_24.0.0.0/` + `…Jellyfin.Plugin.Tvdb.xml` → `<ApiKey>` | the same file → `TVDB_API_KEY=` |
|
||||
| OMDb | `…Jellyfin.Plugin.Omdb.xml` | **not configured** — its key is issued by email, 1 000 req/day, and its ToS forbids storing the returned data; ratings stay Jellyfin's own lookup |
|
||||
|
||||
**None of it is tracked in git.** Those XML files belong to the container and
|
||||
hold credentials, and `tests/jellyfin_providers.bats` fails the suite if one is
|
||||
ever committed. The keys themselves sit in the per-service host env file that is
|
||||
the fleet's convention for container credentials (grafana, caddy, pihole,
|
||||
copyparty, romm, seerr): `provider-keys`, mode 600, beside the data root.
|
||||
`scripts/jellyfin-providers.sh` is the only thing that writes the XML:
|
||||
|
||||
```
|
||||
scripts/jellyfin-providers.sh plan # what would change; prints no key
|
||||
scripts/jellyfin-providers.sh install # plugin + keys, previous copy to trash
|
||||
scripts/jellyfin-providers.sh verify # plugin + key LENGTHS, never key bytes
|
||||
```
|
||||
|
||||
- **A key never enters git, argv, or the tool's output.** The installer reads
|
||||
that 600-perm file itself (a value in argv is world-readable through
|
||||
`/proc/<pid>/cmdline`) and refuses it when it is group- or world-readable —
|
||||
an exposed key is an error, not a warning. A key that shows up in a transcript
|
||||
or `ps` output has leaked: rotate it with the provider, rewrite
|
||||
`provider-keys`, re-run `install`.
|
||||
- **The TVDB artifact is pinned three ways**: sha256
|
||||
`6302fba3857bdde8d186dd099fac09ca43247041f531a528a2c788add70a5319`, the
|
||||
official plugin guid `a677c0da-fac5-4cde-941a-7134223f14c8`, and version
|
||||
`24.0.0.0` (`targetAbi 12.0.0.0`, which fits the deployed 12.4.0). A sha
|
||||
match alone only proves the bytes are the bytes fetched — the guid check is
|
||||
what says it is the official plugin. Rollback: move `thetvdb_24.0.0.0/` into
|
||||
the trash and revert the pin; a second `thetvdb_*` directory is a refusal,
|
||||
never a quiet second copy.
|
||||
- **The homelab TVDB key is a licensed key** (its JWT claims say
|
||||
`community_supported: false`), so it needs **no PIN** — the optional
|
||||
`TVDB_PIN=` line exists only for a subscriber-shaped key.
|
||||
- **TVDB is installed and keyed but is not a metadata agent**: TMDb still
|
||||
identifies everything. Enabling TVDB as a provider would re-fetch show
|
||||
metadata across the library, so it is a deliberate, separate change.
|
||||
- **The deployer places the keys before it restarts jellyfin**
|
||||
(`ensure_provider_keys` in `scripts/deploy-remote.sh`) and warns instead of
|
||||
failing when `provider-keys` is absent, so a host without it still deploys.
|
||||
- **Keys and plugins are read at start-up**: after a manual `install` the
|
||||
change is inert until `./scripts/deploy.sh dankserver --service jellyfin`.
|
||||
- TMDb's terms are the reason this stays narrow: attribution required, no
|
||||
caching beyond 6 months, no derivative works. Fine for a personal homelab,
|
||||
and the reason no bulk enrichment pass runs against it.
|
||||
|
||||
## Configuration workflow
|
||||
|
||||
1. Create libraries pointing at `/data` folders (movies, shows, audio).
|
||||
2. Set metadata/language preferences (`LANG=en_US.UTF-8` pinned in the unit).
|
||||
3. Configure clients to use `http://192.168.0.235:8096` or the guarded vhost.
|
||||
|
||||
## Triggering a scan
|
||||
|
||||
The scan key lives in `~/.config/secrets/jellyfin` (mode 600; the same value is
|
||||
`JELLYFIN_API_KEY` in the ignored `dankserver/.env`). It rides a mode-600 curl
|
||||
config, never a header argument, in the `MediaBrowser` form: 12.4 authenticates
|
||||
only `Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token`
|
||||
for every key, valid or not. Before believing a 401, prove the key against the
|
||||
server's own table — `select count(*) from ApiKeys where AccessToken = ?` on
|
||||
`jellyfin.db` opened `mode=ro&immutable=1`. The backslashes below matter:
|
||||
|
||||
```bash
|
||||
umask 077
|
||||
{ printf 'header = "Authorization: MediaBrowser Token=\\"%s\\""\n' \
|
||||
"$(cat ~/.config/secrets/jellyfin)"
|
||||
echo 'url = "http://127.0.0.1:8096/Library/Refresh"'
|
||||
echo 'request = "POST"'; } >/tmp/jf.hdr
|
||||
curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n"
|
||||
shred -u /tmp/jf.hdr
|
||||
```
|
||||
|
||||
`204` is acceptance; anything else queued no scan, so re-issue the identical
|
||||
request instead of diagnosing. A `401` from a key that answers `200` on a
|
||||
hand-typed request is this block's quoting, not the server: bash `printf`
|
||||
collapses `\"` to `"`, curl ends the quoted value at that quote, and Jellyfin
|
||||
receives `Authorization: MediaBrowser Token=` with nothing after it (`\\"` answers
|
||||
204, `\"` answers 401 — both run on this host).
|
||||
|
||||
One accepted refresh is the whole procedure. A second one while a scan runs moves
|
||||
`Scan Media Library` to `Cancelling`, and once the runner holds `Cancelling` every
|
||||
later `204` is accepted and then discarded — the status code stops meaning
|
||||
anything. The fix is `systemctl --user restart jellyfin.service` (never
|
||||
`deploy.sh`: that runs the never-tested metadata-view cutover), one POST after
|
||||
health answers, and then believe `ScheduledTasks` State at 130 % CPU, not the
|
||||
response code.
|
||||
|
||||
**A scan that must see a new folder has to be recursive.** `POST /Library/Refresh`
|
||||
is the incremental pass, and inotify does not cross a rootless podman bind, so
|
||||
folders created on the host stay invisible to it. The move is
|
||||
`POST /Items/{id}/Refresh?Recursive=true` (204) per library, the id from
|
||||
`GET /Library/VirtualFolders` — its field is `ItemId`, not `Id` — and that same
|
||||
record's `RefreshStatus` is the progress read.
|
||||
|
||||
## See Also
|
||||
|
||||
- [[Plex]] — co-resident media server on the same volume
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
type: reference
|
||||
title: Media Audiobook Relayout
|
||||
description: "Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals."
|
||||
tags: [dankserver, media-server, plex, media-relayout]
|
||||
timestamp: 2026-10-31T00:00:00Z
|
||||
---
|
||||
|
||||
# Media Audiobook Relayout
|
||||
|
||||
`/mnt/entropy/Audiobooks` is the one library here that [[Jellyfin]] does not read
|
||||
at all — its music scanner has no audiobook concept — while [[Plex]] indexes all
|
||||
of it as Artist → Album → Track: the album is the BOOK, that book's single artist
|
||||
is the NARRATOR, both matched out of the files' own tags. Today one book is split
|
||||
across `… [Part 1]`, `… [Part 2]` folders three or four deep inside a collection
|
||||
pack, so a scan serves three quarters of a novel; `scripts/media-audiobooks.py`
|
||||
re-files the tree so the property both servers' book scanners assume is true —
|
||||
|
||||
Audiobooks/<Narrator>/<Book> (<year>)/<the file keeps its own name>
|
||||
|
||||
The shared gates, the executor, the journal law and the rollback live in
|
||||
[[Media Relayout]]; this page is the audiobook arm only.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Plex's chapter rows for this root (the DB opens read-only).
|
||||
python3 scripts/media-census.py audiobooks --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Audiobooks \
|
||||
--out /tmp/ab_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-audiobooks.py --root /mnt/entropy/Audiobooks \
|
||||
--enum /tmp/ab_enum.tsv --rows /tmp/ab_rows.tsv --out /tmp/ab_plan.tsv \
|
||||
--exceptions /tmp/ab_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST. Then the same executor, journal and rollback
|
||||
# as [[Media Relayout]] shows, on its own journal, and refresh the Plex section
|
||||
# as in step 5 of the movie Procedure — from inside the container, never over
|
||||
# the host port.
|
||||
scripts/media-apply.py apply --plan /tmp/ab_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-audiobooks.jsonl --apply
|
||||
```
|
||||
|
||||
The group is the identity Plex asserts in its own database — **(narrator, book,
|
||||
year)** — so one book arriving in four part folders becomes one folder, and a
|
||||
1965 and a 2020 recording that share a title stay two books.
|
||||
|
||||
Two laws are forced by what the rows actually hold:
|
||||
|
||||
+- **A chapter is never renumbered.** Most books contain chapters whose index Plex
|
||||
does not know, so a number written into a name would be invented — and it would
|
||||
buy nothing, because both servers order playback by the embedded track tag, not
|
||||
by the file name. A wrong number in a name is the number a human later trusts.
|
||||
Fix order in the tags: a Plex scan re-reads tags and never renames a file.
|
||||
+- **The shelf is the narrator, not the author.** Every book has exactly one artist
|
||||
row and it is the narrator; the author survives only inside some album titles
|
||||
(`Kryptos by Graham McNeill`) and in no column Plex stores, so deriving it would
|
||||
mean parsing titles where a ripper wrote one and inventing it everywhere else.
|
||||
The author stays visible in the folder name Plex itself shows.
|
||||
|
||||
What it refuses, the music arm's list with one addition: a video container, audio
|
||||
no chapter row names, and a loose file in a folder holding two books all park under
|
||||
`_unsorted` with the path they have today — and so does a chapter whose row names
|
||||
**no book or no narrator**: a placeholder (`[Unknown Album]`, `[Unknown Artist]`)
|
||||
that unrelated books share, or a chapter number mistagged as the artist (`0.4`).
|
||||
`Various Artists` is not a placeholder — that is a multi-cast read.
|
||||
|
||||
The reviewed plan lives at `/mnt/entropy/media-relayout-audiobooks-plan.tsv`; it
|
||||
applies on its own journal, behind the music writer, and then refreshes the Plex
|
||||
section. Verify it with `--db-kind plex` — Jellyfin holds no rows here, so a
|
||||
Jellyfin pass proves nothing either way.
|
||||
|
||||
## See Also
|
||||
|
||||
[[Media Relayout]], [[Media Music Relayout]], [[Plex]]
|
||||
@@ -0,0 +1,130 @@
|
||||
---
|
||||
type: reference
|
||||
title: Media Metadata View
|
||||
description: "The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback."
|
||||
tags: [dankserver, media-server, jellyfin, plex, storage]
|
||||
timestamp: 2026-10-30T00:00:00Z
|
||||
---
|
||||
|
||||
# Media Metadata View
|
||||
|
||||
`/var/lib/media-meta/view` is an OverlayFS merge of the media library: the RAID6
|
||||
array below, a directory on the NVMe above. It exists so [[Jellyfin]] can write
|
||||
Kodi-format sidecars (`movie.nfo`, `tvshow.nfo`, `episode.nfo`, artwork) that
|
||||
[[Plex]] can read. Sidecar metadata is only useful sitting next to the media
|
||||
file, and the array is read-only and slow — so the media bytes stay there and the
|
||||
*writes* land on SSD. One tree to both servers, one copy of the media.
|
||||
|
||||
The mount point is not a volume and holds nothing while unmounted. It sits inside
|
||||
`/var/lib/media-meta` so that layers, kernel bookkeeping, the reap trash and the
|
||||
merged door are one directory — and so the mount table grows no new `/mnt` entry
|
||||
next to the array.
|
||||
|
||||
**Not deployed on Dankserver — verified 2026-10-31.** `scripts/media-view.sh
|
||||
status` answers `view: /var/lib/media-meta/view NOT MOUNTED`,
|
||||
`/var/lib/media-meta` does not exist, and the installed quadlets under
|
||||
`~/.config/containers/systemd/` still carry the pre-view binds — `/mnt/entropy:/data:ro`
|
||||
for [[Jellyfin]], `/mnt/entropy:/mnt/entropy:ro` for [[Plex]]. Everything below
|
||||
is the shape git describes, not the shape the servers run. The cutover is an
|
||||
ordinary deploy: `deploy.sh dankserver --service jellyfin|plex` runs
|
||||
`ensure_media_view` (`install` then `verify`) before it restarts the container,
|
||||
so it must not be fired through the middle of a library scan. Until that deploy
|
||||
lands there is no upper layer, so no relayout can leave ghost metadata and
|
||||
`reap` has nothing to do — `media-apply.py` prints that follow-up only when the
|
||||
view really is a mountpoint.
|
||||
|
||||
## Shape
|
||||
|
||||
| Item | Value |
|
||||
|---|---|
|
||||
| Lower layer | `/mnt/entropy` — the RAID6 array, source of truth, never written |
|
||||
| Upper layer | `/var/lib/media-meta/upper` — NVMe; every write lands here |
|
||||
| Work dir | `/var/lib/media-meta/work` — overlayfs internals, same filesystem |
|
||||
| Mount point | `/var/lib/media-meta/view` — the merged tree both servers bind |
|
||||
| Unit (repo) | `dankserver/host/systemd/media-meta-view.mount` — the source of truth |
|
||||
| Unit (host) | `var-lib-media\x2dmeta-view.mount` — systemd names a mount unit after its escaped path (`-` becomes `\x2d`); `media-view.sh install` writes the repo file under that name |
|
||||
| Installer | `scripts/media-view.sh` — `plan` / `install` / `verify` / `status` / `reap` |
|
||||
| Written by | Jellyfin only (Plex binds read-only); nothing else in the fleet touches it |
|
||||
|
||||
Both consumers guard the view with
|
||||
`UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view`, and
|
||||
`scripts/deploy.sh dankserver --service jellyfin|plex` calls `media-view.sh
|
||||
install` + `verify` before it restarts either container.
|
||||
|
||||
| Consumer | Bind | Mode | Why that path |
|
||||
|---|---|---|---|
|
||||
| [[Jellyfin]] | `/var/lib/media-meta/view:/data` | rw | It is the metadata writer; its sidecars become SSD bytes |
|
||||
| [[Plex]] | `/var/lib/media-meta/view:/mnt/entropy` | ro | Every path in the PMS database was recorded as `/mnt/entropy/...`; only the HOST side of the bind moved, so its database still resolves |
|
||||
|
||||
Putting the sidecars inside Plex's reach is not the same as making Plex use them:
|
||||
a library parses `.nfo` only with its *Local media information* agent enabled, and
|
||||
this change touches no library setting.
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
scripts/media-view.sh plan # print the resolved shape, touch nothing
|
||||
sudo scripts/media-view.sh install # create dirs, install the unit, mount, enable
|
||||
scripts/media-view.sh verify # is the view really the overlay? (deploy gate)
|
||||
scripts/media-view.sh status # mount options + upper-layer size/file count
|
||||
scripts/media-view.sh reap # list ghost metadata (media moved, sidecars left)
|
||||
sudo scripts/media-view.sh reap --apply # move ghosts to the SSD trash, never delete
|
||||
```
|
||||
|
||||
`--upper-base DIR` and `--lower DIR` relocate the layers for `plan` and `reap`;
|
||||
the defaults are the host values above.
|
||||
|
||||
## Laws
|
||||
|
||||
- **The upper layer must sit on non-rotational storage.** `install` and `verify`
|
||||
refuse via `lsblk -dno ROTA` if the upper base lives on the array: an overlay
|
||||
whose upper is the spinners is the original problem with extra steps, and it
|
||||
silently turns every metadata write back into a seek.
|
||||
- **Never mount the view over an unmounted lower.** `RequiresMountsFor=/mnt/entropy`
|
||||
plus the container `ConditionPathIsMountPoint` guards: an empty `lowerdir`
|
||||
presents an *empty library* as truth, and a media scan would then clear the
|
||||
database of everything. systemd does not create a mount point, so `install`
|
||||
creates `view` itself; without it the mount simply fails.
|
||||
- **Never delete through the view.** A delete writes an overlayfs whiteout
|
||||
(`.wh.<name>`) in the upper layer, which hides the array's copy without
|
||||
removing it — the file reappears the moment the view is unmounted, and the
|
||||
upper layer now carries a landmine. Deletions belong on the raw path
|
||||
(`/mnt/entropy`), which the relayout tool already treats as the only writer.
|
||||
`reap` refuses to remove whiteouts and only warns about them.
|
||||
- **A relayout must be followed by `reap`.** Renames happen on the raw path, so
|
||||
sidecars the view already wrote stay at the old coordinates: a ghost folder
|
||||
the servers browse and find nothing in. `reap` lists upper-layer directories
|
||||
with no counterpart below and moves them (with their subtree) into a dated
|
||||
`.AGENTTRASH/` under the SSD base. It is recursive and file-safe: a sidecar
|
||||
that exists only above is normal, so only *directories* are candidates.
|
||||
- **File-watcher events do not cross the overlay.** Plex's NFO agent and
|
||||
Jellyfin's real-time monitor see inotify events for writes that land *in the
|
||||
upper layer*, not for changes on the array underneath. After any relayout or
|
||||
externally-written metadata, trigger a library refresh — see
|
||||
[[Media Relayout]] for the refresh procedure.
|
||||
|
||||
## Failure signatures
|
||||
|
||||
| Symptom | Meaning |
|
||||
|---|---|
|
||||
| `jellyfin.service` / `plex.service` show `ConditionPathIsMountPoint=... was not met` and stay inactive | The view is not mounted. Do not start them anyway: `scripts/media-view.sh install`, then `verify`. |
|
||||
| Jellyfin log: `Read-only file system : '/data/Movies/…/movie.nfo'` | The container is binding the raw array, not the view — the quadlet drifted back or the view is gone. `media-view.sh status`. |
|
||||
| A folder appears in a media server with artwork/`.nfo` but no video | Ghost metadata from a relayout. `media-view.sh reap`. |
|
||||
| `verify` prints `refused: …/view is not a mountpoint` | A plain directory is sitting where the overlay belongs. That is the fail-closed guard working; `install` on the box. |
|
||||
| `systemctl status` finds no such unit | Look under the escaped name, `var-lib-media\x2dmeta-view.mount`, not the repo file name. |
|
||||
|
||||
## Rollback
|
||||
|
||||
Revert the commit that pointed the two quadlets at the view and re-deploy both
|
||||
services — they bind `/mnt/entropy` read-only again, which is the pre-view shape.
|
||||
The NVMe upper layer can stay: nothing reads it once the binds are back, and it
|
||||
holds no state either server needs. Unmounting alone
|
||||
(`systemctl stop 'var-lib-media\x2dmeta-view.mount'`) is safe while no container
|
||||
binds it, and the array is never modified by any of this.
|
||||
|
||||
## Links
|
||||
|
||||
- [[Jellyfin]] — the metadata writer; its data layout section.
|
||||
- [[Plex]] — the read-only consumer; in-place bind-mount history.
|
||||
- [[Media Relayout]] — the tool that moves media on the raw path and the reason `reap` exists.
|
||||
- [[Entropy Volume]] — the array underneath.
|
||||
@@ -0,0 +1,68 @@
|
||||
---
|
||||
type: reference
|
||||
title: Media Music Relayout
|
||||
description: "Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals."
|
||||
tags: [dankserver, media-server, jellyfin, media-relayout]
|
||||
timestamp: 2026-10-31T00:00:00Z
|
||||
---
|
||||
|
||||
# Media Music Relayout
|
||||
|
||||
`/mnt/entropy/Music` is rips and discography packs, and Jellyfin has matched and
|
||||
tagged almost all of it. `scripts/media-music.py` re-files it into the shape
|
||||
Jellyfin's music scanner reads —
|
||||
|
||||
Music/<album artist>/<Album> (<year>)/NN - <Title>.ext
|
||||
|
||||
— and the authority is the tag row of the track, never the name the ripper gave
|
||||
the file. The shared gates, the executor, the journal law and the rollback live in
|
||||
[[Media Relayout]]; this page is the music arm only.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Jellyfin's audio rows, one per track, with the album and
|
||||
# artist ids its own database carries (the DB opens read-only).
|
||||
python3 scripts/media-census.py music --root /mnt/entropy/Music \
|
||||
--out /tmp/music_rows.tsv
|
||||
# 2. What is on disk (stat only).
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/Music \
|
||||
--out /tmp/music_enum.tsv
|
||||
# 3. The plan.
|
||||
python3 scripts/media-music.py --root /mnt/entropy/Music --enum /tmp/music_enum.tsv \
|
||||
--rows /tmp/music_rows.tsv --out /tmp/music_plan.tsv \
|
||||
--exceptions /tmp/music_exceptions.txt
|
||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Then the same
|
||||
# executor, journal and rollback as [[Media Relayout]] shows, on its own journal.
|
||||
scripts/media-apply.py apply --plan /tmp/music_plan.tsv --root /mnt/entropy \
|
||||
--journal /mnt/entropy/media-relayout-music.jsonl --apply
|
||||
```
|
||||
|
||||
The group is what the tags assert, not where the files sit: **(album-artist tag,
|
||||
album, year)**. One album arrives as three disc folders and as two copies of one
|
||||
rip, and one MusicBrainz release group is one album. A compilation is one album
|
||||
whose tracks name many performers, so it files once under `Various Artists`:
|
||||
keying on the performer would file one album once per artist. Two different
|
||||
album-artist tags on one album name are two albums, which is what Jellyfin's two
|
||||
album items say.
|
||||
|
||||
What it refuses:
|
||||
|
||||
+- **A sidecar is placed by the audio around it, never by its own name.** A
|
||||
`.lrc`, `.txt` or picture that shares a track's stem follows that track and
|
||||
takes the name the track now has, or Jellyfin cannot find it; anything else
|
||||
goes to the album its own folder was placed in, or to the folder whose discs
|
||||
sit one level below it. A folder whose audio belongs to two albums parks its
|
||||
sidecar rather than guessing.
|
||||
+- **An image album keeps its name.** A folder holding one audio file and a
|
||||
`.cue` is a whole album in one file; numbering it `01 - …` tells Jellyfin to
|
||||
index it as one track.
|
||||
+- **A video container parks**, because Jellyfin will not index it as music and a
|
||||
rename to `.m4a` would claim it is; **audio no provider row names parks** too,
|
||||
because unmatched audio is a tagging job, not a renaming job.
|
||||
|
||||
ONE writer at a time on this array: two rename passes race, and neither journal
|
||||
tells. Verify the run against Jellyfin's own tables — see *Proving the move
|
||||
landed* in [[Media Relayout]].
|
||||
|
||||
## See Also
|
||||
|
||||
[[Media Relayout]], [[Jellyfin]], [[Media Audiobook Relayout]]
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
type: reference
|
||||
title: Media Relayout
|
||||
description: "Census, plan and apply the movie library into Jellyfin format: the gates to read, and the rollback."
|
||||
description: "Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback."
|
||||
tags: [dankserver, media-server, jellyfin, plex, procedure]
|
||||
timestamp: 2026-10-30T00:00:00Z
|
||||
---
|
||||
@@ -12,17 +12,24 @@ The movie library on [[Entropy Volume]] arrived as release folders —
|
||||
`Some.Movie.2019.1080p.WiKiTaLx.x265-FOO/` with the video inside named after
|
||||
the release. [[Jellyfin]] matches a movie by folder and file name, so the
|
||||
release noise cost posters, extras and versions. These scripts turn that
|
||||
library into Jellyfin's own shape — and move TV back out of it — and nothing
|
||||
else in the repo touches a media file.
|
||||
library into the shape its media server reads — the movies, the TV packs, the TV
|
||||
that turns up in the movie share, the music library, and the audiobooks, whose
|
||||
only reader is Plex — and nothing else in the repo touches a media file.
|
||||
|
||||
| Script | Touches media? | Output |
|
||||
|---|---|---|
|
||||
| `scripts/media-census.py` | reads headers only | `enum`, `probed`, `plex` TSVs |
|
||||
| `scripts/media-census.py` | reads headers and the servers' own DBs, never writes | `enum`, `probed`, and one authority TSV per library |
|
||||
| `scripts/media-plan.py` | no — TSVs in, TSVs out | the plan + the exception report |
|
||||
| `scripts/media-apply.py` | **renames, never copies**; sweeps only what a run emptied | the moves, plus a journal |
|
||||
| `scripts/media-tv.py` | no — TSV out | a plan that moves TV into the TV library |
|
||||
| `scripts/media-shows.py` | no — TSV out | a plan that re-files the TV library itself |
|
||||
| `scripts/media-music.py` | no — TSV out | a plan that re-files the music library by its tags |
|
||||
| `scripts/media-audiobooks.py` | no — TSV out | a plan that re-files the audiobooks by the book Plex matched |
|
||||
| `scripts/media-verify.py` | no — reads the servers' DBs read-only | the verdict that every journaled move landed |
|
||||
|
||||
The seam is `tests/media_relayout.bats`; the target contract is quoted from
|
||||
The seams are `tests/media_relayout.bats`, `tests/media_shows.bats`,
|
||||
`tests/media_music.bats`, `tests/media_audiobooks.bats` and `tests/media_verify.bats`;
|
||||
the movie and TV target contracts are quoted from
|
||||
[jellyfin.org/docs/general/server/media/movies](https://jellyfin.org/docs/general/server/media/movies)
|
||||
in `media-plan.py`'s docstring.
|
||||
|
||||
@@ -40,8 +47,10 @@ $C plex --root /mnt/entropy/Movies --out /tmp/plex-movies.tsv
|
||||
```
|
||||
|
||||
`plex` reads the Plex metadata cache (`--db` defaults to the path the
|
||||
[[Plex]] container bind-mounts at `/var/lib/plexmediaserver`) with
|
||||
`mode=ro&immutable=1`: a live server is never locked or written.
|
||||
[[Plex]] container bind-mounts at `/var/lib/plexmediaserver`) with plain
|
||||
`mode=ro`: a live server is never locked or written, and plain ro is also what
|
||||
makes a scan's uncheckpointed rows visible. The same rule as
|
||||
`scripts/media-verify.py` — see *Proving the move landed*.
|
||||
|
||||
Then plan — this reads only the three TSVs, so it costs seconds and can run
|
||||
anywhere, including a laptop:
|
||||
@@ -72,37 +81,55 @@ An apply also leaves the release folders it emptied standing where they were —
|
||||
hundreds of them at the library root — which is what `sweep` removes, dry-run
|
||||
first, from the same journal.
|
||||
|
||||
After it lands, both servers need a scan or the change is invisible.
|
||||
After it lands, both servers need a scan or the change is invisible. [[Plex]] and
|
||||
[[Jellyfin]] own the exact requests; two laws hold at this step: `204` is
|
||||
acceptance and anything else queued nothing — re-issue the identical request
|
||||
rather than diagnosing — and ONE accepted refresh is the whole procedure, because
|
||||
a second while a scan runs moves `Scan Media Library` to `Cancelling` and every
|
||||
later `204` is accepted and then discarded.
|
||||
|
||||
### Proving the move landed
|
||||
|
||||
A scan answers `204` whether or not it finds anything, so proof is the journal
|
||||
replayed against both servers' own tables: every `move` row has a row at its
|
||||
`dst`, and **zero** rows still name its `src`. `scripts/media-verify.py` is that
|
||||
replay: it opens each database read-only (there is no `sqlite3` binary on these
|
||||
hosts or inside either container) and exits non-zero while a row is stale.
|
||||
|
||||
Both libraries live in WAL mode, so the tool opens them plain `mode=ro`:
|
||||
`immutable=1` promises sqlite the file never changes, which skips the
|
||||
write-ahead log — a scanner's own uncheckpointed rows then read as missing, and
|
||||
a read beside a running scan can fail as "database disk image is malformed".
|
||||
`immutable` stays only the fallback for a database whose `-shm` cannot be read.
|
||||
|
||||
```bash
|
||||
# Run this from the host shell through `podman exec`, never from a neighbour
|
||||
# container: Plex's HTTPD closes without a single byte on any request that
|
||||
# arrives from another container's netns (10.89.x.x reads as "empty reply", not
|
||||
# as a 401). The token rides stdin, never argv, where any process on the host
|
||||
# could read it out of /proc/<pid>/cmdline.
|
||||
umask 077
|
||||
podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \
|
||||
'/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok
|
||||
podman exec -i plex sh -c 'read -r T
|
||||
for k in $(curl -s -H "X-Plex-Token: $T" -H "Accept: application/json" \
|
||||
http://127.0.0.1:32400/library/sections | grep -o "\"key\":\"[0-9]*\"" | cut -d\" -f4); do
|
||||
curl -s -o /dev/null -w "plex section $k: %{http_code}\n" -X POST \
|
||||
-H "X-Plex-Token: $T" "http://127.0.0.1:32400/library/sections/$k/refresh"
|
||||
done' </tmp/px.tok
|
||||
scripts/media-verify.py --journal /mnt/entropy/media-relayout-music.jsonl \
|
||||
--db-kind jellyfin --ignore-ext jpg,jpeg,png,log,cue,toc,nfo,txt,m3u,accurip
|
||||
```
|
||||
|
||||
[[Jellyfin]] has no credential on this box that can trigger a scan: the two keys
|
||||
in its database are Seerr's, unused since 2026-09-01, and answer 401 to every
|
||||
call. Mint one (*Dashboard → Advanced → API Keys*, Administrator) and store it in
|
||||
`~/.config/secrets/jellyfin`, or press *Library → Scan All Libraries* in the web
|
||||
UI. A scripted scan passes the key in a mode-600 curl config, never in a header
|
||||
argument:
|
||||
"No row at the destination" is not one verdict but four. **never indexed**: the
|
||||
server indexes no file of that extension (its own extension census says so) — a
|
||||
sidecar, not a lost film. **gone**: the file sits at neither src nor dst, because
|
||||
a later pass or the server's own extension sweep renamed it. **phantom**: a row
|
||||
still naming src while the file has left src. **REAL**: no row at dst *and* the
|
||||
file still at src — the only class that is this relayout's own damage. Phantoms
|
||||
are normal for both servers (Plex carries thousands that predate any relayout);
|
||||
re-scan to clear them and never read them as loss.
|
||||
|
||||
```bash
|
||||
umask 077; printf 'header = "X-Emby-Token: %s"\n' "$(cat ~/.config/secrets/jellyfin)" >/tmp/jf.hdr
|
||||
curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n" \
|
||||
-X POST http://127.0.0.1:8096/Library/Refresh
|
||||
```
|
||||
`media-verify.py` prints this split itself — `on disk at dst, no row names it`,
|
||||
`REAL — file still at src`, `gone — at neither path` — so nobody hand-stats a
|
||||
thousand paths to find out whether a film is lost or merely unindexed.
|
||||
|
||||
The owning server decides whether a pass can convict anything: Plex's roots are
|
||||
its `sections` rows joined to their location rows (`Movies`, `TVShows`, `Music`,
|
||||
`Audiobooks`, `LiveRecording`) and Jellyfin holds no audiobook library at all, so
|
||||
the audiobook journal verifies with `--db-kind plex` or it proves nothing.
|
||||
|
||||
Three doors that look right and are not: `GET /search?query=` matches metadata
|
||||
titles, never file paths; `?path=` on `/library/sections/N/all` makes Plex's
|
||||
HTTPD close without a single byte; and the same HTTPD drops requests that carry
|
||||
no `Accept` and client-identifier header (see the scan block above for the form
|
||||
that works).
|
||||
|
||||
## TV in the movie library
|
||||
|
||||
@@ -133,13 +160,187 @@ from the box-set folder. What it refuses it prints and leaves alone:
|
||||
- a destination two files both claim: the executor refuses the whole plan
|
||||
before its first rename, and only Frank decides which copy is the keeper.
|
||||
- an episode the destination library already holds at the same byte count. The
|
||||
movie share carried a second copy of all 124 Monk episodes — the TV library
|
||||
has had the same box set for years, every episode the same size, ~31 GiB of
|
||||
it. The mover leaves such a series in place and says so; a duplicate is
|
||||
deleted or kept by Frank, never copied into one library by a tool.
|
||||
movie share once carried a second copy of a whole box set — same episode list,
|
||||
every byte count identical. The mover leaves such a series in place and says
|
||||
so; a duplicate is deleted or kept by Frank, never copied into one library by
|
||||
a tool.
|
||||
|
||||
A sidecar that shares an episode's own name (`.srt`, `.nfo`) moves with it.
|
||||
|
||||
Below the threshold the mover prints and leaves, so a lone downloaded episode of
|
||||
a real show stays in the movie library on purpose.
|
||||
Filing one by hand is the same executor on a hand-written plan and the same
|
||||
journal; the trap is the series year. Read it from the file's own name or the
|
||||
folder it owns, never from the title whose bonus folder it happens to be parked
|
||||
in — `The Cleveland Show S03E07` inside `Die Hard (1988)/extras/` is not a 1988
|
||||
show. A name whose marker is `Part 2` is a film and stays put.
|
||||
|
||||
## The TV library's own relayout
|
||||
|
||||
`/mnt/entropy/TVShows` is scene packs rather than a library, and Jellyfin has
|
||||
already matched and numbered most of it. `scripts/media-shows.py` re-files that
|
||||
library into the shape Jellyfin reads, with **tvdb as the name authority**:
|
||||
|
||||
<Series> (<year>) [tvdbid-NNNN]/Season NN/
|
||||
<Series> (<year>) - SxxEyy - <Episode Title>.ext
|
||||
|
||||
It never guesses: every series name, episode number and title it writes traces to
|
||||
an authority, and what has none is parked for a human to read.
|
||||
|
||||
```bash
|
||||
# 1. The authority: Jellyfin's own matched rows — each carries the series' TMDb
|
||||
# and Tvdb ids and the title the row holds (the DB opens read-only).
|
||||
python3 scripts/media-census.py jellyfin --root /mnt/entropy/TVShows \
|
||||
--out /tmp/tv_titles.tsv
|
||||
# 2. What is on disk (stat only), and the runtimes the film arm needs.
|
||||
python3 scripts/media-census.py enumerate --root /mnt/entropy/TVShows \
|
||||
--out /tmp/tv_enum.tsv
|
||||
python3 scripts/media-census.py probe --root /mnt/entropy/TVShows \
|
||||
--enum /tmp/tv_enum.tsv --jobs 6 --out /tmp/tv_probed.tsv
|
||||
# The probe is the slow step: ffprobe reads both ends of every file. So re-use
|
||||
# the last one — a duration belongs to the file, not to its name: re-key the
|
||||
# previous census through that pass's own journal (`src` -> `dst`) instead of
|
||||
# seeking the array again. It is also the only column the film arm reads, and
|
||||
# only for a file with no marker and no provider row (4,794 of 58,551 today,
|
||||
# all of them covered by a re-key that costs seconds, not an hour).
|
||||
# 3. The plan. --tmdb needs TMDB_API_KEY — the v3 key in ~/.config/secrets/tmdb,
|
||||
# which travels as the `api_key` query parameter. Only a v4 request token is
|
||||
# accepted as a Bearer header, and the wrong shape answers 401 on EVERY call,
|
||||
# which the summary line flags as NOT provider-complete. --tvdb reads its
|
||||
# `api_key=` line from ~/.config/secrets/tvdb, never argv. Both write the SAME
|
||||
# cache file, so the plan is reproducible offline afterwards: run it once to
|
||||
# fill the cache, and quote the second run.
|
||||
# Paths are only strings to a dry run, so this step can run on the workstation
|
||||
# against scp'd TSVs — which is where the two provider secrets live.
|
||||
python3 scripts/media-shows.py --root /mnt/entropy/TVShows --enum /tmp/tv_enum.tsv \
|
||||
--titles /tmp/tv_titles.tsv --probed /tmp/tv_probed.tsv \
|
||||
--tmdb-cache /mnt/entropy/media-tv-names.json --tmdb --tvdb \
|
||||
--park /mnt/entropy/media-park/TVShows \
|
||||
--out /mnt/entropy/media-relayout-shows2-plan.tsv \
|
||||
--exceptions /mnt/entropy/media-relayout-shows2-exceptions.txt
|
||||
# 4. Read the exceptions file FIRST — it is the review, not a log. Same executor
|
||||
# as above, but on this arm's OWN journal, NAMED PER PASS: the journal of an
|
||||
# applied pass is its unwind record, so a second pass that reuses the name
|
||||
# destroys the only map back.
|
||||
scripts/media-apply.py apply --plan /mnt/entropy/media-relayout-shows2-plan.tsv \
|
||||
--root /mnt/entropy --journal /mnt/entropy/media-relayout-shows2.jsonl --apply
|
||||
```
|
||||
|
||||
Evidence, strongest first — the order the planner believes things:
|
||||
|
||||
- the **episode marker the file's own name states** (`S01E02-E03`, `1x04`); a digit
|
||||
run longer than three at that position is NOT a number — `S03E2019` and
|
||||
`S04E47370355` are one box set's airdate fragment and pack index — so such a file
|
||||
is renumbered by tvdb's title or parks, and is never truncated down to fit;
|
||||
- the **`Season NN` folder it sits in and the number its name leads with inside
|
||||
it**: `Season 19/01 Charles Barkley, Nirvana.avi` is S19E01;
|
||||
- the **provider row**: which episode the file IS, and the numbering of a file
|
||||
whose own name says nothing;
|
||||
- the **row's tvdb id, else its TMDb id carried through TMDb's external ids, else
|
||||
an exact tvdb name search**: that id names the SERIES FOLDER and supplies the
|
||||
episode title, because a name the server read off a folder is not a show name.
|
||||
The folder CARRIES the id: `[tvdbid-NNNN]` is what Jellyfin's Tvdb agent and
|
||||
Plex's scanner both read, so a show whose title is ambiguous resolves by number
|
||||
instead of by a human guessing which folder is which. The id rides the folder
|
||||
only — a file name gets 180 characters and the episode title needs the room.
|
||||
- a **bonus folder** anywhere above the file, or Jellyfin's `ExtraType` → the
|
||||
series' canonical extra folder;
|
||||
- a runtime over `SHORT_MAX` makes it film-shaped, and it leaves for `--movies` only
|
||||
when the movie provider's title is a run of words the FILE itself states and its
|
||||
year agrees with the file's: the query is only what `title_of` salvaged from a
|
||||
release name, and the first hit for a short one is a near miss — a 1963 Bond film
|
||||
for the word "russia", *Special Edition I* for a file saying *Edition III*. Else it
|
||||
parks.
|
||||
|
||||
Four refusals carry the design: what the planner will NOT do is the point of it.
|
||||
|
||||
- **Folders resolve a series; they never invent one.** A folder's words may
|
||||
resolve a file to a series a provider has already named — `(CBB) Tengen Toppa
|
||||
Gurren Lagann - The Complete Series (BD 1080p)` names the registered *Tengen
|
||||
Toppa Gurren Lagann* — but a bonus subfolder is not a show. An early run made
|
||||
`…/03. Extra/01. NCOP/NCOP 01.mkv` into a root series called `01 NCOP`.
|
||||
- **A scraped row does not outrank the filesystem.** Jellyfin numbers
|
||||
multi-season packs badly: every file under `SNL 16-20/Season 16..19` carries
|
||||
season 16 episode 20, and one Gundam row says season 2004. The folder, then
|
||||
the file's own leading number, win.
|
||||
- **A season that is a YEAR is the date matcher, not a season.** Jellyfin files
|
||||
specials by airdate, so a row can arrive saying `season 2002`; that is not a
|
||||
season and it never becomes a folder. The one authority on such a row is tvdb's
|
||||
episode list read backwards: the title the row carries names the (season,
|
||||
episode) tvdb gives it. A title tvdb gives to two episodes names nothing, and
|
||||
the file parks with the reason it always carried. `Season 00` IS a legal
|
||||
season — specials live there — so a row that really says 0 files there instead
|
||||
of parking.
|
||||
- **A season tvdb's own list contradicts gets no tvdb title.** An anime pack
|
||||
numbered by its release group books `S05E01` in a season tvdb does not hold, or
|
||||
books more episodes there than tvdb lists: the slot its number points at is a
|
||||
different episode, and dressing it in that episode's title is a confident lie.
|
||||
The file still renames to the standard shape; the title does not ride, and the
|
||||
series is listed under *seasons whose numbering tvdb contradicts* in the
|
||||
exceptions. A season holding FEWER files than tvdb lists stays plausible — a
|
||||
partial pack is the normal shape. Neither reading holds for a list tvdb
|
||||
cut off at its own page limit: the default episode query answers at most
|
||||
500 rows and states no total, and One Piece, Crayon Shin-chan and
|
||||
Detective Conan each answer exactly 500 while holding over a thousand —
|
||||
a page bounds nothing above itself, so it contradicts nothing.
|
||||
|
||||
**A provider name is evidence only when an id backs it.** Jellyfin names a Series
|
||||
after the folder it scanned, so a library filed by release folders ends up with
|
||||
one Series item per box set — `24 (2001) Season 1`, `Modern Family S05` — and
|
||||
copying those names verbatim launders folder junk into the contract. The planner
|
||||
resolves each series to its tvdb id and writes tvdb's name and year; the merged
|
||||
identity is also what turns a box set split across five release folders into ONE
|
||||
folder, and what makes two files claiming one episode MEET in the duplicate rule.
|
||||
A series no id resolves keeps the provider's words and is listed under *no tvdb
|
||||
identity* in the exceptions — that list, not a rename, is the review.
|
||||
|
||||
**The review pile lives outside the library.** Files parked inside
|
||||
`/mnt/entropy/TVShows` were indexed as one show named `_unsorted`, and that
|
||||
show's rows then looked like evidence. So a row whose series is the hidden
|
||||
top-level folder the file itself sits in is dropped — and the folder is matched
|
||||
with its leading underscore stripped, because the server *displays* `_unsorted` as
|
||||
`unsorted`: compare the spelled names and the pile survives the relayout as a
|
||||
visible series folder that real files get renamed into. `--park` outside the root
|
||||
makes the next plan re-file the pile instead of skipping it. Inside a hidden
|
||||
top-level folder a file's own words may MATCH a registered show, never invent one.
|
||||
|
||||
**The operator's id map is the door for a name tvdb cannot be searched by.**
|
||||
`scripts/media-shows.py` reads `scripts/media-shows-tvdb-ids.tsv` by default: one
|
||||
line per pack folder, `folder <TAB> tvdb id <TAB> note`. The file exists because
|
||||
tvdb titles a record in the show's own language and the English name rides in
|
||||
*aliases* the name search never reads: `ブラッククローバー` (331753) answers to no
|
||||
spelling of "Black Clover", and the bare alias `One Piece` belongs to 392276, the
|
||||
2023 live action, not to the 1999 record (81797) the pack here is. The map
|
||||
outranks a provider id — a box set's provider id is the server's own guess at a
|
||||
folder name — and it merges the release folders that are one show. Everything it
|
||||
touched is written to the exceptions: every folder it named, every row that
|
||||
matched no folder, every disagreement with a provider id, every id tvdb never
|
||||
answered. A line whose id is not a number stops the plan, because a typo renames
|
||||
a whole pack to the wrong show. Only folders the search CANNOT reach belong in
|
||||
it; a row for a folder the search would resolve hides a live lookup behind a
|
||||
number nobody re-checks.
|
||||
|
||||
`scripts/media-shows-tvdb.py` makes the sheet the map is built from. It reads a
|
||||
plan's `mkdir` rows, asks tvdb about every destination folder carrying no
|
||||
`[tvdbid-`, and writes one row per folder with the verdict that says how much was
|
||||
guessed to get there: `EXACT` (tvdb's own spelling), `PUNCT` (equal only after
|
||||
release junk is stripped from the folder's words), `NEAR` (tvdb answered
|
||||
something else — the best hit is printed for a human and nothing applies it),
|
||||
`NONE`. Acceptance never moved across the passes that built this: a hit counts
|
||||
only when tvdb's name equals the query with every non-alphanumeric dropped from
|
||||
both sides, and the record's year still has to fall inside any year span the
|
||||
folder names. That gate is what caught three confident picks — Fairy Tail's 2024
|
||||
sequel (410031), the 2026 Hokuto no Ken remake (465216) and `Mugyutto! Black
|
||||
Clover` (378762) offered for Black Clover. It is why the map is approved by hand
|
||||
and the sheet is never fed to the planner.
|
||||
|
||||
## The music and audiobook arms
|
||||
|
||||
Each is a page of its own, because each reads a different authority and refuses
|
||||
for a different reason: [[Media Music Relayout]] files by Jellyfin's track tags,
|
||||
[[Media Audiobook Relayout]] by Plex's chapter rows. Both share this page's
|
||||
executor, the one-journal-per-arm law, the sweep and the rollback.
|
||||
|
||||
## Laws
|
||||
|
||||
- **A moved file is a 404 in [[Plex]] until its section is rescanned.** The
|
||||
@@ -166,6 +367,19 @@ A sidecar that shares an episode's own name (`.srt`, `.nfo`) moves with it.
|
||||
folder holding it, and extras drifted into a neighbour. A bare number counts
|
||||
only when it could be a release year, so `1080` and `2160` never pose as
|
||||
agreement between two names.
|
||||
- **Compare two media libraries with the series engine, never a path
|
||||
substring.** Filtering paths for `monk` also returns *Monkey Dust*, and
|
||||
letting whichever file a walk happens to reach first stand for an episode
|
||||
paired two libraries into a fake duplicate — the same sloppiness pointed the
|
||||
other way deletes a file that is unique. The comparison is `media-titles.ep_of`
|
||||
on both sides: series, season, episode, bytes, and a hash before anything is
|
||||
trashed.
|
||||
- **A deletion is staged on the item's own filesystem.** The trash rule names
|
||||
`~/.AGENTTRASH`, but `/home` and `/mnt/entropy` are different devices: a 31 GiB
|
||||
stage there is a copy rather than a rename, it has to fit in the 187 GiB of
|
||||
free space, and `rename(2)` refuses it anyway. Trash goes beside the item —
|
||||
`/mnt/entropy/.AGENTTRASH/<date>-<topic>/` — so it costs one inode change and
|
||||
rolls back exactly like the move that made it.
|
||||
- **A file whose name carries no words is bonus material, not a folder.**
|
||||
`.Extra 1 ( 1H 27min multisub ).mkv` parses to `1`; a movie folder named `1`
|
||||
is not a movie. It becomes a review line telling a human to name the file.
|
||||
@@ -194,16 +408,15 @@ A sidecar that shares an episode's own name (`.srt`, `.nfo`) moves with it.
|
||||
recreates the folder, so the sweep is undone by the same command as the moves.
|
||||
Proof carries downward: inside a folder the journal proves emptied, every
|
||||
empty directory goes with it. A release's `Screens/` or `Sample/` is never a
|
||||
move source itself, so 146 of them held their emptied parents on disk after
|
||||
the first sweep.
|
||||
move source itself, so it can hold an emptied parent until the next sweep.
|
||||
A failed `rmdir` is not a removal: only a folder that is really gone leaves
|
||||
scope, so the next sweep finishes what a permission problem stopped.
|
||||
- **An episode-shaped name marks a SERIES, not a film — and the series is the
|
||||
name BEFORE the marker.** `Monk - S05E03 - Mr. Monk and the Big Game` is series
|
||||
`Monk`. Keyed on the whole title instead, every episode became a group of one
|
||||
and never reached the five-episode size that calls something a series, so a
|
||||
box set was filed as 124 movies in 124 folders. The threshold still earns its
|
||||
keep: a lone film named `Part 2` stays a film. Episodes of a series are parked
|
||||
`Monk`. Keyed on the whole title instead, every episode became a group of one,
|
||||
never reaching the five-episode size that calls something a series — a whole
|
||||
box set filed as one folder per episode. The threshold still earns its keep:
|
||||
a lone film named `Part 2` stays a film. Episodes of a series are parked
|
||||
in `_TV needs a TV library - <series>` for a human to move to a TV source —
|
||||
the movie tool never files TV.
|
||||
- **One journal is one undo.** The TV move runs on its own journal, and with
|
||||
@@ -211,6 +424,33 @@ A sidecar that shares an episode's own name (`.srt`, `.nfo`) moves with it.
|
||||
path outside `--root`, and a journal shared with the movie relayout would
|
||||
undo both runs when only one of them is wanted.
|
||||
|
||||
- **Sidecar metadata does not follow a rename, because it lives on another
|
||||
filesystem.** The media servers write `.nfo` and artwork through the
|
||||
[[Media Metadata View]] (NVMe upper layer); this tool moves bytes on the raw
|
||||
array path, so where that view is mounted a sweep or a move must be followed by
|
||||
`scripts/media-view.sh reap --apply` — stranded sidecars become ghost folders
|
||||
that both servers browse and report as content. Whether this host mounts the
|
||||
view at all is stated on that page; `media-apply.py` stays silent when it does not.
|
||||
- **Nothing deletes through the metadata view.** A delete there writes an overlayfs
|
||||
whiteout in the upper layer, which hides the array's copy without removing it;
|
||||
the file returns when the view is unmounted. Deletions in this pipeline happen
|
||||
on the raw path only, which is what the journal can undo.
|
||||
- **The music album key is case-folded; the folder name is not.** `Master Of
|
||||
Puppets` and `Master of Puppets` are one album to the matcher, so the spelling
|
||||
that reaches the folder is the one most tracks carry. Taking it from whichever
|
||||
row sorted first made the folder name depend on row order, and re-planning an
|
||||
already-filed library renamed albums that were already in place.
|
||||
- **An occupied path is never a destination**, not even when its occupant is
|
||||
scheduled to leave: the executor refuses an occupied target, so a plan that
|
||||
aims at one fails after the renames that were supposed to clear it. A file
|
||||
standing where the contract puts it is a keep, and a file already carrying a
|
||||
collision suffix keeps that suffix — otherwise the plan is a shuffle, not a
|
||||
fixed point, and a re-run moves albums that are already right.
|
||||
- **`media-census.py music` reads the `BaseItemProviders` table, not the item's
|
||||
JSON.** This Jellyfin build carries an empty `ProviderIds` inside
|
||||
`BaseItems.Data` for albums, so a census that parses the JSON finds no
|
||||
MusicBrainz or Spotify id on any album and reads as "the library is untagged".
|
||||
|
||||
## Rollback
|
||||
|
||||
One command, from the journal of the run:
|
||||
@@ -219,9 +459,17 @@ any folder a sweep removed, restores every source path, removes only the folders
|
||||
that run created *and that are now empty*, and refuses everything if any moved
|
||||
file cannot be accounted for. A rolled-back journal refuses to roll back twice.
|
||||
|
||||
**Roll a chain back newest journal first, or the tool refuses.** Each pass
|
||||
re-files what an earlier one moved, so a journal is a clean inverse only until
|
||||
something downstream touches its files; `ls -l /mnt/entropy/media-relayout*.jsonl`
|
||||
is the order. Refusing is the tool working: on 2026-10-31 the movie journal could
|
||||
not restore files the later TV passes had re-filed, and the stray journal could
|
||||
not restore files Jellyfin's extension sweep had renamed. Roll back what still
|
||||
inverts and leave the rest as the current state — never force it.
|
||||
|
||||
## See Also
|
||||
|
||||
[[Jellyfin]], [[Plex]], [[Entropy Volume]], [[Testing Policy]], [[Storage Policy]]
|
||||
[[Jellyfin]], [[Plex]], [[Media Metadata View]], [[Entropy Volume]], [[Testing Policy]], [[Storage Policy]]
|
||||
|
||||
## Citations
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ description: "Plex Media Server is a **containerized media server on Dankserver*
|
||||
tags:
|
||||
- dankserver
|
||||
- plex
|
||||
timestamp: 2026-10-21
|
||||
timestamp: 2026-10-30
|
||||
---
|
||||
|
||||
# Plex
|
||||
@@ -13,7 +13,8 @@ Plex Media Server is a **containerized media server on Dankserver**: a podman
|
||||
quadlet (`plex.service`) under the systemd **user** generator, generated from
|
||||
`dankserver/plex/quadlet/plex.container`. It shares the host with a second live
|
||||
media server, [[Jellyfin]] (`jellyfin.service`, listening on 8096); both serve
|
||||
from the same read-only Entropy Volume mount.
|
||||
from the same tree — the [[Media Metadata View]] overlay — so the Kodi-format
|
||||
sidecars Jellyfin writes are the files Plex reads, with no copy and no sync.
|
||||
|
||||
**Host:** Dankserver — **Ports:** 32400/tcp (PMS web + API), 32400 is the
|
||||
only port PMS needs for client streaming; LAN clients connect directly,
|
||||
@@ -26,7 +27,7 @@ public traffic can be fronted by Caddy like every other vhost.
|
||||
| Runtime | podman quadlet, `plex.service` (user manager) — migrated from the native `plexmediaserver.service` OS-package unit (2026-09) |
|
||||
| Image | `lscr.io/linuxserver/plex:1.43.5.11029-6bf8af3f0-ls329` (pinned; verified == the registry's `latest` manifest digest at pin time: `sha256:8f655bad…e9e2334`) |
|
||||
| Data root | `/var/lib/plexmediaserver` bind-mounted **in place** as `/config` (RW) — the migration moved zero bytes |
|
||||
| Media | `/mnt/entropy` mounted at the same absolute path **read-only** — every library path in the PMS database stayed valid, zero re-scan |
|
||||
| Media | git: `/var/lib/media-meta/view` (the [[Media Metadata View]] overlay) bound at the same absolute path `/mnt/entropy` **read-only**; live today: `/mnt/entropy` itself, read-only — the view has never been installed. Either way every library path in the PMS database stays valid, zero re-scan |
|
||||
| Identity | fleet rootless model — container root maps to the `dank` user (1000:1000); the deployer chowns the data root on first cutover |
|
||||
| Port | `32400/tcp` published on `0.0.0.0` — zero client reconfiguration |
|
||||
| Network | `dankserver_appnet`, alias `plex`; Tautulli reaches it at `http://plex:32400` |
|
||||
@@ -38,7 +39,9 @@ public traffic can be fronted by Caddy like every other vhost.
|
||||
|
||||
The migration moved zero bytes: the container bind-mounts the native data
|
||||
root where it already lived, and the media archive re-mounts at
|
||||
`/mnt/entropy` inside the container, so every absolute library path in the
|
||||
`/mnt/entropy` inside the container — only the HOST side of that bind has moved
|
||||
(the array today; the overlay view in git has never been deployed), so every
|
||||
absolute library path in the
|
||||
PMS database keeps resolving. The one metadata change: rootless podman
|
||||
cannot map the real host uid 997, so the fleet model applies — container
|
||||
root runs as the `dank` user, and the deployer adopts the data root with
|
||||
@@ -84,3 +87,45 @@ root runs as the `dank` user, and the deployer adopts the data root with
|
||||
while the HTTP layer finishes loading; the first boot of a years-old
|
||||
library took ~10 minutes (2026-09 cutover). The deployer gate now allows
|
||||
600 s before failing; later boots are much faster (warm page cache).
|
||||
|
||||
## Triggering a scan
|
||||
|
||||
A rename on disk is invisible until the section is rescanned, and a moved file is
|
||||
a `Couldn't find the file to stream` 404 until that scan passes it. Run this from
|
||||
the host shell through `podman exec`, never from a neighbour container: PMS's
|
||||
HTTPD closes without a single byte on any request that arrives from another
|
||||
container's netns (10.89.x.x reads as "empty reply", not as a 401). The token
|
||||
rides stdin, never argv, where any process on the host could read it out of
|
||||
`/proc/<pid>/cmdline`; it lives in `Preferences.xml` inside the RW data bind.
|
||||
|
||||
```bash
|
||||
umask 077
|
||||
podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \
|
||||
'/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok
|
||||
cat >/tmp/pxscan.sh <<'EOS'
|
||||
read -r T
|
||||
cfg=$(mktemp)
|
||||
mk() { { printf 'header = "X-Plex-Token: %s"\nheader = "Accept: application/json"\n' "$T"
|
||||
printf 'url = "%s"\n' "$1"; } > "$cfg"; }
|
||||
mk "http://127.0.0.1:32400/library/sections"
|
||||
for k in $(curl -s -K "$cfg" | grep -o '"key":"[0-9]*"' | cut -d\" -f4); do
|
||||
mk "http://127.0.0.1:32400/library/sections/$k/refresh"
|
||||
printf 'request = "POST"\n' >> "$cfg"
|
||||
echo "plex section $k: $(curl -s -o /dev/null -w '%{http_code}' -K "$cfg")"
|
||||
done
|
||||
shred -u "$cfg"
|
||||
EOS
|
||||
podman cp /tmp/pxscan.sh plex:/tmp/pxscan.sh
|
||||
podman exec -i plex sh /tmp/pxscan.sh </tmp/px.tok
|
||||
shred -u /tmp/px.tok
|
||||
```
|
||||
|
||||
Every section answers `204` — or `200` with a Directory body when
|
||||
`Accept: application/json` is sent. Inside the quoted `printf` format a
|
||||
backslash-newline is LITERAL, not a continuation, so `mk` writes two `printf`s:
|
||||
one glued line and one stray backslash is a config file curl rejects, and the
|
||||
failure prints as `option -K: is unknown`, never as a bad config line.
|
||||
|
||||
Proof that a rename landed is not a scan response: `scripts/media-verify.py`
|
||||
replays the run's journal against the database — the only read that
|
||||
distinguishes "not scanned yet" from "gone"; see [[Media Relayout]].
|
||||
@@ -48,10 +48,13 @@ is a locator: one row, one clause, no numbers.
|
||||
| [[Calibre-Web]] | Calibre-Web is a web application that browses, reads, and downloads eBooks from an existing Calibre library database. |
|
||||
| [[Copyparty]] | Copyparty is a lightweight file server and media streamer that runs as a single Python binary. |
|
||||
| [[Dankserver Stack]] | Service inventory and routing overview for the media and library host. |
|
||||
| [[Entropy Volume]] | The Dankserver entropy volume is a RAID-backed LUKS filesystem mounted at |
|
||||
| [[Jellyfin]] | Jellyfin on Dankserver: quadlet unit, entropy-volume data layout, ports, client access, hardware-transcode status. |
|
||||
| [[Entropy Volume]] | Dankserver's /mnt/entropy volume: the LUKS/array chain, the crypttab + fstab boot path, and the containers bound to it |
|
||||
| [[Jellyfin]] | Jellyfin on Dankserver: quadlet unit, overlay metadata view, provider keys, ports, access, transcode status. |
|
||||
| [[mariadb]] | The MariaDB backend for RomM: quadlet unit, network alias, env-file secret handling, and the backup path. |
|
||||
| [[Media Relayout]] | Census, plan and apply the movie library into Jellyfin format: the gates to read, and the rollback. |
|
||||
| [[Media Audiobook Relayout]] | Re-filing /mnt/entropy/Audiobooks by Plex's chapter rows: narrator shelves, no renumbering, the refusals. |
|
||||
| [[Media Metadata View]] | The SSD-backed overlay the two media servers read and write: shape, guards, commands, laws, rollback. |
|
||||
| [[Media Music Relayout]] | Re-filing /mnt/entropy/Music by Jellyfin's tag rows: the census, the album key, the refusals. |
|
||||
| [[Media Relayout]] | Census, plan and apply the movie, TV, music and audiobook libraries: the gates, the refusals, the rollback. |
|
||||
| [[Plex]] | Plex Media Server is a **containerized media server on Dankserver**: a podman |
|
||||
| [[redis]] | The Redis cache and task queue for RomM: quadlet unit, network alias topology, and the backup path. |
|
||||
| [[RomM]] | RomM (ROM Manager) is a self-hosted ROM library manager and player. |
|
||||
|
||||
@@ -133,6 +133,16 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "a compose edit never restarts a Quadlet-owned media server" {
|
||||
# dankserver/compose.yaml is an inventory; the media servers' Quadlets own
|
||||
# their containers. A compose edit that restarted them would cut real viewers
|
||||
# for a change that cannot affect them.
|
||||
run python3 "$PLANNER" --target dankserver --paths dankserver/compose.yaml
|
||||
[ "$status" -eq 0 ]
|
||||
! printf '%s\n' "$output" | grep -qx jellyfin
|
||||
! printf '%s\n' "$output" | grep -qx plex
|
||||
}
|
||||
|
||||
@test "compose planner narrows a compose edit to the changed service" {
|
||||
local base head
|
||||
base=$(git -C "$REPO_ROOT" rev-parse HEAD)
|
||||
|
||||
Executable
+297
@@ -0,0 +1,297 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: scripts/jellyfin-providers.sh — the only mechanism that puts a
|
||||
# metadata-provider key into Jellyfin's plugin configuration on Dankserver.
|
||||
#
|
||||
# Output vocabulary the tests depend on:
|
||||
# "+ <action>" something this run will change
|
||||
# "= <fact>" something already true, so nothing changes
|
||||
#
|
||||
# The contracts, in the order the seam tests them:
|
||||
# 1. plan reports every action and never prints a key;
|
||||
# 2. a secret file that is not mode 600 is refused by name and nothing is
|
||||
# written — a key readable by other users is an error, not a warning;
|
||||
# 3. a downloaded plugin artifact is accepted only on its pinned sha256, its
|
||||
# official guid and its pinned version (a swapped or downgraded artifact
|
||||
# is refused before anything lands in plugins/);
|
||||
# 4. the TMDb key is written into the EXISTING config with every other
|
||||
# setting preserved — Jellyfin owns that file, and clobbering it would
|
||||
# silently reset country, language and adult settings;
|
||||
# 5. a TVDB config the operator already saved through the UI is kept;
|
||||
# 6. install is idempotent: a second run changes no byte;
|
||||
# 7. whatever is replaced is copied to the trash first (fleet law: never
|
||||
# overwrite without a recoverable copy);
|
||||
# 8. verify answers whether the endpoint is configured, key LENGTH only;
|
||||
# 9. no plugin configuration XML is tracked in git, the deployer places the
|
||||
# keys BEFORE it restarts Jellyfin, and the planner routes a change to this
|
||||
# installer to jellyfin.
|
||||
|
||||
load helpers
|
||||
|
||||
PROV="$REPO_ROOT/scripts/jellyfin-providers.sh"
|
||||
TMDB_KEY="tmdbk3a9Zq7Rtestkey"
|
||||
TVDB_KEY="tvdbk8b2Wt4Ptestkey"
|
||||
GUID_OFFICIAL="a677c0da-fac5-4cde-941a-7134223f14c8"
|
||||
|
||||
# make_plugin <file> <guid> <version> — a zip shaped like the official TheTVDB
|
||||
# artifact (meta.json + its two DLLs), so extraction is exercised for real.
|
||||
make_plugin() {
|
||||
python3 - "$1" "$2" "$3" <<'PYEOF'
|
||||
import json, sys, zipfile
|
||||
path, guid, version = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
meta = {
|
||||
"category": "MoviesAndShows",
|
||||
"guid": guid,
|
||||
"name": "TheTVDB",
|
||||
"owner": "jellyfin",
|
||||
"targetAbi": "12.0.0.0",
|
||||
"version": version,
|
||||
}
|
||||
with zipfile.ZipFile(path, "w") as z:
|
||||
z.writestr("meta.json", json.dumps(meta))
|
||||
z.writestr("Jellyfin.Plugin.Tvdb.dll", "dll-bytes")
|
||||
z.writestr("Tvdb.Sdk.dll", "dll-bytes")
|
||||
PYEOF
|
||||
sha256sum "$1" | cut -d' ' -f1
|
||||
}
|
||||
|
||||
setup() {
|
||||
JF="$BATS_TEST_TMPDIR/jf"
|
||||
TRASH="$BATS_TEST_TMPDIR/trash"
|
||||
SEC="$BATS_TEST_TMPDIR/secrets"
|
||||
mkdir -p "$JF/plugins/configurations" "$TRASH" "$SEC"
|
||||
cat >"$JF/plugins/configurations/Jellyfin.Plugin.Tmdb.xml" <<XML
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
|
||||
<MetadataCountryCode>US</MetadataCountryCode>
|
||||
<PreferOriginalLanguage />
|
||||
<Language>en</Language>
|
||||
<EnableCinematicForTrailers>false</EnableCinematicForTrailers>
|
||||
<IncludeAdult>false</IncludeAdult>
|
||||
<MaxCastMembers>25</MaxCastMembers>
|
||||
<AddGenresFromKeywords>true</AddGenresFromKeywords>
|
||||
<TmdbApiKey />
|
||||
</Configuration>
|
||||
XML
|
||||
KEYS="$SEC/provider-keys"
|
||||
printf 'TMDB_API_KEY=%s\nTVDB_API_KEY=%s\n' "$TMDB_KEY" "$TVDB_KEY" >"$KEYS"
|
||||
chmod 600 "$KEYS"
|
||||
ZIP="$BATS_TEST_TMPDIR/thetvdb.zip"
|
||||
PIN_SHA="$(make_plugin "$ZIP" "$GUID_OFFICIAL" 24.0.0.0)"
|
||||
PROV_ENV=(
|
||||
JF_ROOT="$JF"
|
||||
PROV_KEYS="$KEYS"
|
||||
TRASH_DIR="$TRASH"
|
||||
TVDB_PLUGIN_URL="file://$ZIP"
|
||||
TVDB_PLUGIN_SHA256="$PIN_SHA"
|
||||
TVDB_PLUGIN_GUID="$GUID_OFFICIAL"
|
||||
TVDB_PLUGIN_VERSION=24.0.0.0
|
||||
)
|
||||
}
|
||||
|
||||
prov() { env "${PROV_ENV[@]}" bash "$PROV" "$@"; }
|
||||
|
||||
tree_fingerprint() { find "$JF" -type f | sort | xargs -r sha256sum | sha256sum; }
|
||||
|
||||
@test "plan names the plugin and both config files, and prints no key" {
|
||||
run prov plan
|
||||
[ "$status" -eq 0 ]
|
||||
printf '%s\n' "${lines[@]}" | grep -q '^+ install thetvdb_24\.0\.0\.0'
|
||||
printf '%s\n' "${lines[@]}" | grep -q '^+ set TmdbApiKey'
|
||||
printf '%s\n' "${lines[@]}" | grep -q '^+ create Jellyfin\.Plugin\.Tvdb\.xml'
|
||||
# The whole point of running this over a shoulder: a key never appears.
|
||||
! printf '%s\n' "${lines[@]}" | grep -qF "$TMDB_KEY"
|
||||
! printf '%s\n' "${lines[@]}" | grep -qF "$TVDB_KEY"
|
||||
}
|
||||
|
||||
@test "plan writes nothing" {
|
||||
before="$(tree_fingerprint)"
|
||||
run prov plan
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(tree_fingerprint)" = "$before" ]
|
||||
}
|
||||
|
||||
@test "an already-configured endpoint reports no pending action" {
|
||||
prov install >/dev/null
|
||||
run prov plan
|
||||
[ "$status" -eq 0 ]
|
||||
! printf '%s\n' "${lines[@]}" | grep -q '^+ '
|
||||
printf '%s\n' "${lines[@]}" | grep -q '= thetvdb_24\.0\.0\.0 already installed'
|
||||
printf '%s\n' "${lines[@]}" | grep -q '= TmdbApiKey already set'
|
||||
printf '%s\n' "${lines[@]}" | grep -q '= Jellyfin\.Plugin\.Tvdb\.xml already has an ApiKey'
|
||||
}
|
||||
|
||||
@test "a keys file that is not mode 600 is refused and nothing is written" {
|
||||
chmod 644 "$KEYS"
|
||||
before="$(tree_fingerprint)"
|
||||
run prov install
|
||||
[ "$status" -ne 0 ]
|
||||
printf '%s\n' "${lines[@]}" | grep -qF '600'
|
||||
[ "$(tree_fingerprint)" = "$before" ]
|
||||
}
|
||||
|
||||
@test "a missing TVDB key is refused by name before anything is fetched" {
|
||||
printf 'TMDB_API_KEY=%s\n' "$TMDB_KEY" >"$KEYS"
|
||||
chmod 600 "$KEYS"
|
||||
run prov install
|
||||
[ "$status" -ne 0 ]
|
||||
printf '%s\n' "${lines[@]}" | grep -qF 'TVDB_API_KEY'
|
||||
[ ! -d "$JF/plugins/thetvdb_24.0.0.0" ]
|
||||
}
|
||||
|
||||
@test "a root with no plugin directory is refused instead of fabricated" {
|
||||
# Creating plugins/ inside a wrong path would install a plugin Jellyfin
|
||||
# never loads, and would hide the operator's typo.
|
||||
run env "${PROV_ENV[@]}" JF_ROOT="$BATS_TEST_TMPDIR/nope" bash "$PROV" install
|
||||
[ "$status" -ne 0 ]
|
||||
[ ! -d "$BATS_TEST_TMPDIR/nope/plugins" ]
|
||||
}
|
||||
|
||||
@test "an artifact whose sha256 is not the pinned one is refused" {
|
||||
run env "${PROV_ENV[@]}" TVDB_PLUGIN_SHA256="$(printf 'f%.0s' {1..64})" \
|
||||
bash "$PROV" install
|
||||
[ "$status" -ne 0 ]
|
||||
printf '%s\n' "${lines[@]}" | grep -qiF 'sha256'
|
||||
[ ! -d "$JF/plugins/thetvdb_24.0.0.0" ]
|
||||
}
|
||||
|
||||
@test "an artifact that is not the official TVDB plugin is refused" {
|
||||
# A matching sha proves only that the bytes are the bytes we fetched.
|
||||
EVIL="$BATS_TEST_TMPDIR/evil.zip"
|
||||
EVIL_SHA="$(make_plugin "$EVIL" "deadbeef-0000-0000-0000-000000000000" 24.0.0.0)"
|
||||
run env "${PROV_ENV[@]}" TVDB_PLUGIN_URL="file://$EVIL" TVDB_PLUGIN_SHA256="$EVIL_SHA" \
|
||||
bash "$PROV" install
|
||||
[ "$status" -ne 0 ]
|
||||
printf '%s\n' "${lines[@]}" | grep -qiF 'guid'
|
||||
[ ! -d "$JF/plugins/thetvdb_24.0.0.0" ]
|
||||
}
|
||||
|
||||
@test "an artifact that is not the pinned version is refused" {
|
||||
OLD="$BATS_TEST_TMPDIR/old.zip"
|
||||
OLD_SHA="$(make_plugin "$OLD" "$GUID_OFFICIAL" 22.0.0.0)"
|
||||
run env "${PROV_ENV[@]}" TVDB_PLUGIN_URL="file://$OLD" TVDB_PLUGIN_SHA256="$OLD_SHA" \
|
||||
bash "$PROV" install
|
||||
[ "$status" -ne 0 ]
|
||||
printf '%s\n' "${lines[@]}" | grep -qiF 'version'
|
||||
[ ! -d "$JF/plugins/thetvdb_24.0.0.0" ]
|
||||
}
|
||||
|
||||
@test "a different plugin version already in plugins/ is refused, not doubled" {
|
||||
# Two thetvdb_* directories means Jellyfin loads one plugin twice; the
|
||||
# operator removes the old one (into the trash) deliberately, not by an
|
||||
# installer quietly adding a second.
|
||||
prov install >/dev/null
|
||||
OLD="$BATS_TEST_TMPDIR/old.zip"
|
||||
OLD_SHA="$(make_plugin "$OLD" "$GUID_OFFICIAL" 22.0.0.0)"
|
||||
run env "${PROV_ENV[@]}" TVDB_PLUGIN_URL="file://$OLD" TVDB_PLUGIN_SHA256="$OLD_SHA" \
|
||||
TVDB_PLUGIN_VERSION=22.0.0.0 bash "$PROV" install
|
||||
[ "$status" -ne 0 ]
|
||||
printf '%s\n' "${lines[@]}" | grep -qF 'thetvdb_24.0.0.0'
|
||||
[ ! -d "$JF/plugins/thetvdb_22.0.0.0" ]
|
||||
}
|
||||
|
||||
@test "install places the plugin, keys TMDb, and preserves every other setting" {
|
||||
run prov install
|
||||
[ "$status" -eq 0 ]
|
||||
! printf '%s\n' "${lines[@]}" | grep -qF "$TMDB_KEY"
|
||||
! printf '%s\n' "${lines[@]}" | grep -qF "$TVDB_KEY"
|
||||
[ -f "$JF/plugins/thetvdb_24.0.0.0/meta.json" ]
|
||||
[ -f "$JF/plugins/thetvdb_24.0.0.0/Jellyfin.Plugin.Tvdb.dll" ]
|
||||
local cfg="$JF/plugins/configurations/Jellyfin.Plugin.Tmdb.xml"
|
||||
# Jellyfin owns this file: the settings it wrote must survive the edit.
|
||||
for setting in MetadataCountryCode Language IncludeAdult MaxCastMembers AddGenresFromKeywords; do
|
||||
grep -qF "<$setting>" "$cfg" || {
|
||||
echo "lost $setting"
|
||||
return 1
|
||||
}
|
||||
done
|
||||
grep -qF '<IncludeAdult>false</IncludeAdult>' "$cfg"
|
||||
grep -qF "<TmdbApiKey>$TMDB_KEY</TmdbApiKey>" "$cfg"
|
||||
grep -qF "<ApiKey>$TVDB_KEY</ApiKey>" "$JF/plugins/configurations/Jellyfin.Plugin.Tvdb.xml"
|
||||
}
|
||||
|
||||
@test "a TVDB config the operator saved through the UI is kept" {
|
||||
cat >"$JF/plugins/configurations/Jellyfin.Plugin.Tvdb.xml" <<XML
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Configuration xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
|
||||
<ApiKey>ui-saved-key</ApiKey>
|
||||
<EnableSeasonPathMatching>true</EnableSeasonPathMatching>
|
||||
</Configuration>
|
||||
XML
|
||||
run prov install
|
||||
[ "$status" -eq 0 ]
|
||||
printf '%s\n' "${lines[@]}" | grep -qiF 'kept'
|
||||
local cfg="$JF/plugins/configurations/Jellyfin.Plugin.Tvdb.xml"
|
||||
grep -qF '<ApiKey>ui-saved-key</ApiKey>' "$cfg"
|
||||
grep -qF '<EnableSeasonPathMatching>true</EnableSeasonPathMatching>' "$cfg"
|
||||
}
|
||||
|
||||
@test "a pin in the TVDB secret lands in the plugin config" {
|
||||
# A subscriber-shaped key needs the account's PIN beside it; a licensed key
|
||||
# does not, so the element appears only when the secret carries one.
|
||||
printf 'TMDB_API_KEY=%s\nTVDB_API_KEY=%s\nTVDB_PIN=123456\n' "$TMDB_KEY" "$TVDB_KEY" >"$KEYS"
|
||||
chmod 600 "$KEYS"
|
||||
prov install >/dev/null
|
||||
grep -qF '<Pin>123456</Pin>' "$JF/plugins/configurations/Jellyfin.Plugin.Tvdb.xml"
|
||||
}
|
||||
|
||||
@test "install is idempotent: a second run changes no byte" {
|
||||
prov install >/dev/null
|
||||
before="$(tree_fingerprint)"
|
||||
run prov install
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(tree_fingerprint)" = "$before" ]
|
||||
printf '%s\n' "${lines[@]}" | grep -qiF 'no change'
|
||||
# One plugin directory only: two versions in plugins/ means two loads.
|
||||
[ "$(find "$JF/plugins" -maxdepth 1 -type d -name 'thetvdb_*' | wc -l)" -eq 1 ]
|
||||
}
|
||||
|
||||
@test "what install replaces is copied to the trash first" {
|
||||
prov install >/dev/null
|
||||
# The TMDb config existed before this tool touched it, so the pre-edit copy
|
||||
# must stay recoverable.
|
||||
[ -n "$(find "$TRASH" -name 'Jellyfin.Plugin.Tmdb.xml*' -print -quit)" ]
|
||||
grep -qF '<TmdbApiKey />' "$(find "$TRASH" -name 'Jellyfin.Plugin.Tmdb.xml*' | head -1)"
|
||||
}
|
||||
|
||||
@test "verify refuses an unconfigured endpoint and names what is missing" {
|
||||
run prov verify
|
||||
[ "$status" -ne 0 ]
|
||||
printf '%s\n' "${lines[@]}" | grep -qiF 'thetvdb'
|
||||
printf '%s\n' "${lines[@]}" | grep -qiF 'TmdbApiKey'
|
||||
}
|
||||
|
||||
@test "verify passes once the plugin and both keys are in place" {
|
||||
prov install >/dev/null
|
||||
run prov verify
|
||||
[ "$status" -eq 0 ]
|
||||
! printf '%s\n' "${lines[@]}" | grep -qF "$TMDB_KEY"
|
||||
! printf '%s\n' "${lines[@]}" | grep -qF "$TVDB_KEY"
|
||||
# Length, not bytes: the operator sees the key loaded without reading it.
|
||||
printf '%s\n' "${lines[@]}" | grep -qE 'TmdbApiKey .*[0-9]+ chars'
|
||||
}
|
||||
|
||||
@test "no plugin configuration XML is tracked in this repo" {
|
||||
# These files hold provider keys and are owned by the running container;
|
||||
# tracking them would put credentials in git and guarantee drift.
|
||||
run git -C "$REPO_ROOT" ls-files
|
||||
[ "$status" -eq 0 ]
|
||||
! printf '%s\n' "${lines[@]}" | grep -qF 'plugins/configurations'
|
||||
! printf '%s\n' "${lines[@]}" | grep -qiF 'Plugin.Tvdb.xml'
|
||||
}
|
||||
|
||||
@test "the planner routes this installer to jellyfin" {
|
||||
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver \
|
||||
--paths scripts/jellyfin-providers.sh
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$output" = "jellyfin" ]
|
||||
}
|
||||
|
||||
@test "the deployer places the keys before it restarts jellyfin" {
|
||||
# The plugin and both keys are read at start-up, so a restart that runs
|
||||
# ahead of the placement boots the old, keyless state.
|
||||
body="$(sed -n '/^deploy_jellyfin()/,/^\}/p' "$REPO_ROOT/scripts/deploy-remote.sh")"
|
||||
[ -n "$body" ]
|
||||
up="$(printf '%s\n' "$body" | grep -n 'ensure_provider_keys' | head -1 | cut -d: -f1)"
|
||||
restart="$(printf '%s\n' "$body" | grep -n 'systemctl --user restart jellyfin.service' | head -1 | cut -d: -f1)"
|
||||
[ "${up:-999}" -lt "${restart:-0}" ]
|
||||
}
|
||||
Executable
+483
@@ -0,0 +1,483 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the audiobook-library relayout — scripts/media-census.py audiobooks (the
|
||||
# export of the rows Plex matched) and scripts/media-audiobooks.py (the plan).
|
||||
#
|
||||
# The audiobook tree is the one library on this box that Jellyfin does not read
|
||||
# at all and Plex reads completely: every chapter here is a Plex Track row under
|
||||
# an Album row (the BOOK) under one Artist row (the NARRATOR). That is the
|
||||
# authority this seam files by — never the collection folder a release pack made.
|
||||
#
|
||||
# Two laws here are the opposite of the music tool's, and both come from what the
|
||||
# rows actually contain. Files are NEVER renumbered: 7,556 of 10,485 books carry
|
||||
# at least one chapter whose index Plex does not know, so a `00 -` invented from
|
||||
# a missing index would order a book wrong, and playback order comes from the
|
||||
# embedded track tag either way — the name buys nothing and can only lie. And the
|
||||
# folder artist is the NARRATOR even though a reader thinks in authors: every
|
||||
# book has exactly one artist row, while the author survives only in 10% of the
|
||||
# album titles (`Kryptos by Graham McNeill`) and in no column Plex stores, so one
|
||||
# rule beats a shelf that is half authors and half narrators.
|
||||
#
|
||||
# Every test states which authority a file has and asserts where the PLAN files
|
||||
# it, never how the planner decided.
|
||||
|
||||
load helpers
|
||||
|
||||
PLAN="$REPO_ROOT/scripts/media-audiobooks.py"
|
||||
CENSUS="$REPO_ROOT/scripts/media-census.py"
|
||||
APPLY="$REPO_ROOT/scripts/media-apply.py"
|
||||
|
||||
# at PATH SIZE — a file with deterministic content.
|
||||
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
|
||||
|
||||
# row FILE NARRATOR BOOK YEAR TRACK CHAPTER
|
||||
# One row of the authority export scripts/media-census.py audiobooks writes.
|
||||
row() {
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$1" "$2" "$3" "${4:-}" "${5:-}" "$6" >> "$ROWS"
|
||||
}
|
||||
|
||||
# dst SRC — the destination the plan gives SRC, or nothing when it does not move.
|
||||
dst() {
|
||||
awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $3}' "$OUT"
|
||||
}
|
||||
|
||||
# kind SRC — the kind the plan gave SRC's move row.
|
||||
kind() {
|
||||
awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $4}' "$OUT"
|
||||
}
|
||||
|
||||
setup() {
|
||||
AB="$BATS_TEST_TMPDIR/Audiobooks"
|
||||
PARK="$AB/_unsorted"
|
||||
ENUM="$BATS_TEST_TMPDIR/enum.tsv"
|
||||
ROWS="$BATS_TEST_TMPDIR/rows.tsv"
|
||||
OUT="$BATS_TEST_TMPDIR/plan.tsv"
|
||||
EXC="$BATS_TEST_TMPDIR/exceptions.txt"
|
||||
# The header is part of the authority export, so the fixture carries it.
|
||||
printf 'file\tnarrator\tbook\tyear\ttrack\tchapter\n' > "$ROWS"
|
||||
|
||||
# One book of one author, split across the two part folders a pack shipped it
|
||||
# in, with a cover above part 1 and a playlist beside the chapters.
|
||||
A="$AB/Stephen King/It (1986)"
|
||||
at "$A [Part 1]/01 - Chapter 1.mp3" 9000
|
||||
at "$A [Part 1]/02 - Chapter 2.mp3" 9000
|
||||
at "$A [Part 1]/cover.jpg" 20
|
||||
at "$A [Part 1]/It.m3u" 30
|
||||
at "$A [Part 2]/03 - Chapter 3.mp3" 9000
|
||||
row "$A [Part 1]/01 - Chapter 1.mp3" "George Guidall" "It" 1986 1 "Chapter 1"
|
||||
row "$A [Part 1]/02 - Chapter 2.mp3" "George Guidall" "It" 1986 2 "Chapter 2"
|
||||
row "$A [Part 2]/03 - Chapter 3.mp3" "George Guidall" "It" 1986 3 "Chapter 3"
|
||||
|
||||
# A book whose album row carries no year at all.
|
||||
B="$AB/Collected/Project Gold"
|
||||
at "$B/01.ogg" 9000
|
||||
row "$B/01.ogg" "Operator X" "Project Gold" "" 1 "Opening"
|
||||
|
||||
# One book title, two narrators: two different books.
|
||||
C="$AB/Shared"
|
||||
at "$C/Dune side a.m4b" 9000
|
||||
at "$C/Dune side b.m4b" 9000
|
||||
at "$C/Dune 2020.m4b" 9000
|
||||
row "$C/Dune side a.m4b" "Scott Brick" "Dune" 1965 1 "Prologue"
|
||||
row "$C/Dune side b.m4b" "Scott Brick" "Dune" 1965 2 "Dune"
|
||||
row "$C/Dune 2020.m4b" "Malcolm Hillgartner" "Dune" 2020 1 "Book One"
|
||||
|
||||
# Two part folders of one book whose chapters share a file name.
|
||||
D="$AB/Pulp"
|
||||
at "$D/Book A/vol1.mp3" 9000
|
||||
at "$D/Book A vol2/vol1.mp3" 9000
|
||||
row "$D/Book A/vol1.mp3" "Nora Pace" "Hard Food" 2001 1 "Vol 1"
|
||||
row "$D/Book A vol2/vol1.mp3" "Nora Pace" "Hard Food" 2001 2 "Vol 1"
|
||||
|
||||
# The same narrator spelled two ways across one book's chapters.
|
||||
E="$AB/Spell"
|
||||
at "$E/kharn 1.mp3" 9000
|
||||
at "$E/kharn 2.mp3" 9000
|
||||
at "$E/kharn 3.mp3" 9000
|
||||
row "$E/kharn 1.mp3" "David Timson" "Kharn" 2013 1 "One"
|
||||
row "$E/kharn 2.mp3" "David timson" "Kharn" 2013 2 "Two"
|
||||
row "$E/kharn 3.mp3" "David Timson" "Kharn" 2013 3 "Three"
|
||||
|
||||
# A folder holding two different books: its loose note has no owner.
|
||||
F="$AB/Mixed"
|
||||
at "$F/book one chapter.mp3" 9000
|
||||
at "$F/book two chapter.mp3" 9000
|
||||
at "$F/notes.txt" 10
|
||||
row "$F/book one chapter.mp3" "Ann Reader" "Book One" 2001 1 "One"
|
||||
row "$F/book two chapter.mp3" "Bob Reader" "Book Two" 2002 1 "Two"
|
||||
|
||||
# A file already parked: the pile is for a human, so a row that would file it
|
||||
# must not pull it back into the tree.
|
||||
at "$PARK/Old/nobody.mp3" 9000
|
||||
row "$PARK/Old/nobody.mp3" "Ann Reader" "Book One" 2001 1 "One"
|
||||
|
||||
# The odd cases: no row, a video container, a name carrying a character Plex
|
||||
# says breaks playback, rows with a blank book and a blank narrator, and a
|
||||
# matched chapter whose file has no extension.
|
||||
G="$AB/Odd"
|
||||
at "$G/loose.mp3" 9000
|
||||
at "$G/concert.mp4" 9000
|
||||
at "$G/Bad: Name/01 chapter.mp3" 9000
|
||||
row "$G/Bad: Name/01 chapter.mp3" "Some Reader" "Trail: Marked" 1999 1 "Marked"
|
||||
at "$G/blank book.mp3" 9000
|
||||
row "$G/blank book.mp3" "Some Reader" "" 2000 1 "No Book"
|
||||
at "$G/blank narrator.mp3" 9000
|
||||
row "$G/blank narrator.mp3" "" "No Artist" 2000 1 "No Artist"
|
||||
at "$G/NoExt/part01" 9000
|
||||
row "$G/NoExt/part01" "Archivist" "Tapes" 1971 1 "Tape One"
|
||||
|
||||
# Tag-scan placeholders. Two unrelated books whose album rows both read
|
||||
# `[Unknown Album]`: the name is shared, so filing on it would merge books
|
||||
# that have nothing to do with each other. A rip timestamp can be glued to
|
||||
# the placeholder, and a multi-cast read says `Various Artists` — which IS a
|
||||
# narrator.
|
||||
H="$AB/Packs"
|
||||
at "$H/Stross/Ship/c1.mp3" 9000
|
||||
at "$H/Leckie/State/c1.mp3" 9000
|
||||
at "$H/Rip/tape.mp3" 9000
|
||||
at "$H/NoWho/Real Book.mp3" 9000
|
||||
at "$H/Multicast/Genesis.mp3" 9000
|
||||
row "$H/Stross/Ship/c1.mp3" "Various Artists" "[Unknown Album]" "" 1 "Part 1"
|
||||
row "$H/Leckie/State/c1.mp3" "Various Artists" "[Unknown Album]" "" 1 "Part 1"
|
||||
row "$H/Rip/tape.mp3" "Ann Reader" "Unknown album (03/03/2015 21:39:55)" "" 1 "One"
|
||||
row "$H/NoWho/Real Book.mp3" "[Unknown Artist]" "A Named Book" 2001 1 "One"
|
||||
row "$H/Multicast/Genesis.mp3" "Various Artists" "The Bible" "" 1 "Genesis"
|
||||
at "$H/Numbered/01 - Grimoire.mp3" 9000
|
||||
at "$H/Digits/1984.mp3" 9000
|
||||
row "$H/Numbered/01 - Grimoire.mp3" "0.4" "Grimoire of the Lamb" "" 1 "One"
|
||||
row "$H/Digits/1984.mp3" "Ann Reader" "1984" 1949 1 "One"
|
||||
|
||||
# The enum TSV is what scripts/media-census.py enumerate writes: bytes, ext,
|
||||
# path, no header.
|
||||
python3 - "$AB" "$ENUM" <<'PY'
|
||||
import os, sys
|
||||
root, out = sys.argv[1], sys.argv[2]
|
||||
with open(out, "w") as fh:
|
||||
for dirpath, _dirs, files in os.walk(root):
|
||||
for fn in files:
|
||||
p = os.path.join(dirpath, fn)
|
||||
fh.write(f"{os.path.getsize(p)}\t{os.path.splitext(fn)[1].lower()}\t{p}\n")
|
||||
PY
|
||||
}
|
||||
|
||||
plan() {
|
||||
run python3 "$PLAN" --root "$AB" --enum "$ENUM" --rows "$ROWS" \
|
||||
--out "$OUT" --exceptions "$EXC"
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "planner said: $output"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# db_with_part DB FILE NARRATOR BOOK YEAR TRACK CHAPTER — one Plex Track row
|
||||
# under an Album under an Artist, in the three tables the export joins.
|
||||
db_with_part() {
|
||||
local db="$1" file="$2" narrator="$3" book="$4" year="$5" track="$6" chapter="$7"
|
||||
local del="${8:-0}"
|
||||
python3 - "$db" "$file" "$narrator" "$book" "$year" "$track" "$chapter" "$del" <<'PY'
|
||||
import sqlite3, sys
|
||||
db, file, narrator, book, year, track, chapter, deleted = sys.argv[1:9]
|
||||
c = sqlite3.connect(db)
|
||||
c.execute("create table if not exists media_parts (id integer primary key,"
|
||||
" media_item_id, file, deleted_at)")
|
||||
c.execute("create table if not exists media_items (id integer primary key,"
|
||||
" metadata_item_id, deleted_at)")
|
||||
c.execute("""create table if not exists metadata_items (id integer primary key,
|
||||
parent_id, metadata_type, title, year, "index", deleted_at)""")
|
||||
now = None if deleted == "0" else "2026-01-01"
|
||||
ar = c.execute("insert into metadata_items (parent_id, metadata_type, title, year,"
|
||||
" \"index\", deleted_at) VALUES (NULL, 8, ?, NULL, NULL, NULL)",
|
||||
(narrator,)).lastrowid
|
||||
al = c.execute("insert into metadata_items (parent_id, metadata_type, title, year,"
|
||||
" \"index\", deleted_at) VALUES (?, 9, ?, ?, NULL, NULL)",
|
||||
(ar, book, int(year) if year else None)).lastrowid
|
||||
mi = c.execute("insert into metadata_items (parent_id, metadata_type, title, year,"
|
||||
" \"index\", deleted_at) VALUES (?, 10, ?, NULL, ?, NULL)",
|
||||
(al, chapter, int(track) if track else -1)).lastrowid
|
||||
it = c.execute("insert into media_items (metadata_item_id, deleted_at) VALUES (?, NULL)",
|
||||
(mi,)).lastrowid
|
||||
c.execute("insert into media_parts (media_item_id, file, deleted_at) VALUES (?,?,?)",
|
||||
(it, file, now))
|
||||
c.commit()
|
||||
PY
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ the export
|
||||
|
||||
@test "the census exports the narrator, book and chapter Plex matched for a part" {
|
||||
DB="$BATS_TEST_TMPDIR/plex.db"
|
||||
db_with_part "$DB" "$BATS_TEST_TMPDIR/Audiobooks/Pack/It part 1/01.mp3" \
|
||||
"George Guidall" "It" 1986 1 "Chapter 1"
|
||||
run python3 "$CENSUS" audiobooks --root "$BATS_TEST_TMPDIR/Audiobooks" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
run head -1 "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "${lines[0]}" = "file narrator book year track chapter" ]
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$BATS_TEST_TMPDIR/Audiobooks/Pack/It part 1/01\.mp3 George Guidall It 1986 1 Chapter 1$"
|
||||
}
|
||||
|
||||
@test "the census exports nothing outside the root, and nothing Plex deleted" {
|
||||
DB="$BATS_TEST_TMPDIR/plex.db"
|
||||
db_with_part "$DB" "/mnt/other/Library/01.mp3" "Someone" "Elsewhere" 1990 1 "One"
|
||||
db_with_part "$DB" "$BATS_TEST_TMPDIR/Audiobooks/Gone/01.mp3" "Someone" \
|
||||
"Deleted" 1990 1 "One" 1
|
||||
run python3 "$CENSUS" audiobooks --root "$BATS_TEST_TMPDIR/Audiobooks" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 1 ] # header only
|
||||
}
|
||||
|
||||
# wal_fresh DB ROOT OUT — a row Plex matched during a scan and committed, but
|
||||
# has not checkpointed: the normal state of a library being walked. The holder
|
||||
# connection stops the WAL folding into the main file when the writer closes,
|
||||
# and the census runs as a real subprocess against those same files.
|
||||
wal_fresh() {
|
||||
local db="$1" root="$2" out="$3"
|
||||
python3 - "$db" "$root" "$out" "$CENSUS" <<'PY'
|
||||
import sqlite3, subprocess, sys
|
||||
db, root, out, census = sys.argv[1:5]
|
||||
w = sqlite3.connect(db)
|
||||
w.execute("pragma journal_mode=wal")
|
||||
w.execute("pragma wal_autocheckpoint=0")
|
||||
w.commit()
|
||||
w.execute("pragma wal_checkpoint(truncate)") # the rows so far are on disk
|
||||
holder = sqlite3.connect(db) # last close would checkpoint
|
||||
ar = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||
' year, "index", deleted_at) VALUES (NULL, 8, ?, NULL, NULL, NULL)',
|
||||
("Fresh Narrator",)).lastrowid
|
||||
al = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||
' year, "index", deleted_at) VALUES (?, 9, ?, ?, NULL, NULL)',
|
||||
(ar, "Fresh Book", 2026)).lastrowid
|
||||
mi = w.execute('insert into metadata_items (parent_id, metadata_type, title,'
|
||||
' year, "index", deleted_at) VALUES (?, 10, ?, NULL, ?, NULL)',
|
||||
(al, "Fresh Chapter", 1)).lastrowid
|
||||
it = w.execute("insert into media_items (metadata_item_id, deleted_at)"
|
||||
" VALUES (?, NULL)", (mi,)).lastrowid
|
||||
w.execute("insert into media_parts (media_item_id, file, deleted_at)"
|
||||
" VALUES (?, ?, NULL)", (it, root + "/Fresh/01.mp3"))
|
||||
w.commit()
|
||||
sys.exit(subprocess.run([sys.executable, census, "audiobooks", "--root", root,
|
||||
"--db", db, "--out", out]).returncode)
|
||||
PY
|
||||
}
|
||||
|
||||
@test "the census exports a row the scanner committed but has not checkpointed" {
|
||||
DB="$BATS_TEST_TMPDIR/plex.db"
|
||||
db_with_part "$DB" "$BATS_TEST_TMPDIR/Audiobooks/Old/01.mp3" \
|
||||
"Old Narrator" "Old Book" 1990 1 "Old Chapter"
|
||||
run wal_fresh "$DB" "$BATS_TEST_TMPDIR/Audiobooks" "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(wc -l < "$BATS_TEST_TMPDIR/exp.tsv")" -eq 3 ] # header + both rows
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$BATS_TEST_TMPDIR/Audiobooks/Fresh/01\.mp3 Fresh Narrator Fresh Book 2026 1 Fresh Chapter$"
|
||||
}
|
||||
|
||||
# -------------------------------------------------------------------- the plan
|
||||
|
||||
@test "a matched chapter files under its narrator and its book, name untouched" {
|
||||
plan
|
||||
[ "$(dst "$A [Part 1]/01 - Chapter 1.mp3")" = \
|
||||
"$AB/George Guidall/It (1986)/01 - Chapter 1.mp3" ]
|
||||
}
|
||||
|
||||
@test "one book arrives in one folder however many part folders it shipped in" {
|
||||
plan
|
||||
[ "$(dst "$A [Part 2]/03 - Chapter 3.mp3")" = \
|
||||
"$AB/George Guidall/It (1986)/03 - Chapter 3.mp3" ]
|
||||
[ "$(dst "$A [Part 1]/02 - Chapter 2.mp3")" = \
|
||||
"$AB/George Guidall/It (1986)/02 - Chapter 2.mp3" ]
|
||||
}
|
||||
|
||||
@test "a chapter is never renumbered, whether or not Plex knows its index" {
|
||||
# The law this guards: playback order comes from the embedded track tag, and
|
||||
# three quarters of the books have chapters with no index at all, so a
|
||||
# number written into a name would be invented and would lie.
|
||||
plan
|
||||
[ "$(dst "$B/01.ogg")" = "$AB/Operator X/Project Gold/01.ogg" ]
|
||||
[ "$(dst "$A [Part 1]/01 - Chapter 1.mp3")" = \
|
||||
"$AB/George Guidall/It (1986)/01 - Chapter 1.mp3" ]
|
||||
}
|
||||
|
||||
@test "a book whose album row has no year gets no parentheses" {
|
||||
plan
|
||||
[ "$(dst "$B/01.ogg")" = "$AB/Operator X/Project Gold/01.ogg" ]
|
||||
}
|
||||
|
||||
@test "two chapters claiming one name inside one book are named apart" {
|
||||
plan
|
||||
first="$(dst "$D/Book A/vol1.mp3")"
|
||||
second="$(dst "$D/Book A vol2/vol1.mp3")"
|
||||
[ "$first" != "$second" ]
|
||||
[ -n "$first" ] && [ -n "$second" ]
|
||||
# and the plan still passes the executor's preflight, which refuses a target
|
||||
# two sources claim.
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$AB"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" != *"double-claim"* ]]
|
||||
}
|
||||
|
||||
@test "a cover follows the book whose folder it lies in" {
|
||||
plan
|
||||
[ "$(dst "$A [Part 1]/cover.jpg")" = "$AB/George Guidall/It (1986)/cover.jpg" ]
|
||||
}
|
||||
|
||||
@test "a playlist moves with its book and its bytes are never rewritten" {
|
||||
plan
|
||||
[ "$(dst "$A [Part 1]/It.m3u")" = "$AB/George Guidall/It (1986)/It.m3u" ]
|
||||
before=$(md5sum "$A [Part 1]/It.m3u" | cut -d" " -f1)
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$AB" --journal "$BATS_TEST_TMPDIR/j.jsonl" --apply
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(md5sum "$AB/George Guidall/It (1986)/It.m3u" | cut -d" " -f1)" = "$before" ]
|
||||
# the stored paths inside it still name the old file names, which the tool
|
||||
# leaves alone rather than editing a file the operator owns.
|
||||
[ "$(kind "$A [Part 1]/It.m3u")" != "chapter" ]
|
||||
}
|
||||
|
||||
@test "a note in a folder holding two books has no owner and parks" {
|
||||
plan
|
||||
[ "$(kind "$F/notes.txt")" = "park" ]
|
||||
[ "$(dst "$F/notes.txt")" = "$PARK/Mixed/notes.txt" ]
|
||||
}
|
||||
|
||||
@test "an audio file no row names parks with the path it has today" {
|
||||
plan
|
||||
[ "$(kind "$G/loose.mp3")" = "park" ]
|
||||
[ "$(dst "$G/loose.mp3")" = "$PARK/Odd/loose.mp3" ]
|
||||
}
|
||||
|
||||
@test "a video container parks instead of being renamed into audio" {
|
||||
plan
|
||||
[ "$(kind "$G/concert.mp4")" = "park" ]
|
||||
run grep -c "concert\.mp4" "$EXC"
|
||||
[ "$output" != "0" ]
|
||||
}
|
||||
|
||||
@test "a character Plex says breaks playback never reaches a destination" {
|
||||
plan
|
||||
[ "$(dst "$G/Bad: Name/01 chapter.mp3")" = \
|
||||
"$AB/Some Reader/Trail Marked (1999)/01 chapter.mp3" ]
|
||||
run cut -f3 "$OUT"
|
||||
[[ "$output" != *":"* ]]
|
||||
}
|
||||
|
||||
@test "one book title with two narrators stays two books" {
|
||||
plan
|
||||
[ "$(dst "$C/Dune side a.m4b")" = "$AB/Scott Brick/Dune (1965)/Dune side a.m4b" ]
|
||||
[ "$(dst "$C/Dune 2020.m4b")" = \
|
||||
"$AB/Malcolm Hillgartner/Dune (2020)/Dune 2020.m4b" ]
|
||||
}
|
||||
|
||||
@test "the spelling most chapters carry names the folder, and they share one folder" {
|
||||
plan
|
||||
[ "$(dst "$E/kharn 1.mp3")" = "$AB/David Timson/Kharn (2013)/kharn 1.mp3" ]
|
||||
[ "$(dst "$E/kharn 2.mp3")" = "$AB/David Timson/Kharn (2013)/kharn 2.mp3" ]
|
||||
[ "$(dst "$E/kharn 3.mp3")" = "$AB/David Timson/Kharn (2013)/kharn 3.mp3" ]
|
||||
}
|
||||
|
||||
@test "a row with no book title parks rather than inventing a folder" {
|
||||
plan
|
||||
[ "$(kind "$G/blank book.mp3")" = "park" ]
|
||||
}
|
||||
|
||||
@test "a row with no narrator parks rather than inventing a shelf" {
|
||||
plan
|
||||
[ "$(kind "$G/blank narrator.mp3")" = "park" ]
|
||||
}
|
||||
|
||||
@test "books that share a placeholder album name never share a shelf" {
|
||||
plan
|
||||
[ "$(dst "$H/Stross/Ship/c1.mp3")" = "$PARK/Packs/Stross/Ship/c1.mp3" ]
|
||||
[ "$(dst "$H/Leckie/State/c1.mp3")" = "$PARK/Packs/Leckie/State/c1.mp3" ]
|
||||
}
|
||||
|
||||
@test "a placeholder with a rip timestamp glued to it is still a placeholder" {
|
||||
plan
|
||||
[ "$(kind "$H/Rip/tape.mp3")" = "park" ]
|
||||
}
|
||||
|
||||
@test "a narrator placeholder parks even when the book is named" {
|
||||
plan
|
||||
[ "$(kind "$H/NoWho/Real Book.mp3")" = "park" ]
|
||||
}
|
||||
|
||||
@test "Various Artists is a multi-cast narrator and files its book" {
|
||||
plan
|
||||
[ "$(dst "$H/Multicast/Genesis.mp3")" = \
|
||||
"$AB/Various Artists/The Bible/Genesis.mp3" ]
|
||||
}
|
||||
|
||||
@test "a chapter number mistagged as the narrator parks" {
|
||||
plan
|
||||
[ "$(kind "$H/Numbered/01 - Grimoire.mp3")" = "park" ]
|
||||
}
|
||||
|
||||
@test "a book whose title is digits still files, because only a name must have a letter" {
|
||||
plan
|
||||
[ "$(dst "$H/Digits/1984.mp3")" = "$AB/Ann Reader/1984 (1949)/1984.mp3" ]
|
||||
}
|
||||
|
||||
@test "the row, not the extension, decides what a chapter is" {
|
||||
plan
|
||||
[ "$(kind "$G/NoExt/part01")" = "chapter" ]
|
||||
[ "$(dst "$G/NoExt/part01")" = "$AB/Archivist/Tapes (1971)/part01" ]
|
||||
}
|
||||
|
||||
@test "the plan replays through the executor with nothing refused" {
|
||||
plan
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$AB" \
|
||||
--journal "$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" != *"refusing"* ]]
|
||||
# The refusal this guards is two sources claiming one target: the executor
|
||||
# stops the whole run on it, so the plan must never hand a path out twice.
|
||||
[ -z "$(awk -F '\t' '$1 == "move" {print $3}' "$OUT" | sort | uniq -d)" ]
|
||||
}
|
||||
|
||||
@test "an occupied destination is refused rather than overwritten" {
|
||||
plan
|
||||
mkdir -p "$AB/George Guidall/It (1986)"
|
||||
at "$AB/George Guidall/It (1986)/01 - Chapter 1.mp3" 1
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$AB"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"occupied"* ]]
|
||||
[ -e "$A [Part 1]/01 - Chapter 1.mp3" ]
|
||||
}
|
||||
|
||||
@test "applying the plan and re-planning moves nothing" {
|
||||
plan
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$AB" \
|
||||
--journal "$BATS_TEST_TMPDIR/j.jsonl" --apply
|
||||
[ "$status" -eq 0 ]
|
||||
python3 - "$AB" "$ENUM" <<'PY'
|
||||
import os, sys
|
||||
root, out = sys.argv[1], sys.argv[2]
|
||||
with open(out, "w") as fh:
|
||||
for dirpath, _dirs, files in os.walk(root):
|
||||
for fn in files:
|
||||
p = os.path.join(dirpath, fn)
|
||||
fh.write(f"{os.path.getsize(p)}\t{os.path.splitext(fn)[1].lower()}\t{p}\n")
|
||||
PY
|
||||
# The rows still name the paths the chapters were placed at: that is what a
|
||||
# server re-scan of the moved tree reports.
|
||||
python3 - "$OUT" "$ROWS" <<'PY'
|
||||
import csv, sys
|
||||
out, rows = sys.argv[1], sys.argv[2]
|
||||
moved = {r[1]: r[2] for r in csv.reader(open(out), delimiter="\t") if r[0] == "move"}
|
||||
with open(rows) as fh, open(rows + ".2", "w") as fh2:
|
||||
fh2.write(fh.readline().rstrip("\n") + "\n")
|
||||
for line in fh:
|
||||
f = line.split("\t")[0]
|
||||
fh2.write("\t".join([moved.get(f, f)] + line.rstrip("\n").split("\t")[1:]) + "\n")
|
||||
PY
|
||||
run python3 "$PLAN" --root "$AB" --enum "$ENUM" --rows "$ROWS.2" \
|
||||
--out "$OUT.2" --exceptions "$EXC.2"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(awk -F '\t' '$1 == "move"' "$OUT.2" | wc -l)" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "nothing under the park root is planned again" {
|
||||
plan
|
||||
run awk -F '\t' '$1 == "move" && $2 ~ /_unsorted/' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
Executable
+373
@@ -0,0 +1,373 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the music-library relayout — scripts/media-census.py music (the tag and
|
||||
# provider authority export) and scripts/media-music.py (the plan).
|
||||
#
|
||||
# Jellyfin's music rule is the opposite of the movie and TV ones: the embedded
|
||||
# tags are the metadata and the file name is only a fallback title, so the
|
||||
# authority here is the Audio row Jellyfin already matched (its album-artist,
|
||||
# album, year, disc and track columns) and NOT the path a ripper chose. Every
|
||||
# test states which authority a file has — row only, row with no album item,
|
||||
# sidecar of a file that has a row, nothing at all — and asserts where the PLAN
|
||||
# files it, never how the planner decided.
|
||||
#
|
||||
# The three outcomes that cost a re-run of a 90k-file plan: a track filed under
|
||||
# the wrong artist (it splits one album across two artists), a destination that
|
||||
# two tracks claim (the executor refuses, and the operator loses the run), and a
|
||||
# lyric file that stops matching the basename of the track it belongs to, which
|
||||
# silently loses the lyrics. Everything below is written to catch those, plus the
|
||||
# gate that matters most: the plan must replay through scripts/media-apply.py,
|
||||
# whose preflight is the fleet's only overwrite guard.
|
||||
|
||||
load helpers
|
||||
|
||||
PLAN="$REPO_ROOT/scripts/media-music.py"
|
||||
CENSUS="$REPO_ROOT/scripts/media-census.py"
|
||||
APPLY="$REPO_ROOT/scripts/media-apply.py"
|
||||
|
||||
# at PATH SIZE — a file with deterministic content.
|
||||
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
|
||||
|
||||
# row FILE ARTIST ALBUM YEAR DISC TRACK TITLE [MB] [PERFORMER]
|
||||
# One row of the authority export scripts/media-census.py music writes.
|
||||
row() {
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$1" "$2" "$3" "${4:-}" "${5:-}" "${6:-}" "$7" "${8:-}" "${9:-}" >> "$ROWS"
|
||||
}
|
||||
|
||||
# dst SRC — the destination the plan gives SRC, or nothing when it does not move.
|
||||
dst() {
|
||||
awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $3}' "$OUT"
|
||||
}
|
||||
|
||||
setup() {
|
||||
MUSIC="$BATS_TEST_TMPDIR/Music"
|
||||
PARK="$MUSIC/_unsorted"
|
||||
ENUM="$BATS_TEST_TMPDIR/enum.tsv"
|
||||
ROWS="$BATS_TEST_TMPDIR/rows.tsv"
|
||||
OUT="$BATS_TEST_TMPDIR/plan.tsv"
|
||||
EXC="$BATS_TEST_TMPDIR/exceptions.txt"
|
||||
|
||||
# A discography pack: the artist's own albums buried under "01. Albums",
|
||||
# with a lyric, a cover, an EAC log and a playlist beside the tracks.
|
||||
A="$MUSIC/Artist One - Discography [FLAC]/01. Albums/Album X"
|
||||
at "$A/01. First.flac" 9000
|
||||
at "$A/02. Second.flac" 9000
|
||||
at "$A/01. First.lrc" 10
|
||||
at "$A/cover.jpg" 20
|
||||
at "$A/rip.log" 10
|
||||
at "$A/Album X.m3u" 10
|
||||
|
||||
# A multi-disc compilation: one album (the album artist is the compilation
|
||||
# tag), two discs, and a different performer on each track.
|
||||
B="$MUSIC/Box (2019)"
|
||||
at "$B/Disc 1/01 Track A.flac" 9000
|
||||
at "$B/Disc 2/01 Track B.flac" 9000
|
||||
|
||||
# The box cover lies above the two disc folders: the discs below it own it.
|
||||
at "$B/box.jpg" 10
|
||||
|
||||
# One image plus its cuesheet: a whole album in one file.
|
||||
C="$MUSIC/Image Cue"
|
||||
at "$C/Image.flac" 9000
|
||||
at "$C/Image.cue" 10
|
||||
|
||||
# The hard cases: no row at all, a video container, two tracks claiming one
|
||||
# number and one title, a track with no number, and a track whose album
|
||||
# artist tag is empty so only the performer names it.
|
||||
D="$MUSIC/Odd"
|
||||
at "$D/loose.flac" 9000
|
||||
at "$D/concert.mkv" 9000
|
||||
at "$D/dup1.flac" 9000
|
||||
at "$D/dup2.flac" 9000
|
||||
at "$D/unnumbered.flac" 9000
|
||||
at "$D/lone.flac" 9000
|
||||
|
||||
# A sidecar in a folder that holds two different albums.
|
||||
E="$MUSIC/Mixed"
|
||||
at "$E/one.flac" 9000
|
||||
at "$E/two.flac" 9000
|
||||
at "$E/notes.pdf" 10
|
||||
|
||||
# An album title carrying a character Jellyfin says breaks playback.
|
||||
F="$MUSIC/Question Mark"
|
||||
at "$F/01. Bang.flac" 9000
|
||||
|
||||
# An album whose tags name no album artist at all, and two different
|
||||
# performers; and one album title carried by two different album artists.
|
||||
G="$MUSIC/Tagless"
|
||||
at "$G/one.flac" 9000
|
||||
at "$G/two.flac" 9000
|
||||
H="$MUSIC/Same Title"
|
||||
at "$H/a.flac" 9000
|
||||
at "$H/b.flac" 9000
|
||||
|
||||
# One album whose tracks disagree on the spelling of its name; the minority
|
||||
# spelling sorts first, so it must not win the folder.
|
||||
I="$MUSIC/Puppets"
|
||||
at "$I/one.flac" 9000
|
||||
at "$I/two.flac" 9000
|
||||
at "$I/three.flac" 9000
|
||||
|
||||
# The authority export carries a header; the planner reads it by column name.
|
||||
printf 'file\tartist\talbum\tyear\tdisc\ttrack\ttitle\tmb\tperformer\n' > "$ROWS"
|
||||
row "$A/01. First.flac" "Artist One" "Album X" 2015 1 1 "First Song"
|
||||
row "$A/02. Second.flac" "Artist One" "Album X" 2015 1 2 "Second Song: Reprise"
|
||||
row "$B/Disc 1/01 Track A.flac" "Various Artists" "Best Of" 2019 1 1 "Track A" \
|
||||
"" "Aaa Singer"
|
||||
row "$B/Disc 2/01 Track B.flac" "Various Artists" "Best Of" 2019 2 1 "Track B" \
|
||||
"" "Bbb Singer"
|
||||
row "$C/Image.flac" "Solo" "Single Album" 2001 1 1 "Single Album"
|
||||
row "$D/dup1.flac" "Dup Artist" "Dupes" "" "" 1 "Same"
|
||||
row "$D/dup2.flac" "Dup Artist" "Dupes" "" "" 1 "Same"
|
||||
row "$D/unnumbered.flac" "No Number" "No Number Album" "" "" "" "Untitled"
|
||||
row "$D/lone.flac" "" "Session Works" "" "" 1 "Lone" "" "Session Singer"
|
||||
row "$E/one.flac" "Mix Artist" "First Set" 2000 "" 1 "One"
|
||||
row "$E/two.flac" "Mix Artist" "Second Set" 2001 "" 1 "Two"
|
||||
row "$F/01. Bang.flac" "Billie" "WHEN WE ALL FALL ASLEEP, WHERE DO WE GO?" \
|
||||
2019 "" 1 "!!!!!!"
|
||||
row "$G/one.flac" "" "Two Seasons" 2005 "" 1 "One" "" "Alpha Band"
|
||||
row "$G/two.flac" "" "Two Seasons" 2005 "" 2 "Two" "" "Beta Band"
|
||||
row "$H/a.flac" "Aaa Singer" "Same Title" 2011 "" 1 "A Song"
|
||||
row "$H/b.flac" "Bbb Singer" "Same Title" 2011 "" 1 "B Song"
|
||||
row "$I/one.flac" "Metallica" "Master Of Puppets" 2015 "" 1 "One"
|
||||
row "$I/two.flac" "Metallica" "Master of Puppets" 2015 "" 2 "Two"
|
||||
row "$I/three.flac" "Metallica" "Master of Puppets" 2015 "" 3 "Three"
|
||||
|
||||
# The enum TSV is what scripts/media-census.py enumerate writes: bytes, ext,
|
||||
# path, no header.
|
||||
python3 - "$MUSIC" "$ENUM" <<'PY'
|
||||
import os, sys
|
||||
root, out = sys.argv[1], sys.argv[2]
|
||||
with open(out, "w") as fh:
|
||||
for dirpath, _dirs, files in os.walk(root):
|
||||
for fn in files:
|
||||
p = os.path.join(dirpath, fn)
|
||||
fh.write(f"{os.path.getsize(p)}\t{os.path.splitext(fn)[1].lower()}\t{p}\n")
|
||||
PY
|
||||
}
|
||||
|
||||
plan() {
|
||||
run python3 "$PLAN" --root "$MUSIC" --enum "$ENUM" --rows "$ROWS" \
|
||||
--out "$OUT" --exceptions "$EXC"
|
||||
[ "$status" -eq 0 ] || echo "planner said: $output"
|
||||
}
|
||||
|
||||
@test "the census exports the tag and provider row Jellyfin matched for a track" {
|
||||
DB="$BATS_TEST_TMPDIR/jellyfin.db"
|
||||
python3 - "$DB" <<'PY'
|
||||
import sqlite3, sys, uuid
|
||||
c = sqlite3.connect(sys.argv[1])
|
||||
c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber,
|
||||
ParentIndexNumber, ProductionYear, Album,
|
||||
AlbumArtists, Artists, Data)""")
|
||||
c.execute("""create table BaseItemProviders (ItemId, ProviderId, ProviderValue)""")
|
||||
AU = "MediaBrowser.Controller.Entities.Audio.Audio"
|
||||
AL = "MediaBrowser.Controller.Entities.Audio.MusicAlbum"
|
||||
aid, trk = str(uuid.uuid4()), "/data/Music/Pack/Album X/01. First.flac"
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(aid, None, AL, "Album X", "/data/Music/Pack/Album X", None, None, 2015,
|
||||
None, "Artist One", None, '{}'))
|
||||
c.execute("insert into BaseItemProviders values (?,?,?)",
|
||||
(aid, "MusicBrainzReleaseGroup", "rg-1234"))
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), aid, AU, "First Song", trk, 1, 1, 2015,
|
||||
"album x (the tag)", "Artist One", "Artist One", '{}'))
|
||||
c.commit()
|
||||
PY
|
||||
run python3 "$CENSUS" music --root "$BATS_TEST_TMPDIR/Music" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
run head -1 "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "${lines[0]}" = "file artist album year disc track title mb performer" ]
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$BATS_TEST_TMPDIR/Music/Pack/Album X/01\. First\.flac Artist One Album X 2015 1 1 First Song rg-1234 Artist One$"
|
||||
}
|
||||
|
||||
@test "a track whose folder is no album item still exports from its own tags" {
|
||||
DB="$BATS_TEST_TMPDIR/jellyfin.db"
|
||||
python3 - "$DB" <<'PY'
|
||||
import sqlite3, sys, uuid
|
||||
c = sqlite3.connect(sys.argv[1])
|
||||
c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber,
|
||||
ParentIndexNumber, ProductionYear, Album,
|
||||
AlbumArtists, Artists, Data)""")
|
||||
c.execute("""create table BaseItemProviders (ItemId, ProviderId, ProviderValue)""")
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), None, "MediaBrowser.Controller.Entities.Audio.Audio",
|
||||
"Lone", "/data/Music/Pack/lone.flac", 1, None, None,
|
||||
"Session Works", None, "Session Singer", '{}'))
|
||||
c.commit()
|
||||
PY
|
||||
run python3 "$CENSUS" music --root "$BATS_TEST_TMPDIR/Music" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$BATS_TEST_TMPDIR/Music/Pack/lone\.flac Session Singer Session Works 1 Lone Session Singer$"
|
||||
}
|
||||
|
||||
@test "a tagged track files under artist, album and year with a numbered title" {
|
||||
plan
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(dst "$A/01. First.flac")" = "$MUSIC/Artist One/Album X (2015)/01 - First Song.flac" ]
|
||||
}
|
||||
|
||||
@test "the tag title wins over the name the ripper gave the file" {
|
||||
plan
|
||||
[ "$(dst "$A/02. Second.flac")" = "$MUSIC/Artist One/Album X (2015)/02 - Second Song Reprise.flac" ]
|
||||
}
|
||||
|
||||
@test "a character Jellyfin says breaks playback never reaches a destination" {
|
||||
plan
|
||||
[ "$(dst "$F/01. Bang.flac")" = "$MUSIC/Billie/WHEN WE ALL FALL ASLEEP, WHERE DO WE GO (2019)/01 - !!!!!!.flac" ]
|
||||
run awk -F '\t' '$1 == "move" && $3 ~ /[<>:"|?*]/ {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "every disc of a multi-disc album lands in one folder, numbered apart" {
|
||||
plan
|
||||
[ "$(dst "$B/Disc 1/01 Track A.flac")" = "$MUSIC/Various Artists/Best Of (2019)/1.01 - Track A.flac" ]
|
||||
[ "$(dst "$B/Disc 2/01 Track B.flac")" = "$MUSIC/Various Artists/Best Of (2019)/2.01 - Track B.flac" ]
|
||||
}
|
||||
|
||||
@test "an album whose tracks name several artists files under Various Artists" {
|
||||
plan
|
||||
run awk -F '\t' '$1 == "move" && $3 ~ /\/Best Of \(2019\)\/.*\.flac$/ {print $3}' "$OUT"
|
||||
[ "${#lines[@]}" -eq 2 ]
|
||||
[[ "${lines[0]}" == "$MUSIC/Various Artists/"* ]]
|
||||
}
|
||||
|
||||
@test "an album with no album artist and two performers is a compilation" {
|
||||
plan
|
||||
[ "$(dst "$G/one.flac")" = "$MUSIC/Various Artists/Two Seasons (2005)/01 - One.flac" ]
|
||||
[ "$(dst "$G/two.flac")" = "$MUSIC/Various Artists/Two Seasons (2005)/02 - Two.flac" ]
|
||||
}
|
||||
|
||||
@test "the same album title under two album artists is two albums" {
|
||||
plan
|
||||
[ "$(dst "$H/a.flac")" = "$MUSIC/Aaa Singer/Same Title (2011)/01 - A Song.flac" ]
|
||||
[ "$(dst "$H/b.flac")" = "$MUSIC/Bbb Singer/Same Title (2011)/01 - B Song.flac" ]
|
||||
}
|
||||
|
||||
@test "the folder carries the album spelling most tracks carry" {
|
||||
plan
|
||||
[ "$(dst "$I/one.flac")" = "$MUSIC/Metallica/Master of Puppets (2015)/01 - One.flac" ]
|
||||
[ "$(dst "$I/two.flac")" = "$MUSIC/Metallica/Master of Puppets (2015)/02 - Two.flac" ]
|
||||
[ "$(dst "$I/three.flac")" = "$MUSIC/Metallica/Master of Puppets (2015)/03 - Three.flac" ]
|
||||
}
|
||||
|
||||
@test "a lyric file is renamed to match the track it belongs to" {
|
||||
plan
|
||||
[ "$(dst "$A/01. First.lrc")" = "$MUSIC/Artist One/Album X (2015)/01 - First Song.lrc" ]
|
||||
}
|
||||
|
||||
@test "the album image, the rip log and the playlist follow the album folder" {
|
||||
plan
|
||||
[ "$(dst "$A/cover.jpg")" = "$MUSIC/Artist One/Album X (2015)/cover.jpg" ]
|
||||
[ "$(dst "$A/rip.log")" = "$MUSIC/Artist One/Album X (2015)/rip.log" ]
|
||||
[ "$(dst "$A/Album X.m3u")" = "$MUSIC/Artist One/Album X (2015)/Album X.m3u" ]
|
||||
}
|
||||
|
||||
@test "a cover above the disc folders of one album follows that album" {
|
||||
plan
|
||||
[ "$(dst "$B/box.jpg")" = "$MUSIC/Various Artists/Best Of (2019)/box.jpg" ]
|
||||
}
|
||||
|
||||
@test "an image carrying a whole album keeps its own name beside its cuesheet" {
|
||||
plan
|
||||
[ "$(dst "$C/Image.flac")" = "$MUSIC/Solo/Single Album (2001)/Image.flac" ]
|
||||
[ "$(dst "$C/Image.cue")" = "$MUSIC/Solo/Single Album (2001)/Image.cue" ]
|
||||
}
|
||||
|
||||
@test "an audio file no provider row names is parked, not filed under an artist" {
|
||||
plan
|
||||
[[ "$(dst "$D/loose.flac")" == "$PARK/Odd/loose.flac" ]]
|
||||
}
|
||||
|
||||
@test "a video container in the music library is parked, never renamed to an audio one" {
|
||||
plan
|
||||
[[ "$(dst "$D/concert.mkv")" == "$PARK/Odd/concert.mkv" ]]
|
||||
run awk -F '\t' '$1 == "move" && $3 ~ /\.mka$|\.m4a$/ {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "two tracks claiming one number and one title get two destinations" {
|
||||
plan
|
||||
[ "$(dst "$D/dup1.flac")" = "$MUSIC/Dup Artist/Dupes/01 - Same.flac" ]
|
||||
[ "$(dst "$D/dup2.flac")" = "$MUSIC/Dup Artist/Dupes/01 - Same (2).flac" ]
|
||||
}
|
||||
|
||||
@test "an album with no year is named without one, a track with no number likewise" {
|
||||
plan
|
||||
[ "$(dst "$D/unnumbered.flac")" = "$MUSIC/No Number/No Number Album/Untitled.flac" ]
|
||||
}
|
||||
|
||||
@test "a track naming only a performer files under that performer" {
|
||||
plan
|
||||
[ "$(dst "$D/lone.flac")" = "$MUSIC/Session Singer/Session Works/01 - Lone.flac" ]
|
||||
}
|
||||
|
||||
@test "a sidecar in a folder holding two albums parks instead of guessing" {
|
||||
plan
|
||||
[[ "$(dst "$E/notes.pdf")" == "$PARK/Mixed/notes.pdf" ]]
|
||||
}
|
||||
|
||||
@test "the plan creates every folder before the move that needs it" {
|
||||
plan
|
||||
python3 - "$OUT" "$MUSIC" <<'PY'
|
||||
import os, sys
|
||||
made = set()
|
||||
for line in open(sys.argv[1]).read().splitlines()[1:]:
|
||||
action, _src, dest, _kind = line.split("\t")
|
||||
if action == "mkdir":
|
||||
made.add(dest)
|
||||
elif action == "move":
|
||||
parent = os.path.dirname(dest)
|
||||
while parent and parent != sys.argv[2]:
|
||||
assert parent in made, f"{dest} needs {parent}, which is never made"
|
||||
parent = os.path.dirname(parent)
|
||||
PY
|
||||
}
|
||||
|
||||
@test "no move lands outside the music library" {
|
||||
plan
|
||||
run awk -F '\t' -v l="$MUSIC" \
|
||||
'$1 == "move" && $3 !~ "^" l "/" {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "the executor preflights the plan and no target is claimed twice" {
|
||||
plan
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$BATS_TEST_TMPDIR" \
|
||||
--journal "$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
[ "$status" -eq 0 ]
|
||||
dup=$(awk -F '\t' '$1 == "move" {print $3}' "$OUT" | sort | uniq -d)
|
||||
[ -z "$dup" ]
|
||||
}
|
||||
|
||||
@test "re-planning a library the plan already satisfies moves nothing" {
|
||||
plan
|
||||
# Applying the plan and re-censusing is what the operator does next: both
|
||||
# authority files are keyed by path, so both are replayed through the plan
|
||||
# the way a fresh census of the moved tree would see them. (The enum has no
|
||||
# header; the rows export does, and its header row is not a path.)
|
||||
python3 - "$OUT" "$ENUM" "$ROWS" <<'PY'
|
||||
import os, sys
|
||||
plan = sys.argv[1]
|
||||
moves = {}
|
||||
for line in open(plan).read().splitlines()[1:]:
|
||||
action, src, dest, _kind = line.split("\t")
|
||||
if action == "move":
|
||||
moves[src] = dest
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
os.rename(src, dest)
|
||||
for path, col in ((sys.argv[2], 2), (sys.argv[3], 0)):
|
||||
rows = [line.rstrip("\n").split("\t") for line in open(path)]
|
||||
with open(path, "w") as fh:
|
||||
for f in rows:
|
||||
f[col] = moves.get(f[col], f[col])
|
||||
fh.write("\t".join(f) + "\n")
|
||||
PY
|
||||
plan
|
||||
run awk -F '\t' '$1 == "move" {print $2 " -> " $3}' "$OUT"
|
||||
[ -z "$output" ] || { echo "moved again: $output"; false; }
|
||||
}
|
||||
Executable
+161
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the SSD-backed metadata view (OverlayFS) that lets Jellyfin write
|
||||
# Kodi-format sidecars for Plex to read, without ever writing to the spindle
|
||||
# array and without letting either media server start against a missing view.
|
||||
#
|
||||
# The contracts, in the order the seam tests them:
|
||||
# 1. the plan puts the writable layer on the SSD and the media layer below it;
|
||||
# 2. bad input is refused by name instead of mounting something surprising;
|
||||
# 3. verify refuses a view that is not mounted (so nothing trusts a directory);
|
||||
# 4. both media units bind the VIEW and refuse to start without it;
|
||||
# 5. only those two units touch the view — the LLM and library tenants keep the
|
||||
# raw volume, so the blast radius is two containers;
|
||||
# 6. the deployer brings the view up BEFORE it restarts either server;
|
||||
# 7. the planner routes a change to these files to both media services.
|
||||
|
||||
load helpers
|
||||
|
||||
VIEW="$REPO_ROOT/scripts/media-view.sh"
|
||||
JELLY="$REPO_ROOT/dankserver/jellyfin/quadlet/jellyfin.container"
|
||||
PLEX="$REPO_ROOT/dankserver/plex/quadlet/plex.container"
|
||||
MOUNT=/var/lib/media-meta/view
|
||||
UNIT="$REPO_ROOT/dankserver/host/systemd/media-meta-view.mount"
|
||||
|
||||
@test "the plan names an overlay with the spindle below and the SSD above" {
|
||||
run bash "$VIEW" plan
|
||||
[ "$status" -eq 0 ]
|
||||
# Lower = the array; upper/work = the NVMe. A user asked for metadata to be
|
||||
# SSD-resident, so a spindle upper layer would defeat the whole mechanism.
|
||||
[ "${lines[0]}" = "lowerdir=/mnt/entropy" ]
|
||||
printf '%s\n' "${lines[@]}" | grep -qx 'upperdir=/var/lib/media-meta/upper'
|
||||
printf '%s\n' "${lines[@]}" | grep -qx 'workdir=/var/lib/media-meta/work'
|
||||
printf '%s\n' "${lines[@]}" | grep -Fqx "mountpoint=$MOUNT"
|
||||
# systemd names a mount unit after its escaped path, backslash and all.
|
||||
printf '%s\n' "${lines[@]}" | grep -Fqx \
|
||||
'unit=/etc/systemd/system/var-lib-media\x2dmeta-view.mount'
|
||||
}
|
||||
|
||||
@test "install refuses a caller who cannot write the unit directory or mount" {
|
||||
# Half an install is worse than none: the unit would exist without the
|
||||
# directories behind it. Refuse by name before anything is created.
|
||||
if [ "$(id -u)" -eq 0 ]; then
|
||||
skip "this seam is running as root"
|
||||
fi
|
||||
run bash "$VIEW" install
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"root"* ]]
|
||||
}
|
||||
|
||||
@test "verify refuses a view that is not mounted" {
|
||||
# Every consumer trusts the mount point to BE the library. On a host where it
|
||||
# is a plain empty directory, verify must fail loudly: that is the difference
|
||||
# between a scan that finds nothing and a scan that empties a library DB.
|
||||
if mountpoint -q "$MOUNT"; then
|
||||
skip "$MOUNT is a real mountpoint on this host"
|
||||
fi
|
||||
run bash "$VIEW" verify
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"$MOUNT"* ]]
|
||||
}
|
||||
|
||||
@test "both media servers bind the view and refuse to start without it" {
|
||||
for unit in "$JELLY" "$PLEX"; do
|
||||
grep -q "^Volume=$MOUNT:" "$unit" || return 1
|
||||
grep -q "^UnitConfigItem=ConditionPathIsMountPoint=$MOUNT$" "$unit" || return 1
|
||||
# The raw array must no longer be a container source: that bind is what
|
||||
# made sidecar sharing impossible (read-only) and metadata slow (seeks).
|
||||
! grep -q '^Volume=/mnt/entropy' "$unit"
|
||||
done
|
||||
# Jellyfin is the writer, so it needs the view writable; Plex only reads the
|
||||
# sidecars, and keeping its bind read-only costs nothing.
|
||||
grep -q "^Volume=$MOUNT:/data$" "$JELLY"
|
||||
grep -q "^Volume=$MOUNT:/mnt/entropy:ro$" "$PLEX"
|
||||
}
|
||||
|
||||
@test "only the two media units reference the view" {
|
||||
run grep -rl --include='*.container' "$MOUNT" "$REPO_ROOT"/dankserver \
|
||||
"$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro 2>/dev/null
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(printf '%s\n' "$output" | wc -l)" -eq 2 ]
|
||||
printf '%s\n' "$output" | grep -q 'jellyfin/quadlet/jellyfin.container'
|
||||
printf '%s\n' "$output" | grep -q 'plex/quadlet/plex.container'
|
||||
}
|
||||
|
||||
@test "the mount unit is an overlay whose upper layer lives on the SSD" {
|
||||
[ -f "$UNIT" ]
|
||||
run grep -E '^Options=lowerdir=/mnt/entropy,upperdir=/var/lib/media-meta/upper,workdir=/var/lib/media-meta/work' "$UNIT"
|
||||
[ "$status" -eq 0 ]
|
||||
# The unit must mount exactly where the two containers bind, or the guard and
|
||||
# the bind disagree and a media server browses an empty directory.
|
||||
run grep -Fqx "Where=$MOUNT" "$UNIT"
|
||||
[ "$status" -eq 0 ]
|
||||
# It must wait for the array to be unlocked: mounting an overlay whose lower
|
||||
# dir is an empty /mnt/entropy would present an empty library, not an error.
|
||||
run grep -E '^RequiresMountsFor=/mnt/entropy$' "$UNIT"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "the deployer raises the view before it restarts either media server" {
|
||||
for svc in jellyfin plex; do
|
||||
body="$(sed -n "/^deploy_${svc}()/,/^\}/p" "$REPO_ROOT/scripts/deploy-remote.sh")"
|
||||
[ -n "$body" ]
|
||||
up="$(printf '%s\n' "$body" | grep -n 'ensure_media_view' | head -1 | cut -d: -f1)"
|
||||
up_line="${up:-999}"
|
||||
restart="$(printf '%s\n' "$body" | grep -n "systemctl --user restart $svc.service" | head -1 | cut -d: -f1)"
|
||||
restart_line="${restart:-0}"
|
||||
[ "$up_line" -lt "$restart_line" ]
|
||||
done
|
||||
}
|
||||
|
||||
@test "the planner routes the view files to both media services" {
|
||||
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver --paths \
|
||||
scripts/media-view.sh dankserver/host/systemd/media-meta-view.mount
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$output" = "jellyfin
|
||||
plex" ]
|
||||
}
|
||||
|
||||
@test "a docs-only change still plans nothing for the view" {
|
||||
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankserver --paths \
|
||||
"serverAppConfigs-docs/Dankserver/Media Metadata View.md"
|
||||
[ "$status" -eq 0 ]
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "reap names the metadata whose media has moved and touches nothing" {
|
||||
# A relayout moves media on the RAW path; the sidecars the view already wrote
|
||||
# stay behind at the old coordinates. Left alone they become folders a media
|
||||
# server browses and finds nothing in.
|
||||
tmp="$(mktemp -d)"
|
||||
mkdir -p "$tmp/meta/upper/Movies/Ghost (2020)" "$tmp/meta/upper/Movies/Keeper (2020)" \
|
||||
"$tmp/meta/upper/TVShows/Big Little Lies/Season 01" "$tmp/meta/work" \
|
||||
"$tmp/array/Movies/Keeper (2020)" "$tmp/array/TVShows"
|
||||
: > "$tmp/meta/upper/Movies/Ghost (2020)/movie.nfo"
|
||||
: > "$tmp/meta/upper/Movies/Keeper (2020)/movie.nfo"
|
||||
: > "$tmp/meta/upper/TVShows/Big Little Lies/Season 01/show.nfo"
|
||||
: > "$tmp/array/Movies/Keeper (2020)/Keeper (2020).mkv"
|
||||
|
||||
run bash "$VIEW" reap --upper-base "$tmp/meta" --lower "$tmp/array"
|
||||
[ "$status" -eq 0 ]
|
||||
printf '%s\n' "$output" | grep -qxF 'Movies/Ghost (2020)'
|
||||
# The series folder is still below, so the ghost is the season inside it — and
|
||||
# it is reported at the level that is actually missing, not deeper.
|
||||
printf '%s\n' "$output" | grep -qxF 'TVShows/Big Little Lies'
|
||||
! printf '%s\n' "$output" | grep -q 'Keeper'
|
||||
[ -f "$tmp/meta/upper/Movies/Ghost (2020)/movie.nfo" ]
|
||||
}
|
||||
|
||||
@test "reap --apply moves ghosts to the SSD trash and keeps live metadata" {
|
||||
tmp="$(mktemp -d)"
|
||||
mkdir -p "$tmp/meta/upper/Movies/Ghost (2020)" "$tmp/meta/upper/Movies/Keeper (2020)" \
|
||||
"$tmp/array/Movies/Keeper (2020)"
|
||||
: > "$tmp/meta/upper/Movies/Ghost (2020)/movie.nfo"
|
||||
: > "$tmp/meta/upper/Movies/Keeper (2020)/movie.nfo"
|
||||
|
||||
run bash "$VIEW" reap --apply --upper-base "$tmp/meta" --lower "$tmp/array"
|
||||
[ "$status" -eq 0 ]
|
||||
[ ! -e "$tmp/meta/upper/Movies/Ghost (2020)" ]
|
||||
[ -f "$tmp/meta/upper/Movies/Keeper (2020)/movie.nfo" ]
|
||||
# Recoverable, not deleted: the item lands in a dated trash under the SSD base.
|
||||
[ "$(find "$tmp/meta/.AGENTTRASH" -name movie.nfo | wc -l)" -eq 1 ]
|
||||
}
|
||||
@@ -374,7 +374,10 @@ PY
|
||||
# apply_plan — plan, then apply for real; the journal goes in the tmpdir.
|
||||
apply_plan() {
|
||||
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
||||
# `/` is always a mountpoint, so the overlay note behaves as it does on a
|
||||
# host that runs the metadata view.
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" \
|
||||
--journal "$JOURNAL" --metadata-view / --apply
|
||||
}
|
||||
|
||||
@test "a run without --apply says what it would do and touches nothing" {
|
||||
@@ -402,10 +405,39 @@ apply_plan() {
|
||||
moved=$(python3 -c 'import json,sys
|
||||
print(sum(1 for l in open(sys.argv[1]) if json.loads(l).get("op") == "move"))' "$JOURNAL")
|
||||
[ "$moved" -eq "$planned" ]
|
||||
# The media servers' sidecars live in the metadata overlay and did not follow
|
||||
# these moves, so the run names that cleanup at the moment it is needed.
|
||||
[[ "$output" == *"media-view.sh reap"* ]]
|
||||
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
# The overlay note is only true where the metadata view is really mounted. On a
|
||||
# host whose media servers bind the array directly (dankserver today: both
|
||||
# quadlets mount /mnt/entropy, /var/lib/media-meta does not exist) the sidecars
|
||||
# moved with the media, and "next: run reap" is an order that cannot be carried
|
||||
# out. --metadata-view is how the caller names that path; the default is the
|
||||
# host's view path, and the note rides only when it is a mountpoint.
|
||||
@test "an apply with no mounted metadata view does not order a reap" {
|
||||
plan
|
||||
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"moved"* ]]
|
||||
[[ "$output" != *"media-view.sh reap"* ]]
|
||||
}
|
||||
|
||||
@test "a rollback with no mounted metadata view does not order a reap" {
|
||||
plan
|
||||
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
||||
[ "$status" -eq 0 ]
|
||||
run python3 "$APPLY" rollback --journal "$JOURNAL" --root "$LIB" --apply
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"rolled back"* ]]
|
||||
[[ "$output" != *"media-view.sh reap"* ]]
|
||||
}
|
||||
|
||||
@test "a target already holding a different file stops the whole run" {
|
||||
plan
|
||||
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
|
||||
Executable
+751
@@ -0,0 +1,751 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the TV-library relayout — scripts/media-census.py jellyfin (the metadata
|
||||
# authority export) and scripts/media-shows.py (the plan).
|
||||
#
|
||||
# The TV library is 52k files written in five different release layouts, and the
|
||||
# one thing that makes a rename of that size survivable is that the plan is
|
||||
# derived from what a metadata provider already matched, not from a guess at the
|
||||
# file name. So every test here states which authority a file had — name only,
|
||||
# provider only, both, neither — and asserts where the PLAN files it, never how
|
||||
# the planner decided. A fixture Jellyfin database stands in for the live one and
|
||||
# the name caches are input files, so no test touches the network.
|
||||
#
|
||||
# The target contract is tvdb-grade naming:
|
||||
# <Series> (<year>) [tvdbid-NNNN]/Season NN/<Series> (<year>) - SxxEyy - <Title>.<ext>
|
||||
# The id rides the FOLDER only: it is what both matchers read the series by, and a
|
||||
# 180-character ceiling on a file name is not the place to spend it. Season 00
|
||||
# holds specials; a provider row whose "season" is a YEAR is renumbered by the
|
||||
# episode title tvdb states for the series' id, never by a guess.
|
||||
# tvdb is the IDENTITY authority: a provider name a server derived from a folder
|
||||
# is not a series name, so a polluted name is undone through the series' tvdb id.
|
||||
# The review pile parks OUTSIDE the library — a parked folder inside it indexes
|
||||
# as one junk show, and that show's rows are then mistaken for evidence.
|
||||
#
|
||||
# The two outcomes that cost a re-run of a 50k-file plan are a file filed as an
|
||||
# episode that is not one (it steals an episode number from its real owner) and a
|
||||
# move that lands on an existing file. Everything below is written to catch
|
||||
# exactly those, plus the gate that matters most: the plan must be replayable by
|
||||
# scripts/media-apply.py, whose preflight is the fleet's only overwrite guard.
|
||||
|
||||
load helpers
|
||||
|
||||
PLAN="$REPO_ROOT/scripts/media-shows.py"
|
||||
CENSUS="$REPO_ROOT/scripts/media-census.py"
|
||||
APPLY="$REPO_ROOT/scripts/media-apply.py"
|
||||
|
||||
# at PATH SIZE — a file with deterministic content.
|
||||
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
|
||||
|
||||
# title_row FILE SERIES YEAR SEASON EPISODE [EXTRA] [TMDB] [TITLE] [TVDB]
|
||||
# — one row of the authority export scripts/media-census.py jellyfin writes.
|
||||
title_row() {
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "${4:-}" \
|
||||
"${5:-}" "${6:-}" "${7:-}" "${8:-}" "${9:-}" >> "$TITLES"
|
||||
}
|
||||
|
||||
# file PATH SIZE — `at` plus the row the enum census would have carried had the
|
||||
# file existed when setup walked the tree. A test that needs a shape the fixture
|
||||
# does not ship adds it here, or the planner never sees it.
|
||||
file() {
|
||||
at "$1" "$2"
|
||||
local b="${1##*/}"
|
||||
printf '%s\t.%s\t%s\n' "$2" "${b##*.}" "$1" >> "$ENUM"
|
||||
}
|
||||
|
||||
# dst SRC — the destination the plan gives SRC, or nothing when it does not move.
|
||||
dst() {
|
||||
awk -F '\t' -v s="$1" '$1 == "move" && $2 == s {print $3}' "$OUT"
|
||||
}
|
||||
|
||||
setup() {
|
||||
LIB="$BATS_TEST_TMPDIR/TVShows"
|
||||
PARK="$LIB/_unsorted"
|
||||
OUTSIDE="$BATS_TEST_TMPDIR/park"
|
||||
MOVIES="$BATS_TEST_TMPDIR/Movies"
|
||||
ENUM="$BATS_TEST_TMPDIR/enum.tsv"
|
||||
PROBED="$BATS_TEST_TMPDIR/probed.tsv"
|
||||
TITLES="$BATS_TEST_TMPDIR/titles.tsv"
|
||||
CACHE="$BATS_TEST_TMPDIR/tmdb.json"
|
||||
OUT="$BATS_TEST_TMPDIR/plan.tsv"
|
||||
EXC="$BATS_TEST_TMPDIR/exceptions.txt"
|
||||
: > "$PROBED"
|
||||
|
||||
SEASON="$LIB/The Examples (2010) Season 1 S01 (1080p BluRay x265 Silence)"
|
||||
# An episode its own name states, with a subtitle, a season image and a
|
||||
# trailer: four things that have to end up in four different places.
|
||||
at "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" 9000
|
||||
at "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt" 200
|
||||
at "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv.forced.srt" 250
|
||||
# A scene pack buries its subtitles two levels down, in Subs/<release-name>/.
|
||||
at "$SEASON/Subs/The.Examples.S01E01.Pilot.1080p.x265/The.Examples.S01E01.Pilot.1080p.x265.jpn.srt" 220
|
||||
at "$SEASON/folder.jpg" 300
|
||||
at "$SEASON/Trailers/teaser.mkv" 1000
|
||||
at "$SEASON/tvshow.nfo" 400
|
||||
# An anime pack: no episode marker anywhere in its name, numbered by the
|
||||
# provider only. Its year is the one thing no authority states.
|
||||
ANIME="$LIB/Anime Pack [grp] (1080p HEVC)"
|
||||
at "$ANIME/[grp] Anime Pack - 05.mkv" 5000
|
||||
# Two releases claiming one episode, and a file holding two episodes.
|
||||
at "$LIB/Dupes/Show Two S01E01.mkv" 4000
|
||||
at "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" 7000
|
||||
at "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" 7000
|
||||
# A file no authority numbers, and a long file the movie provider knows.
|
||||
at "$LIB/Orphan Tapes/clip001.mp4" 3000
|
||||
at "$LIB/Longform/Some Documentary Special (2015).mkv" 8000
|
||||
# A bonus folder inside a release pack whose own name is not the show's. The
|
||||
# pack's rows name the show; the bonus folder may never invent a series.
|
||||
at "$ANIME/01. Extras/NCOP 01.mkv" 1200
|
||||
# A season folder whose rows are stale: the database numbers every file of this
|
||||
# pack season 16 episode 20, four seasons' worth of them.
|
||||
at "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" 6000
|
||||
at "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" 4000
|
||||
# A box-set folder whose Series item the server named AFTER THE FOLDER: the
|
||||
# row's own name is junk, and only a provider id can undo it.
|
||||
BOX="$LIB/Show Box (2016) Season 1 S01 (1080p WEB-DL)"
|
||||
at "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" 6000
|
||||
# The park inside the library: the server indexed the whole pile as ONE show
|
||||
# named after the folder, so its rows state a series that is not a show.
|
||||
at "$PARK/Park Show S01E01.mkv" 5000
|
||||
at "$PARK/Twisted.Metal.S01E01.1080p.WEB-DL.mkv" 5200
|
||||
|
||||
# The enum census lists every file; the planner reads that instead of walking
|
||||
# the tree, so the fixture writes the same three columns.
|
||||
python3 - "$LIB" "$ENUM" <<'PY'
|
||||
import os, sys
|
||||
root, out = sys.argv[1], sys.argv[2]
|
||||
with open(out, "w") as fh:
|
||||
for d, _, fs in os.walk(root):
|
||||
for f in fs:
|
||||
p = os.path.join(d, f)
|
||||
fh.write(f"{os.stat(p).st_size}\t{os.path.splitext(f)[1]}\t{p}\n")
|
||||
PY
|
||||
|
||||
printf 'file\tseries\tyear\tseason\tepisode\textra\ttmdb\ttitle\ttvdb\n' > "$TITLES"
|
||||
printf '{}\n' > "$CACHE"
|
||||
title_row "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv" \
|
||||
"The Examples" 2010 1 1 "" "" "Pilot"
|
||||
title_row "$ANIME/[grp] Anime Pack - 05.mkv" "Anime Pack" "" 1 5
|
||||
title_row "$LIB/Dupes/Show Two S01E01.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots"
|
||||
title_row "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv" "Show Two" 2016 1 1 "" "" "Two Pilots"
|
||||
title_row "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv" "Show Two" 2016 1 2 "" "" "Double Trouble"
|
||||
title_row "$LIB/Loose.Show.S02E03.720p.WEB-DL.mkv" "Loose Show" 2019 2 3 "" "" "Loose Ends"
|
||||
title_row "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi" \
|
||||
"SNL Pack" "" 16 20
|
||||
title_row "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" \
|
||||
"Show Box (2016) Season 1" 2016 1 1 "" "2334" "Pilot" "78850"
|
||||
# The row the server wrote for a parked file: the series is the PARK FOLDER.
|
||||
title_row "$PARK/Park Show S01E01.mkv" "_unsorted" "" 1 1
|
||||
}
|
||||
|
||||
plan() {
|
||||
run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --titles "$TITLES" \
|
||||
--probed "$PROBED" --tmdb-cache "$CACHE" --movies "$MOVIES" \
|
||||
--out "$OUT" --exceptions "$EXC"
|
||||
}
|
||||
|
||||
plan_park() { # the review pile outside the library
|
||||
run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --titles "$TITLES" \
|
||||
--probed "$PROBED" --tmdb-cache "$CACHE" --movies "$MOVIES" \
|
||||
--park "$OUTSIDE" --out "$OUT" --exceptions "$EXC"
|
||||
}
|
||||
|
||||
@test "--tmdb with no key is reported as an unfinished plan, not a finished one" {
|
||||
# A dead provider is invisible where it happens: every 401 is one line on
|
||||
# stderr, and a terminal that scrolled past them shows a plan that reads as
|
||||
# complete while every folder still carries the metadata provider's junk. The
|
||||
# plan is still written — the answers that DID arrive are worth their rename —
|
||||
# so the state has to ride in the summary line and in the review file.
|
||||
# env -u keeps this offline no matter what the developer's shell exports.
|
||||
run env -u TMDB_API_KEY python3 "$PLAN" --root "$LIB" --enum "$ENUM" \
|
||||
--titles "$TITLES" --probed "$PROBED" --tmdb-cache "$CACHE" \
|
||||
--movies "$MOVIES" --tmdb --out "$OUT" --exceptions "$EXC"
|
||||
[ "$status" -eq 0 ]
|
||||
grep -q 'tmdb NOT ASKED (TMDB_API_KEY unset): NOT provider-complete' <<<"$output"
|
||||
grep -q 'WARNING: tmdb NOT ASKED (TMDB_API_KEY unset) — NOT provider-complete' "$EXC"
|
||||
grep -qE '^plan: [1-9][0-9]* moves' <<<"$output"
|
||||
}
|
||||
|
||||
@test "the census exports the provider's episode numbering as a TSV" {
|
||||
DB="$BATS_TEST_TMPDIR/jellyfin.db"
|
||||
python3 - "$DB" "$SEASON" <<'PY'
|
||||
import os, sqlite3, sys, uuid
|
||||
db, season = sys.argv[1], sys.argv[2]
|
||||
c = sqlite3.connect(db)
|
||||
pre = "/data/TVShows/" + os.path.basename(season)
|
||||
c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber,
|
||||
ParentIndexNumber, ProductionYear, ExtraType,
|
||||
IsVirtualItem, Data)""")
|
||||
sid = str(uuid.uuid4())
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples",
|
||||
pre, None, None, 2010, None, 0,
|
||||
'{"ProviderIds":{"Tmdb":"1234","Tvdb":"78850"}}'))
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), sid, "MediaBrowser.Controller.Entities.TV.Episode",
|
||||
"Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1,
|
||||
None, None, 0, '{}'))
|
||||
c.commit()
|
||||
PY
|
||||
run python3 "$CENSUS" jellyfin --root "$LIB" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
run head -1 "$BATS_TEST_TMPDIR/exp.tsv"
|
||||
[ "${lines[0]}" = "file series year season episode extra tmdb title tvdb" ]
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp.tsv" \
|
||||
| grep -q "^$SEASON/The\.Examples\.S01E01\.Pilot\.1080p\.x265\.mkv The Examples 2010 1 1 1234 Pilot 78850$"
|
||||
}
|
||||
|
||||
@test "the census reads provider ids from the table newer Jellyfin keeps" {
|
||||
DB="$BATS_TEST_TMPDIR/jellyfin12.db"
|
||||
python3 - "$DB" "$SEASON" <<'PY'
|
||||
import os, sqlite3, sys, uuid
|
||||
db, season = sys.argv[1], sys.argv[2]
|
||||
c = sqlite3.connect(db)
|
||||
pre = "/data/TVShows/" + os.path.basename(season)
|
||||
c.execute("""create table BaseItems (Id, ParentId, Type, Name, Path, IndexNumber,
|
||||
ParentIndexNumber, ProductionYear, ExtraType,
|
||||
IsVirtualItem, Data)""")
|
||||
# Jellyfin 12 moved the ids out of the Data blob and into their own table.
|
||||
c.execute("create table BaseItemProviders (ItemId, ProviderId, ProviderValue)")
|
||||
sid = str(uuid.uuid4())
|
||||
eid = str(uuid.uuid4())
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(sid, None, "MediaBrowser.Controller.Entities.TV.Series", "The Examples",
|
||||
pre, None, None, 2010, None, 0, '{}'))
|
||||
c.execute("insert into BaseItems values (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(eid, sid, "MediaBrowser.Controller.Entities.TV.Episode",
|
||||
"Pilot", pre + "/The.Examples.S01E01.Pilot.1080p.x265.mkv", 1, 1,
|
||||
None, None, 0, '{}'))
|
||||
c.execute("insert into BaseItemProviders values (?,?,?)", (sid, "Tvdb", "9999"))
|
||||
c.commit()
|
||||
PY
|
||||
run python3 "$CENSUS" jellyfin --root "$LIB" --db "$DB" \
|
||||
--out "$BATS_TEST_TMPDIR/exp12.tsv"
|
||||
[ "$status" -eq 0 ]
|
||||
tail -1 "$BATS_TEST_TMPDIR/exp12.tsv" \
|
||||
| grep -q " Pilot 9999$"
|
||||
}
|
||||
|
||||
@test "an episode its own name states files as the titled standard name" {
|
||||
plan
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv")" \
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.mkv" ]
|
||||
}
|
||||
|
||||
@test "the tvdb identity names the series folder, id and all, not the folder's echo" {
|
||||
printf '{"tvdb:series:78850": {"name": "Show Box", "year": "2016"}}\n' > "$CACHE"
|
||||
plan
|
||||
want="$LIB/Show Box (2016) [tvdbid-78850]/Season 01"
|
||||
want+="/Show Box (2016) - S01E01 - Pilot.mkv"
|
||||
[ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" = "$want" ]
|
||||
}
|
||||
|
||||
@test "a provider name that already carries the year is not given it twice" {
|
||||
# TVDb answers the reboot as `Acapulco (2021)`. The folder contract is ONE year,
|
||||
# and `Acapulco (2021) (2021)` is a folder neither matcher nor human reuses.
|
||||
printf '{"tvdb:series:78850": {"name": "Show Box (2016)", "year": "2016"}}\n' > "$CACHE"
|
||||
plan
|
||||
want="$LIB/Show Box (2016) [tvdbid-78850]/Season 01"
|
||||
want+="/Show Box (2016) - S01E01 - Pilot.mkv"
|
||||
[ "$(dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv")" = "$want" ]
|
||||
run grep -F '(2016) (2016)' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "a series no authority resolves keeps the provider's name and is reported" {
|
||||
plan
|
||||
[ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \
|
||||
= "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ]
|
||||
grep -A9 -i "no tvdb" "$EXC" | grep -q "SNL Pack"
|
||||
}
|
||||
|
||||
@test "a row whose series is the park folder is not evidence of a show" {
|
||||
plan_park
|
||||
run awk -F '\t' -v p="$PARK/" '$1 == "move" && index($3, p) == 1 {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
run awk -F '\t' '$1 == "move" && $2 ~ /Park Show S01E01/ {print $3}' "$OUT"
|
||||
[[ "$output" == "$OUTSIDE/"* ]]
|
||||
}
|
||||
|
||||
@test "the pile's sanitized display name is still the pile, not a series" {
|
||||
# Jellyfin drops the leading underscore when it shows the hidden folder, so the
|
||||
# row says the series is "unsorted" while the file sits in `_unsorted`. Match
|
||||
# only the underscored spelling and the plan renames real files into a visible
|
||||
# folder named after the review pile — which is how 617 of them got there.
|
||||
title_row "$PARK/Twisted.Metal.S01E01.1080p.WEB-DL.mkv" "unsorted" "" 1 1 \
|
||||
"" "" "Twisted Metal"
|
||||
plan_park
|
||||
run awk -F '\t' -v l="$LIB" \
|
||||
'$1 == "move" && index($3, l "/unsorted/") == 1 {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
run awk -F '\t' '$1 == "move" && $2 ~ /Twisted\.Metal\.S01E01/ {print $3}' "$OUT"
|
||||
[[ "$output" == "$OUTSIDE/"* ]]
|
||||
}
|
||||
|
||||
@test "a subtitle follows its episode and keeps its language flag" {
|
||||
plan
|
||||
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.en.srt")" \
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.en.srt" ]
|
||||
}
|
||||
|
||||
@test "a subtitle two folders down inside Subs/<release-name>/ still follows its episode" {
|
||||
plan
|
||||
[ "$(dst "$SEASON/Subs/The.Examples.S01E01.Pilot.1080p.x265/The.Examples.S01E01.Pilot.1080p.x265.jpn.srt")" \
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.jpn.srt" ]
|
||||
}
|
||||
|
||||
@test "a subtitle that carries the container's word in its own name still follows the episode" {
|
||||
plan
|
||||
[ "$(dst "$SEASON/The.Examples.S01E01.Pilot.1080p.x265.mkv.forced.srt")" \
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E01 - Pilot.forced.srt" ]
|
||||
}
|
||||
|
||||
@test "a season image keeps its own name; a series nfo keeps the series folder" {
|
||||
plan
|
||||
[ "$(dst "$SEASON/folder.jpg")" = "$LIB/The Examples (2010)/Season 01/folder.jpg" ]
|
||||
[ "$(dst "$SEASON/tvshow.nfo")" = "$LIB/The Examples (2010)/tvshow.nfo" ]
|
||||
}
|
||||
|
||||
@test "bonus material files as an extra, never as an episode" {
|
||||
plan
|
||||
[ "$(dst "$SEASON/Trailers/teaser.mkv")" = "$LIB/The Examples (2010)/trailers/teaser.mkv" ]
|
||||
[ -z "$(awk -F '\t' '$3 ~ /Season .*teaser/ {print $3}' "$OUT")" ]
|
||||
}
|
||||
|
||||
@test "a file only the provider numbers files at the episode it is matched to, title or no title" {
|
||||
plan
|
||||
[ "$(dst "$ANIME/[grp] Anime Pack - 05.mkv")" \
|
||||
= "$LIB/Anime Pack/Season 01/Anime Pack - S01E05.mkv" ]
|
||||
}
|
||||
|
||||
@test "a bonus folder inside a release pack belongs to the pack's show, not to its own name" {
|
||||
plan
|
||||
[ "$(dst "$ANIME/01. Extras/NCOP 01.mkv")" = "$LIB/Anime Pack/extras/NCOP 01.mkv" ]
|
||||
[ -z "$(awk -F '\t' '$3 ~ /\/01\. Extras\// {print $3}' "$OUT")" ]
|
||||
}
|
||||
|
||||
@test "a Season folder outvotes the stale row its files carry" {
|
||||
plan
|
||||
[ "$(dst "$LIB/SNL Pack/Season 19/01 Charles Barkley, Nirvana.avi")" \
|
||||
= "$LIB/SNL Pack/Season 19/SNL Pack - S19E01.avi" ]
|
||||
}
|
||||
|
||||
|
||||
@test "one file holding two episodes keeps both numbers in its name" {
|
||||
plan
|
||||
[ "$(dst "$LIB/Dupes/Show.Two.S01E02-E03.720p.mkv")" \
|
||||
= "$LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E02-E03 - Double Trouble.mkv" ]
|
||||
}
|
||||
|
||||
@test "two files claiming one episode: one keeps the name, the other parks" {
|
||||
plan
|
||||
n=$(grep -c " $LIB/Show Two (2016)/Season 01/Show Two (2016) - S01E01 - Two Pilots.mkv " "$OUT")
|
||||
[ "$n" -eq 1 ]
|
||||
run dst "$LIB/Dupes/Show Two S01E01.mkv"
|
||||
[[ "$output" == "$PARK/"* ]]
|
||||
run dst "$LIB/Dupes/Show.Two.S01E01.720p.other.mkv"
|
||||
[[ "$output" == "$LIB/Show Two (2016)/Season 01/"* ]]
|
||||
}
|
||||
|
||||
@test "a file no authority numbers is parked, never renamed into the library" {
|
||||
plan
|
||||
run dst "$LIB/Orphan Tapes/clip001.mp4"
|
||||
[[ "$output" == "$PARK/Orphan Tapes/clip001.mp4" ]]
|
||||
grep -q "clip001.mp4" "$EXC"
|
||||
}
|
||||
|
||||
@test "a long file the movie provider knows leaves the TV library for the movie one" {
|
||||
printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \
|
||||
"$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED"
|
||||
python3 -c 'import json, sys
|
||||
json.dump({"movie:some documentary":
|
||||
{"name": "Some Documentary Special", "year": 2015, "id": 55}},
|
||||
open(sys.argv[1], "w"))' "$CACHE"
|
||||
plan
|
||||
[ "$(dst "$LIB/Longform/Some Documentary Special (2015).mkv")" \
|
||||
= "$MOVIES/Some Documentary Special (2015)/Some Documentary Special (2015).mkv" ]
|
||||
}
|
||||
|
||||
@test "a long file no movie provider knows is parked, not moved into Movies" {
|
||||
printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \
|
||||
"$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED"
|
||||
plan
|
||||
run dst "$LIB/Longform/Some Documentary Special (2015).mkv"
|
||||
[[ "$output" == "$PARK/"* ]]
|
||||
}
|
||||
|
||||
@test "a movie answer the file does not itself name stays in the review pile" {
|
||||
# TMDb's top hit for a truncated query is a NEAR miss, not an answer: "mobile
|
||||
# suit gundam 00" comes back as Special Edition I while the file is Special
|
||||
# Edition III, and a one-word query "russia" comes back as a 1963 Bond film. A
|
||||
# file leaves the TV library only when it states the title the provider names.
|
||||
printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \
|
||||
"$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED"
|
||||
python3 -c 'import json, sys
|
||||
json.dump({"movie:some documentary":
|
||||
{"name": "Some Documentary Special Part Two", "year": "2015", "id": 55}},
|
||||
open(sys.argv[1], "w"))' "$CACHE"
|
||||
plan
|
||||
run dst "$LIB/Longform/Some Documentary Special (2015).mkv"
|
||||
[[ "$output" == "$PARK/"* ]]
|
||||
run awk -F '\t' -v m="$MOVIES/" '$1 == "move" && index($3, m) == 1 {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "a movie answer from another year is not that film" {
|
||||
# The year the file states outranks a provider's eagerness: the 2015 special is
|
||||
# not the 1999 one, however alike their names are.
|
||||
printf '8000\t%s\t7200\thevc\t1920x1080\t23.976\t4000000\tyuv420p10le\taac:eng\t0\t0\tmatroska,webm\n' \
|
||||
"$LIB/Longform/Some Documentary Special (2015).mkv" > "$PROBED"
|
||||
python3 -c 'import json, sys
|
||||
json.dump({"movie:some documentary":
|
||||
{"name": "Some Documentary Special", "year": "1999", "id": 55}},
|
||||
open(sys.argv[1], "w"))' "$CACHE"
|
||||
plan
|
||||
run dst "$LIB/Longform/Some Documentary Special (2015).mkv"
|
||||
[[ "$output" == "$PARK/"* ]]
|
||||
}
|
||||
|
||||
@test "the plan creates every folder before the move that needs it" {
|
||||
plan
|
||||
python3 - "$OUT" "$LIB" "$MOVIES" "$PARK" <<'PY'
|
||||
import os, sys
|
||||
plan, known = sys.argv[1], set(sys.argv[2:])
|
||||
made, bad = set(), []
|
||||
for line in open(plan).read().splitlines()[1:]:
|
||||
action, _src, dest, _kind = line.split("\t")
|
||||
if action == "mkdir":
|
||||
made.add(dest)
|
||||
elif action == "move":
|
||||
d = os.path.dirname(dest)
|
||||
while d not in known and d != os.path.dirname(d):
|
||||
if d not in made:
|
||||
bad.append(f"{dest}: {d} was never made")
|
||||
d = os.path.dirname(d)
|
||||
print("\n".join(bad[:5]))
|
||||
sys.exit(1 if bad else 0)
|
||||
PY
|
||||
}
|
||||
|
||||
@test "no move lands outside the library or the movie library" {
|
||||
plan
|
||||
run awk -F '\t' -v l="$LIB" -v m="$MOVIES" \
|
||||
'$1 == "move" && $3 !~ "^" l "/" && $3 !~ "^" m "/" {print $3}' "$OUT"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "the executor preflights the plan and no target is claimed twice" {
|
||||
plan
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$BATS_TEST_TMPDIR" \
|
||||
--journal "$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
[ "$status" -eq 0 ]
|
||||
dup=$(awk -F '\t' '$1 == "move" {print $3}' "$OUT" | sort | uniq -d)
|
||||
[ -z "$dup" ]
|
||||
}
|
||||
|
||||
@test "re-planning a library the plan already satisfies moves nothing" {
|
||||
plan
|
||||
# Applying the plan and re-censusing is exactly what the operator does next:
|
||||
# every authority file is keyed by path, so all three are replayed through the
|
||||
# plan the way a fresh census would see them. (The enum has no header; the
|
||||
# titles export does, and its header row is not a path so it survives.)
|
||||
python3 - "$OUT" "$ENUM" "$TITLES" "$PROBED" <<'PY'
|
||||
import os, sys
|
||||
plan = sys.argv[1]
|
||||
moves = {}
|
||||
for line in open(plan).read().splitlines()[1:]:
|
||||
action, src, dest, _kind = line.split("\t")
|
||||
if action == "move":
|
||||
moves[src] = dest
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
os.rename(src, dest)
|
||||
for path, col in ((sys.argv[2], 2), (sys.argv[3], 0), (sys.argv[4], 1)):
|
||||
if not os.path.getsize(path):
|
||||
continue
|
||||
rows = [line.rstrip("\n").split("\t") for line in open(path)]
|
||||
with open(path, "w") as fh:
|
||||
for f in rows:
|
||||
f[col] = moves.get(f[col], f[col])
|
||||
fh.write("\t".join(f) + "\n")
|
||||
PY
|
||||
plan
|
||||
[ -z "$(awk -F '\t' '$1 == "move" {print $2}' "$OUT")" ]
|
||||
}
|
||||
|
||||
# Specials and the year-shaped season. Jellyfin's contract puts specials in
|
||||
# Season 00, and its date matcher hands back rows whose "season" is the AIRDATE
|
||||
# YEAR. Both used to park — 1,412 and 1,033 files of one 52k plan — although one
|
||||
# is a legal season and the other is renumberable from the title tvdb states.
|
||||
@test "a provider's season 0 is a special, so it files under Season 00" {
|
||||
file "$LIB/Show Bumps/Best Of The Examples.mkv" 5000
|
||||
title_row "$LIB/Show Bumps/Best Of The Examples.mkv" "The Examples" 2010 0 2
|
||||
plan
|
||||
[ "$(dst "$LIB/Show Bumps/Best Of The Examples.mkv")" \
|
||||
= "$LIB/The Examples (2010)/Season 00/The Examples (2010) - S00E02.mkv" ]
|
||||
}
|
||||
|
||||
# These three tests carry a tvdb id of their own. The fixture's box set is 78850 and
|
||||
# ONE id means ONE show: reusing it here would merge these files into that series and
|
||||
# the duplicate rule would park them before any numbering question is asked.
|
||||
@test "a row whose season is a year is renumbered by the title tvdb states" {
|
||||
printf '{"tvdb:series:4242": %s, "tvdb:episodes:4242": %s}\n' \
|
||||
'{"name": "The Examples", "year": "2010"}' \
|
||||
'{"1/7": "Best Of"}' > "$CACHE"
|
||||
file "$LIB/Show Bumps/Best Of The Examples.mkv" 5000
|
||||
title_row "$LIB/Show Bumps/Best Of The Examples.mkv" \
|
||||
"The Examples" 2010 2010 11 "" "" "Best Of" "4242"
|
||||
plan
|
||||
want="$LIB/The Examples (2010) [tvdbid-4242]/Season 01"
|
||||
want+="/The Examples (2010) - S01E07 - Best Of.mkv"
|
||||
[ "$(dst "$LIB/Show Bumps/Best Of The Examples.mkv")" = "$want" ]
|
||||
}
|
||||
|
||||
@test "a title tvdb gives twice never renumbers an episode" {
|
||||
printf '{"tvdb:series:4242": %s, "tvdb:episodes:4242": %s}\n' \
|
||||
'{"name": "The Examples", "year": "2010"}' \
|
||||
'{"1/7": "Best Of", "2/3": "Best Of"}' > "$CACHE"
|
||||
file "$LIB/Show Bumps/Best Of The Examples.mkv" 5000
|
||||
title_row "$LIB/Show Bumps/Best Of The Examples.mkv" \
|
||||
"The Examples" 2010 2010 11 "" "" "Best Of" "4242"
|
||||
plan_park
|
||||
run awk -F '\t' -v p="$OUTSIDE/" '$1 == "move" && index($3, p) == 1 {print $3}' "$OUT"
|
||||
[[ "$output" == *"Best Of The Examples.mkv"* ]]
|
||||
}
|
||||
|
||||
|
||||
# A release pack can carry a number that is not an episode at all: two files of
|
||||
# the Attack on Titan box set on the array are named S03E2019 and S04E47370355,
|
||||
# and the provider rows the server recorded for them say the same junk. Reading
|
||||
# either as "the first three digits" files the box set under E201 and E473 and
|
||||
# steals those episode slots from their real owners, so a run too long to be an
|
||||
# episode is no marker and no number: the file is renumbered from tvdb's title
|
||||
# when the series' id answers it, and parks when nothing answers.
|
||||
@test "an episode number longer than three digits never files as an episode" {
|
||||
file "$LIB/Junk Numbers/Attack Pack S03E2019.mkv" 5000
|
||||
title_row "$LIB/Junk Numbers/Attack Pack S03E2019.mkv" "Attack Pack" 2013 3 2019
|
||||
plan_park
|
||||
run grep -c -- "- S03E201 -" "$OUT"
|
||||
[ "$output" = "0" ]
|
||||
[[ "$(dst "$LIB/Junk Numbers/Attack Pack S03E2019.mkv")" == "$OUTSIDE/"* ]]
|
||||
grep -q "a provider row of season 3 episode 2019" "$EXC"
|
||||
}
|
||||
|
||||
@test "a provider episode too long to be one is renumbered by the title tvdb states" {
|
||||
printf '{"tvdb:series:4242": %s, "tvdb:episodes:4242": %s}\n' \
|
||||
'{"name": "Attack Pack", "year": "2013"}' \
|
||||
'{"3/7": "A Title tvdb Holds"}' > "$CACHE"
|
||||
file "$LIB/Junk Numbers/Attack Pack S03E2019.mkv" 5000
|
||||
title_row "$LIB/Junk Numbers/Attack Pack S03E2019.mkv" \
|
||||
"Attack Pack" 2013 3 2019 "" "" "A Title tvdb Holds" "4242"
|
||||
plan
|
||||
want="$LIB/Attack Pack (2013) [tvdbid-4242]/Season 03"
|
||||
want+="/Attack Pack (2013) - S03E07 - A Title tvdb Holds.mkv"
|
||||
[ "$(dst "$LIB/Junk Numbers/Attack Pack S03E2019.mkv")" = "$want" ]
|
||||
}
|
||||
|
||||
@test "a resolution glued to the marker still reads as one episode" {
|
||||
file "$LIB/Junk Numbers/Glued Mark S01E02-1080p.mkv" 5000
|
||||
title_row "$LIB/Junk Numbers/Glued Mark S01E02-1080p.mkv" \
|
||||
"The Examples" 2010 1 2 "" "" "Second Look"
|
||||
plan
|
||||
[ "$(dst "$LIB/Junk Numbers/Glued Mark S01E02-1080p.mkv")" \
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E02 - Second Look.mkv" ]
|
||||
}
|
||||
|
||||
# Anime numbering is the one place a tvdb title can be confidently wrong: a pack
|
||||
# numbered by its release group still names S05E01, and tvdb answers that slot with
|
||||
# a different episode. A season the authority does not hold, or holds fewer of than
|
||||
# the pack books, is CONTRADICTED: the rename still happens, the title does not.
|
||||
@test "a season tvdb contradicts keeps the rename but never its titles" {
|
||||
printf '{"tvdb:series:4242": %s, "tvdb:episodes:4242": %s}\n' \
|
||||
'{"name": "Anime Gate", "year": "2010"}' \
|
||||
'{"1/1": "Pilot"}' > "$CACHE"
|
||||
file "$LIB/Gate Pack/Season 01/one S01E01.mkv" 5000
|
||||
file "$LIB/Gate Pack/Season 05/two S05E01.mkv" 5000
|
||||
file "$LIB/Gate Pack/Season 05/three S05E02.mkv" 5000
|
||||
title_row "$LIB/Gate Pack/Season 01/one S01E01.mkv" \
|
||||
"Anime Gate" 2010 1 1 "" "" "" "4242"
|
||||
title_row "$LIB/Gate Pack/Season 05/two S05E01.mkv" \
|
||||
"Anime Gate" 2010 5 1 "" "" "" "4242"
|
||||
title_row "$LIB/Gate Pack/Season 05/three S05E02.mkv" \
|
||||
"Anime Gate" 2010 5 2 "" "" "" "4242"
|
||||
plan
|
||||
want="$LIB/Anime Gate (2010) [tvdbid-4242]/Season 01"
|
||||
want+="/Anime Gate (2010) - S01E01 - Pilot.mkv"
|
||||
[ "$(dst "$LIB/Gate Pack/Season 01/one S01E01.mkv")" = "$want" ]
|
||||
want="$LIB/Anime Gate (2010) [tvdbid-4242]/Season 05"
|
||||
want+="/Anime Gate (2010) - S05E01.mkv"
|
||||
[ "$(dst "$LIB/Gate Pack/Season 05/two S05E01.mkv")" = "$want" ]
|
||||
grep -A6 -i "numbering tvdb contradicts" "$EXC" | grep -q "Anime Gate"
|
||||
}
|
||||
|
||||
# Tvdb's default episode query answers at most 500 rows and states no total, so a
|
||||
# list that long is a PAGE, not a list: One Piece (1999), Crayon Shin-chan (1992)
|
||||
# and Detective Conan (1996) each answer exactly 500 while holding over a thousand.
|
||||
# Read as the whole truth, that page contradicts every long-running show's pack
|
||||
# and strips the tvdb title from all of it.
|
||||
@test "an episode list the authority paginates never contradicts the pack" {
|
||||
python3 - "$CACHE" <<'PY'
|
||||
import json, sys
|
||||
json.dump({"tvdb:series:4242": {"name": "Long Run", "year": "1999"},
|
||||
"tvdb:episodes:4242": {f"1/{n}": f"Episode {n}"
|
||||
for n in range(1, 501)}},
|
||||
open(sys.argv[1], "w"))
|
||||
PY
|
||||
for n in $(seq 1 501); do
|
||||
ep=$(printf '%02d' "$n")
|
||||
file "$LIB/Long Pack/Season 01/long $ep.mkv" 1000
|
||||
title_row "$LIB/Long Pack/Season 01/long $ep.mkv" \
|
||||
"Long Run" 1999 1 "$n" "" "" "" "4242"
|
||||
done
|
||||
plan
|
||||
want="$LIB/Long Run (1999) [tvdbid-4242]/Season 01"
|
||||
want+="/Long Run (1999) - S01E01 - Episode 1.mkv"
|
||||
[ "$(dst "$LIB/Long Pack/Season 01/long 01.mkv")" = "$want" ]
|
||||
run grep -i "numbering tvdb contradicts" "$EXC"
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------- the operator map
|
||||
# Most of this library carries no tvdb id anywhere: the authority titles anime by
|
||||
# its own record — in the show's language — and the English name rides in aliases
|
||||
# the planner's name search never reads, so a search that demands tvdb's own
|
||||
# spelling cannot reach it. The operator's map is the other door: one line per
|
||||
# folder, `folder <TAB> tvdb id <TAB> note`, each id checked by hand against the
|
||||
# record it names. It outranks a provider id, it merges the release folders that
|
||||
# are one show, and every folder it touched is named in the exceptions: one wrong
|
||||
# row there is 1,268 renames to the wrong show, so nothing about it is silent.
|
||||
|
||||
plan_map() { # the plan, with the operator's map in hand
|
||||
run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --titles "$TITLES" \
|
||||
--probed "$PROBED" --tmdb-cache "$CACHE" --movies "$MOVIES" \
|
||||
--tvdb-ids "$1" --out "$OUT" --exceptions "$EXC"
|
||||
}
|
||||
|
||||
@test "a folder the operator mapped to a tvdb id renames to that record" {
|
||||
printf '{"tvdb:series:331753": %s}\n' \
|
||||
'{"name": "Black Clover", "year": "2017"}' > "$CACHE"
|
||||
{
|
||||
printf '# the note column carries why the name search cannot reach it\n'
|
||||
printf 'Black Clover 001\t331753\tthe record holds no alias\n'
|
||||
} > "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
file "$LIB/Black Clover 001/Season 01/Black.Clover.001.mkv" 5000
|
||||
title_row "$LIB/Black Clover 001/Season 01/Black.Clover.001.mkv" \
|
||||
"Black Clover 001" "" 1 1
|
||||
plan_map "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
want="$LIB/Black Clover (2017) [tvdbid-331753]/Season 01"
|
||||
want+="/Black Clover (2017) - S01E01.mkv"
|
||||
[ "$(dst "$LIB/Black Clover 001/Season 01/Black.Clover.001.mkv")" = "$want" ]
|
||||
}
|
||||
|
||||
@test "release folders one id maps are ONE series folder" {
|
||||
printf '{"tvdb:series:81797": %s}\n' \
|
||||
'{"name": "One Piece", "year": "1999"}' > "$CACHE"
|
||||
{
|
||||
printf '[Anime Time] One Piece (0001-1071)\t81797\tbare "One Piece" is the 2023 live action\n'
|
||||
printf 'One Piece Ship Collection\t81797\tthe same record, split by release\n'
|
||||
} > "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
file "$LIB/[Anime Time] One Piece (0001-1071)/One.Piece.0001.mkv" 5000
|
||||
file "$LIB/One Piece Ship Collection/One.Piece.0002.mkv" 6000
|
||||
title_row "$LIB/[Anime Time] One Piece (0001-1071)/One.Piece.0001.mkv" \
|
||||
"One Piece" 1999 1 1
|
||||
title_row "$LIB/One Piece Ship Collection/One.Piece.0002.mkv" \
|
||||
"One Piece Ship Collection" "" 1 2
|
||||
plan_map "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
want="$LIB/One Piece (1999) [tvdbid-81797]/Season 01"
|
||||
want+="/One Piece (1999) - S01E01.mkv"
|
||||
[ "$(dst "$LIB/[Anime Time] One Piece (0001-1071)/One.Piece.0001.mkv")" \
|
||||
= "$want" ]
|
||||
want="$LIB/One Piece (1999) [tvdbid-81797]/Season 01"
|
||||
want+="/One Piece (1999) - S01E02.mkv"
|
||||
[ "$(dst "$LIB/One Piece Ship Collection/One.Piece.0002.mkv")" = "$want" ]
|
||||
}
|
||||
|
||||
@test "the map outranks a provider id and says which two disagreed" {
|
||||
printf '{"tvdb:series:4242": %s, "tvdb:series:78850": %s}\n' \
|
||||
'{"name": "Real Ship", "year": "1999"}' \
|
||||
'{"name": "Box Junk", "year": "2016"}' > "$CACHE"
|
||||
printf '%s\t4242\thand-verified against the record\n' \
|
||||
"Show Box (2016) Season 1 S01 (1080p WEB-DL)" > "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
plan_map "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
dst "$BOX/Show.Box.S01E01.1080p.WEB-DL.mkv" \
|
||||
| grep -q "^$LIB/Real Ship (1999) \[tvdbid-4242\]/Season 01/"
|
||||
run grep -E '78850.*4242|4242.*78850' "$EXC"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "a map id the authority never answered keeps the provider's name and is named" {
|
||||
printf 'Black Clover 001\t331753\ttvdb never answered\n' \
|
||||
> "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
file "$LIB/Black Clover 001/Season 01/Black.Clover.001.mkv" 5000
|
||||
title_row "$LIB/Black Clover 001/Season 01/Black.Clover.001.mkv" \
|
||||
"Black Clover 001" "" 1 1
|
||||
plan_map "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
dst "$LIB/Black Clover 001/Season 01/Black.Clover.001.mkv" \
|
||||
| grep -q "^$LIB/Black Clover 001 \[tvdbid-331753\]/Season 01/"
|
||||
run grep -q "331753" "$EXC"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "a map row for a folder the array no longer holds is reported" {
|
||||
printf 'Gone Pack\t4242\tmoved away\n' > "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
plan_map "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
run grep "Gone Pack" "$EXC"
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "a map row whose folder is on the array but never became a series says so" {
|
||||
# The array holds three folders exactly like this one: a credits/extras pack
|
||||
# the operator mapped to a record, whose every file is bonus material with no
|
||||
# series to own it. The folder IS on the array, and telling the operator the
|
||||
# pack moved sends them hunting for a folder that never left.
|
||||
printf 'Credit Pack\t114801\tthe record tvdb names\n' > "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
file "$LIB/Credit Pack/01. Extras/NCOP 01.mkv" 1200
|
||||
plan_map "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
grep -q "bonus material with no series to own it" "$EXC"
|
||||
grep -q "never became a series" "$EXC"
|
||||
run grep "no folder on the array" "$EXC"
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "a map line whose id is not a number stops the plan" {
|
||||
printf 'Black Clover 001\tnotanid\ttypo\n' > "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
file "$LIB/Black Clover 001/Season 01/Black.Clover.001.mkv" 5000
|
||||
title_row "$LIB/Black Clover 001/Season 01/Black.Clover.001.mkv" \
|
||||
"Black Clover 001" "" 1 1
|
||||
plan_map "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"notanid"* ]]
|
||||
}
|
||||
|
||||
# The map the fleet ships is scripts/media-shows-tvdb-ids.tsv, and the planner
|
||||
# reads it by default: an unusable line there stops every plan the operator runs
|
||||
# from the checkout, so the seam validates the real file, not a fixture copy.
|
||||
@test "the shipped operator map parses and never stops a plan" {
|
||||
MAPS="$REPO_ROOT/scripts/media-shows-tvdb-ids.tsv"
|
||||
run awk -F '\t' '!/^#/ && NF {
|
||||
if (NF < 2 || $2 !~ /^[0-9]+$/) { print "unusable row: " $0; bad = 1 }
|
||||
} END { exit bad }' "$MAPS"
|
||||
[ "$status" -eq 0 ]
|
||||
dup=$(awk -F '\t' '!/^#/ && NF {print $1}' "$MAPS" | sort | uniq -d)
|
||||
[ -z "$dup" ]
|
||||
plan_map "$MAPS"
|
||||
[ "$status" -eq 0 ]
|
||||
grep -qi "operator's map" "$EXC"
|
||||
}
|
||||
|
||||
@test "a map that names one folder twice stops the plan" {
|
||||
printf '{"tvdb:series:78850": %s}\n' \
|
||||
'{"name": "Naruto", "year": "2002"}' > "$CACHE"
|
||||
{
|
||||
printf 'Naruto Complete Series\t78850\tthe dub pack\n'
|
||||
printf 'Naruto Complete Series\t331753\ta paste of another row\n'
|
||||
} > "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
file "$LIB/Naruto Complete Series/Season 01/naruto 01.mkv" 5000
|
||||
title_row "$LIB/Naruto Complete Series/Season 01/naruto 01.mkv" \
|
||||
"Naruto Complete Series" "" 1 1
|
||||
plan_map "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"named twice"* ]]
|
||||
}
|
||||
Executable
+282
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: scripts/media-verify.py — the replay of a relayout journal against a
|
||||
# media server's OWN database, which is the documented proof that a rename
|
||||
# landed (a scan answers 204 whether or not it finds anything).
|
||||
#
|
||||
# Every test states what the database holds and asserts what the command prints
|
||||
# and returns; nothing here reaches into the script's internals. The fixtures are
|
||||
# minimal sqlite files with the columns the two servers really use, so the seam
|
||||
# fails if the tool ever reads the wrong table, forgets Plex's `deleted_at`
|
||||
# filter, or forgets that Jellyfin stores the container-side path (/data/…) while
|
||||
# Plex stores the host path verbatim.
|
||||
|
||||
load helpers
|
||||
|
||||
VERIFY="$REPO_ROOT/scripts/media-verify.py"
|
||||
|
||||
# jf_mk <db> — the Jellyfin shape: one BaseItems table, Path is the column.
|
||||
jf_mk() {
|
||||
python3 - "$1" <<'PY'
|
||||
import sqlite3, sys
|
||||
con = sqlite3.connect(sys.argv[1])
|
||||
con.execute("create table BaseItems (Id INTEGER PRIMARY KEY, Path TEXT, Type TEXT,"
|
||||
" ParentId INTEGER, Name TEXT)")
|
||||
con.commit()
|
||||
PY
|
||||
}
|
||||
|
||||
# jf_row <db> <path> — one item the server believes it has.
|
||||
jf_row() {
|
||||
python3 - "$1" "$2" <<'PY'
|
||||
import sqlite3, sys
|
||||
con = sqlite3.connect(sys.argv[1])
|
||||
con.execute("insert into BaseItems (Path, Type) values (?, 'x')", (sys.argv[2],))
|
||||
con.commit()
|
||||
PY
|
||||
}
|
||||
|
||||
# px_mk <db> — the Plex shape: media_parts joined up to metadata_items.
|
||||
px_mk() {
|
||||
python3 - "$1" <<'PY'
|
||||
import sqlite3, sys
|
||||
con = sqlite3.connect(sys.argv[1])
|
||||
con.execute("create table metadata_items (id INTEGER PRIMARY KEY, title TEXT,"
|
||||
" deleted_at TEXT)")
|
||||
con.execute("create table media_items (id INTEGER PRIMARY KEY, metadata_item_id"
|
||||
" INTEGER, container TEXT)")
|
||||
con.execute("create table media_parts (id INTEGER PRIMARY KEY, media_item_id"
|
||||
" INTEGER, file TEXT, deleted_at TEXT)")
|
||||
con.commit()
|
||||
PY
|
||||
}
|
||||
|
||||
# px_row <db> <path> [DELETED_AT] — one part; a non-empty DELETED_AT is a row a
|
||||
# deleted scan left behind, which must never count as proof.
|
||||
px_row() {
|
||||
python3 - "$1" "$2" "${3:-}" <<'PY'
|
||||
import sqlite3, sys
|
||||
db, path, gone = sys.argv[1], sys.argv[2], sys.argv[3] or None
|
||||
con = sqlite3.connect(db)
|
||||
cur = con.execute("insert into metadata_items (title, deleted_at) values ('t', ?)",
|
||||
(gone,))
|
||||
mi = cur.lastrowid
|
||||
cur = con.execute("insert into media_items (metadata_item_id, container)"
|
||||
" values (?, 'mkv')", (mi,))
|
||||
con.execute("insert into media_parts (media_item_id, file, deleted_at)"
|
||||
" values (?, ?, NULL)", (cur.lastrowid, path))
|
||||
con.commit()
|
||||
PY
|
||||
}
|
||||
|
||||
# move_row <src> <dst> — one move in the journal.
|
||||
move_row() {
|
||||
printf '{"op": "move", "src": "%s", "dst": "%s"}\n' "$1" "$2" >> "$JOURNAL"
|
||||
}
|
||||
|
||||
setup() {
|
||||
SHARE="$BATS_TEST_TMPDIR/mnt/entropy"
|
||||
JFDB="$BATS_TEST_TMPDIR/jellyfin.db"
|
||||
PXDB="$BATS_TEST_TMPDIR/plex.db"
|
||||
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
||||
: > "$JOURNAL"
|
||||
jf_mk "$JFDB"
|
||||
px_mk "$PXDB"
|
||||
}
|
||||
|
||||
@test "every move that the database holds at its destination is reported landed" {
|
||||
move_row "$SHARE/Movies/Old Name (1999).mkv" "$SHARE/Movies/Old Name (1999)/Old Name (1999).mkv"
|
||||
move_row "$SHARE/Movies/Two.mkv" "$SHARE/Movies/Two/Two.mkv"
|
||||
# Jellyfin stores the container-side path: /data stands for the share.
|
||||
jf_row "$JFDB" "/data/Movies/Old Name (1999)/Old Name (1999).mkv"
|
||||
jf_row "$JFDB" "/data/Movies/Two/Two.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[0]}" = "jellyfin: 2 moves checked" ]
|
||||
[ "${lines[1]}" = "landed: 2" ]
|
||||
[ "${lines[2]}" = "still at src: 0" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||
}
|
||||
|
||||
@test "a database that has not rescanned yet is refused, naming the stale rows" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
jf_row "$JFDB" "/data/Movies/Old.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[1]}" = "landed: 0" ]
|
||||
[ "${lines[2]}" = "still at src: 1" ]
|
||||
[[ "$output" == *"$SHARE/Movies/Old.mkv"* ]]
|
||||
}
|
||||
|
||||
@test "a moved file no row of the database names is reported missing" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 1" ]
|
||||
[[ "$output" == *"$SHARE/Movies/Old/Old.mkv"* ]]
|
||||
}
|
||||
|
||||
@test "Plex stores the host path verbatim, so a row at the destination lands" {
|
||||
move_row "$SHARE/TVShows/Show/Season 01/Show S01E01.mkv" \
|
||||
"$SHARE/TVShows/Show (2002)/Season 01/Show (2002) S01E01.mkv"
|
||||
px_row "$PXDB" "$SHARE/TVShows/Show (2002)/Season 01/Show (2002) S01E01.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind plex --db "$PXDB" \
|
||||
--share "$SHARE"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[0]}" = "plex: 1 moves checked" ]
|
||||
[ "${lines[1]}" = "landed: 1" ]
|
||||
}
|
||||
|
||||
@test "a Plex row a deleted scan left behind is not proof a move landed" {
|
||||
move_row "$SHARE/TVShows/Old S01E01.mkv" "$SHARE/TVShows/Show/Old S01E01.mkv"
|
||||
px_row "$PXDB" "$SHARE/TVShows/Show/Old S01E01.mkv" "2026-10-30 00:00:00"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind plex --db "$PXDB" \
|
||||
--share "$SHARE"
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[1]}" = "landed: 0" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 1" ]
|
||||
}
|
||||
|
||||
@test "a database without the columns that kind needs is an error, not a zero" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$PXDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 2 ]
|
||||
[[ "$output" == *"$PXDB"* ]]
|
||||
[[ "$output" == *"BaseItems"* ]]
|
||||
}
|
||||
|
||||
@test "only move rows are counted, so folder operations cannot inflate the tally" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
jf_row "$JFDB" "/data/Movies/Old/Old.mkv"
|
||||
printf '{"op": "mkdir", "dst": "%s/Movies/Old"}\n' "$SHARE" >> "$JOURNAL"
|
||||
printf '{"op": "rmdir", "path": "%s/Movies/Empty"}\n' "$SHARE" >> "$JOURNAL"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[0]}" = "jellyfin: 1 moves checked" ]
|
||||
}
|
||||
|
||||
@test "a journal with no moves is clean, not a failure" {
|
||||
printf '{"op": "mkdir", "dst": "%s/Movies/Old"}\n' "$SHARE" >> "$JOURNAL"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[1]}" = "landed: 0" ]
|
||||
}
|
||||
|
||||
@test "the offender list stops at --show so a stale library stays readable" {
|
||||
for n in 1 2 3 4; do
|
||||
move_row "$SHARE/Movies/F$n.mkv" "$SHARE/Movies/F$n/F$n.mkv"
|
||||
done
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data --show 2
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 4" ]
|
||||
[[ "$output" == *"$SHARE/Movies/F1/F1.mkv"* ]]
|
||||
[[ "$output" != *"$SHARE/Movies/F4/F4.mkv"* ]]
|
||||
}
|
||||
|
||||
@test "the missing rows are counted by extension, so a sidecar is not mistaken for a lost film" {
|
||||
move_row "$SHARE/Music/Album/Mandatory Fun.log" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Mandatory Fun.log"
|
||||
move_row "$SHARE/Music/Album/Film.mkv" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Film.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 2" ]
|
||||
[[ "$output" == *"missing by extension: .log 1, .mkv 1"* ]]
|
||||
}
|
||||
|
||||
@test "--ignore-ext keeps a file the server never indexes out of the verdict" {
|
||||
move_row "$SHARE/Music/Album/Song.flac" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Song.flac"
|
||||
jf_row "$JFDB" "/data/Music/Weird Al/Mandatory Fun (2014)/Song.flac"
|
||||
move_row "$SHARE/Music/Album/Mandatory Fun.cue" "$SHARE/Music/Weird Al/Mandatory Fun (2014)/Mandatory Fun.cue"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data --ignore-ext log,cue
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[1]}" = "landed: 1" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||
[[ "$output" == *"not indexed (extension ignored): 1"* ]]
|
||||
}
|
||||
|
||||
# wal_case <db> — a Jellyfin database whose NEW row lives only in the
|
||||
# write-ahead log: the server has committed the rename but not checkpointed it,
|
||||
# which is the normal state of a library a scan is writing. The holder
|
||||
# connection keeps the WAL from folding into the main file, and the verifier
|
||||
# runs as a real subprocess so it faces the same files Plex and Jellyfin leave.
|
||||
wal_case() {
|
||||
python3 - "$JFDB" "$JOURNAL" "$SHARE" "$VERIFY" <<'PY'
|
||||
import sqlite3, subprocess, sys
|
||||
db, journal, share, verify = sys.argv[1:5]
|
||||
w = sqlite3.connect(db)
|
||||
w.execute("pragma journal_mode=wal")
|
||||
w.execute("pragma wal_autocheckpoint=0")
|
||||
w.execute("insert into BaseItems (Path, Type) values ('/data/Movies/Old.mkv', 'x')")
|
||||
w.commit()
|
||||
w.execute("pragma wal_checkpoint(truncate)")
|
||||
holder = sqlite3.connect(db) # last-connection close would checkpoint
|
||||
w.execute("update BaseItems set Path='/data/Movies/Old/Old.mkv'")
|
||||
w.commit()
|
||||
r = subprocess.run([sys.executable, verify, "--journal", journal,
|
||||
"--db-kind", "jellyfin", "--db", db,
|
||||
"--share", share, "--container", "/data"],
|
||||
capture_output=True, text=True)
|
||||
print(r.stdout, end="")
|
||||
sys.exit(r.returncode)
|
||||
PY
|
||||
}
|
||||
|
||||
@test "a row the server committed but has not checkpointed is proof, not a miss" {
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
run wal_case
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${lines[1]}" = "landed: 1" ]
|
||||
[ "${lines[3]}" = "missing from the DB: 0" ]
|
||||
}
|
||||
|
||||
# The DB alone cannot tell a lost file from a sidecar: both are "no row at the
|
||||
# destination". The tool therefore stats the two paths and separates the classes
|
||||
# the runbook names, so an operator never reads a `.pdf` as a lost film or,
|
||||
# worse, reads a real loss as a sidecar.
|
||||
@test "a file the database does not name that still sits at src is the pass's own damage" {
|
||||
mkdir -p "$SHARE/Movies"
|
||||
echo film > "$SHARE/Movies/Old.mkv"
|
||||
move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 1" ]
|
||||
[ "${lines[4]}" = " on disk at dst, no row names it: 0" ]
|
||||
[ "${lines[5]}" = " REAL — file still at src: 1" ]
|
||||
[ "${lines[6]}" = " gone — at neither path: 0" ]
|
||||
[[ "$output" == *"real: $SHARE/Movies/Old.mkv"* ]]
|
||||
}
|
||||
|
||||
@test "a file at neither path is gone, and a file at dst the server ignores is not a loss" {
|
||||
move_row "$SHARE/Movies/Gone.mkv" "$SHARE/Movies/Gone/Gone.mkv"
|
||||
mkdir -p "$SHARE/Movies/Sidecar"
|
||||
echo cover > "$SHARE/Movies/Sidecar/cover.jpg"
|
||||
move_row "$SHARE/Movies/Album/cover.jpg" "$SHARE/Movies/Sidecar/cover.jpg"
|
||||
|
||||
run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
|
||||
--share "$SHARE" --container /data
|
||||
[ "$status" -eq 1 ]
|
||||
[ "${lines[3]}" = "missing from the DB: 2" ]
|
||||
[ "${lines[4]}" = " on disk at dst, no row names it: 1" ]
|
||||
[ "${lines[5]}" = " REAL — file still at src: 0" ]
|
||||
[ "${lines[6]}" = " gone — at neither path: 1" ]
|
||||
[[ "$output" == *"gone: $SHARE/Movies/Gone/Gone.mkv"* ]]
|
||||
}
|
||||
Reference in new issue
Block a user