Frank
9eb85ea1f9
Every apply and rollback printed "next: scripts/media-view.sh reap", but on Dankserver there is nothing to reap: the metadata view has never been installed (`media-view.sh status` -> NOT MOUNTED, `/var/lib/media-meta` absent) and the live quadlets still bind the array read-only. All three relayout arms tonight issued that order — 242,084 moves — and it pointed at a directory that does not exist. Where no overlay is mounted the servers bind the array itself, so the sidecars moved with the media. `--metadata-view` (default /var/lib/media-meta/view) names the overlay and the note now rides only when that path is a mountpoint. Red first: two new cases in tests/media_relayout.bats (apply and rollback); the shared helper names a real mountpoint so the pre-existing presence assertion keeps its meaning. 37 ok.
653 lines
27 KiB
Bash
Executable File
653 lines
27 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the movie-library relayout — scripts/media-census.py (the read-only
|
|
# census), scripts/media-plan.py (the plan) and scripts/media-apply.py (the only
|
|
# thing that moves files).
|
|
#
|
|
# The library is 8k files on a 100 %-full array, so the two things this seam
|
|
# exists to catch are the ones that cost a re-run of a 20-minute census: a file
|
|
# that ends up in the WRONG movie folder, and a move that destroys something.
|
|
# Everything below runs on a synthetic library in the test's own tmpdir: no
|
|
# ffprobe, no network, no server. The probe step is the one part a fixture
|
|
# cannot run (it opens real containers), so the tests feed the planner the
|
|
# probed TSV in the column order scripts/media-census.py exports — which pins
|
|
# that contract as a side effect.
|
|
#
|
|
# The plan's own gate lines (unplanned, left-unmoved, double-claim, occupied)
|
|
# are asserted as output, because they are how the operator learns the plan is
|
|
# trustworthy before pointing the executor at 24 TB.
|
|
|
|
load helpers
|
|
|
|
PLAN="$REPO_ROOT/scripts/media-plan.py"
|
|
CENSUS="$REPO_ROOT/scripts/media-census.py"
|
|
APPLY="$REPO_ROOT/scripts/media-apply.py"
|
|
TV="$REPO_ROOT/scripts/media-tv.py"
|
|
|
|
# probe_row FILE DUR [RES] [AUDIO] [SUBS] — one probed-TSV row: the twelve
|
|
# columns media-census.py probe writes (bytes, path, dur, vcodec, res, fps,
|
|
# vbit, pix, audio, subs, spare, fmt).
|
|
probe_row() {
|
|
local p="$1" dur="$2" res="${3:-1920x1080}" aud="${4:-aac+aac}" subs="${5:-0}"
|
|
printf '%s\t%s\t%s\thevc\t%s\t23.976\t5000000\t8bit\t%s\t%s\t0\tmatroska,webm\n' \
|
|
"$(stat -c %s "$p")" "$p" "$dur" "$res" "$aud" "$subs" >> "$PROBED"
|
|
}
|
|
|
|
# plex_row FILE TITLE YEAR — one Plex metadata row; the planner reads these
|
|
# three columns and treats a row without a plausible year as no match at all.
|
|
plex_row() {
|
|
printf '%s\t%s\t%s\n' "$1" "$2" "$3" >> "$PLEX"
|
|
}
|
|
|
|
# at PATH SIZE — a file with deterministic content, big enough to matter.
|
|
at() { mkdir -p "$(dirname "$1")"; head -c "$2" /dev/zero > "$1"; }
|
|
|
|
setup() {
|
|
LIB="$BATS_TEST_TMPDIR/Movies"
|
|
ENUM="$BATS_TEST_TMPDIR/enum.tsv"
|
|
PROBED="$BATS_TEST_TMPDIR/probed.tsv"
|
|
PLEX="$BATS_TEST_TMPDIR/plex.tsv"
|
|
OUT="$BATS_TEST_TMPDIR/plan.tsv"
|
|
EXC="$BATS_TEST_TMPDIR/exceptions.txt"
|
|
TVLIB="$BATS_TEST_TMPDIR/TVShows"
|
|
TVPLAN="$BATS_TEST_TMPDIR/tv.tsv"
|
|
TVJ="$BATS_TEST_TMPDIR/tv.jsonl"
|
|
: > "$PROBED"
|
|
printf 'file\tleaf_title\tleaf_year\n' > "$PLEX"
|
|
|
|
# Two films that share a box set and a word, plus their bonus material:
|
|
# the shape that made one film's featurettes land in another's folder.
|
|
at "$LIB/Alien (1979)/Alien.1979.1080p.BluRay.x265.10bit.mkv" 4000
|
|
at "$LIB/Alien (1979)/Featurettes/A Bug And A Bite - Making Alien.mkv" 3300000
|
|
at "$LIB/Alien (1979)/Extras/Extra - Logo.mkv" 900000
|
|
at "$LIB/Alien (1979)/subs/3_Eng.srt" 40000
|
|
at "$LIB/Marnie (1964)/Marnie.1964.1080p.BluRay.x265.mkv" 5000000
|
|
at "$LIB/Marnie (1964)/Featurettes/The Trouble With Marnie.mkv" 2400000
|
|
# A long extra: over 45 minutes, so only the folder may call it bonus.
|
|
at "$LIB/Marnie (1964)/Featurettes/Commentary And Archives.mkv" 2600000
|
|
# A misspelled bonus folder: no bonus vocabulary, so only the folder that
|
|
# holds the feature can decide that this is not a film.
|
|
at "$LIB/Marnie (1964)/Exrtas/Marnie - A Closer Look.mkv" 2500000
|
|
# A Plex title that carries "/" and a Plex row that was never matched.
|
|
at "$LIB/VHS 2 (2013)/VHS.2.2013.1080p.BluRay.x265.mkv" 4200000
|
|
at "$LIB/AMERICAN POP/AMERICAN.POP.2023.1080p.WEB-DL.mkv" 3100000
|
|
# Junk the layout has to park somewhere other than a movie folder.
|
|
at "$LIB/.DS_Store" 6148
|
|
at "$LIB/Alien (1979)/._Alien.1979.1080p.BluRay.x265.10bit.mkv" 220
|
|
# A bonus file whose name is a number: never a movie folder named "1".
|
|
at "$LIB/Bruce Lee Collection 1971-1981/.Extra 1.mkv" 2700000
|
|
# A TV box set sitting in the movie library. This one names each file
|
|
# "Show - SxxEyy - Episode Title", so the episode title comes AFTER the
|
|
# marker; that convention is what made 124 episodes of one series look like
|
|
# 124 different films, each under its own threshold.
|
|
for e in 01 02 03 04 05; do
|
|
at "$LIB/Monk 2002 Seasons 1 to 8/Season 1/Monk - S01E$e - Mr Monk and the Rival.mkv" 3000000
|
|
done
|
|
# The same trouble named the other way round, with no episode title at all.
|
|
for e in 01 02 03 04 05; do
|
|
at "$LIB/Big Little Lies Season 1/Big Little Lies S01E$e.mp4" 3000000
|
|
done
|
|
# A film whose own name carries a bare "Part 2": one episode-shaped token is
|
|
# not a series, and it must not be dragged into the TV pile.
|
|
at "$LIB/The Godfather Part 2 (1974)/The.Godfather.Part.2.1974.1080p.BluRay.mkv" 9000000
|
|
|
|
probe_row "$LIB/Alien (1979)/Alien.1979.1080p.BluRay.x265.10bit.mkv" 4700
|
|
probe_row "$LIB/Alien (1979)/Featurettes/A Bug And A Bite - Making Alien.mkv" 2700
|
|
probe_row "$LIB/Alien (1979)/Extras/Extra - Logo.mkv" 30
|
|
probe_row "$LIB/Marnie (1964)/Marnie.1964.1080p.BluRay.x265.mkv" 4800
|
|
probe_row "$LIB/Marnie (1964)/Featurettes/The Trouble With Marnie.mkv" 1200
|
|
probe_row "$LIB/Marnie (1964)/Featurettes/Commentary And Archives.mkv" 3000
|
|
probe_row "$LIB/Marnie (1964)/Exrtas/Marnie - A Closer Look.mkv" 2600
|
|
probe_row "$LIB/VHS 2 (2013)/VHS.2.2013.1080p.BluRay.x265.mkv" 5700
|
|
probe_row "$LIB/AMERICAN POP/AMERICAN.POP.2023.1080p.WEB-DL.mkv" 6300
|
|
probe_row "$LIB/Bruce Lee Collection 1971-1981/.Extra 1.mkv" 5220
|
|
for e in 01 02 03 04 05; do
|
|
probe_row "$LIB/Monk 2002 Seasons 1 to 8/Season 1/Monk - S01E$e - Mr Monk and the Rival.mkv" 2600
|
|
probe_row "$LIB/Big Little Lies Season 1/Big Little Lies S01E$e.mp4" 3300
|
|
done
|
|
probe_row "$LIB/The Godfather Part 2 (1974)/The.Godfather.Part.2.1974.1080p.BluRay.mkv" 6200
|
|
plex_row "$LIB/Alien (1979)/Alien.1979.1080p.BluRay.x265.10bit.mkv" "Alien" "1979"
|
|
plex_row "$LIB/Marnie (1964)/Marnie.1964.1080p.BluRay.x265.mkv" "Marnie" "1964"
|
|
plex_row "$LIB/VHS 2 (2013)/VHS.2.2013.1080p.BluRay.x265.mkv" "V/H/S/2" "2013"
|
|
# An unmatched Plex row: the leaf title echoes the file and the year is 0.
|
|
plex_row "$LIB/AMERICAN POP/AMERICAN.POP.2023.1080p.WEB-DL.mkv" "AMERICAN POP" "0"
|
|
plex_row "$LIB/The Godfather Part 2 (1974)/The.Godfather.Part.2.1974.1080p.BluRay.mkv" \
|
|
"The Godfather Part 2" "1974"
|
|
|
|
export MEDIA_ROOT="$LIB"
|
|
run python3 "$CENSUS" enumerate --root "$LIB" --out "$ENUM"
|
|
[ "$status" -eq 0 ]
|
|
}
|
|
|
|
# plan — run the planner over the fixture, keeping its plan TSV, its exception
|
|
# report and the gate lines it prints (that stdout is the operator's evidence
|
|
# that the plan accounts for every file, so the tests read it too).
|
|
plan() {
|
|
PLANOUT="$BATS_TEST_TMPDIR/plan.out"
|
|
run python3 "$PLAN" --root "$LIB" --enum "$ENUM" --probed "$PROBED" \
|
|
--plex "$PLEX" --out "$OUT" --exceptions "$EXC"
|
|
[ "$status" -eq 0 ]
|
|
printf '%s\n' "$output" > "$PLANOUT"
|
|
}
|
|
|
|
# dst_of SRC — the destination the plan gives one file.
|
|
dst_of() {
|
|
python3 -c '
|
|
import sys
|
|
for line in open(sys.argv[1]):
|
|
f = line.rstrip("\n").split("\t")
|
|
if len(f) > 2 and f[1] == sys.argv[2]:
|
|
print(f[2]); break
|
|
' "$OUT" "$1"
|
|
}
|
|
|
|
# tv_plan — the TV mover over the fixture library; its stdout is the operator's
|
|
# census of what it called a series and what it refused to touch.
|
|
tv_plan() {
|
|
run python3 "$TV" --root "$LIB" --library "$TVLIB" --out "$TVPLAN"
|
|
[ "$status" -eq 0 ]
|
|
printf '%s\n' "$output" > "$BATS_TEST_TMPDIR/tv.out"
|
|
}
|
|
|
|
# tv_apply — that plan through the same executor. The destination is a sibling
|
|
# share, so the plan is applied at the root that holds both libraries.
|
|
tv_apply() {
|
|
run python3 "$APPLY" apply --plan "$TVPLAN" --root "$BATS_TEST_TMPDIR" \
|
|
--journal "$TVJ" --apply
|
|
[ "$status" -eq 0 ]
|
|
}
|
|
|
|
# tv_dst_of SRC — the destination the TV plan gives one file, empty if none.
|
|
tv_dst_of() {
|
|
python3 -c '
|
|
import sys
|
|
for line in open(sys.argv[1]):
|
|
f = line.rstrip("\n").split("\t")
|
|
if len(f) > 2 and f[1] == sys.argv[2]:
|
|
print(f[2]); break
|
|
' "$TVPLAN" "$1"
|
|
}
|
|
|
|
@test "a film lands in Title (year)/Title (year).mkv, named by Plex" {
|
|
plan
|
|
[ "$(dst_of "$LIB/Alien (1979)/Alien.1979.1080p.BluRay.x265.10bit.mkv")" \
|
|
= "$LIB/Alien (1979)/Alien (1979).mkv" ]
|
|
}
|
|
|
|
@test "a Plex title that carries a slash becomes one folder, not a nested tree" {
|
|
plan
|
|
local d; d="$(dst_of "$LIB/VHS 2 (2013)/VHS.2.2013.1080p.BluRay.x265.mkv")"
|
|
[ "$(dirname "$d")" = "$LIB/V-H-S-2 (2013)" ]
|
|
[ ! -e "$LIB/V" ]
|
|
# the folder must be ONE mkdir: an un-sanitized name would make the
|
|
# executor build Movies/V/H/S/2 (2013) instead
|
|
run grep -F "mkdir" "$OUT"
|
|
[[ "$output" == *"$(printf '%s' "$LIB/V-H-S-2 (2013)")"* ]]
|
|
run awk -F'\t' '$1=="mkdir"{print $3}' "$OUT"
|
|
[[ "$output" != *"$LIB/V/"* ]]
|
|
}
|
|
|
|
@test "an unmatched Plex row never names a folder with a zero year" {
|
|
plan
|
|
run awk -F'\t' '{print $3}' "$OUT"
|
|
[[ "$output" != *"(0)"* ]]
|
|
# the filename's own year carries the folder, the folder and the file share
|
|
# the name (Jellyfin's rule), and the film sits at the library root rather
|
|
# than in an unsorted pile
|
|
local d; d="$(dst_of "$LIB/AMERICAN POP/AMERICAN.POP.2023.1080p.WEB-DL.mkv")"
|
|
[ "$(dirname "$(dirname "$d")")" = "$LIB" ]
|
|
[[ "$(basename "$(dirname "$d")")" == *" (2023)" ]]
|
|
[ "$(basename "$d")" = "$(basename "$(dirname "$d")").mkv" ]
|
|
}
|
|
|
|
@test "a bonus folder keeps its featurettes inside its own film, not a neighbour's" {
|
|
plan
|
|
local d
|
|
d="$(dst_of "$LIB/Alien (1979)/Featurettes/A Bug And A Bite - Making Alien.mkv")"
|
|
[ "$(dirname "$(dirname "$d")")" = "$LIB/Alien (1979)" ]
|
|
d="$(dst_of "$LIB/Marnie (1964)/Featurettes/The Trouble With Marnie.mkv")"
|
|
[ "$(dirname "$(dirname "$d")")" = "$LIB/Marnie (1964)" ]
|
|
d="$(dst_of "$LIB/Alien (1979)/Extras/Extra - Logo.mkv")"
|
|
[ "$(dirname "$(dirname "$d")")" = "$LIB/Alien (1979)" ]
|
|
# the feature-length extra is bonus too: it sits in a bonus folder and no
|
|
# metadata matched it, so it may not become a movie of its own
|
|
d="$(dst_of "$LIB/Marnie (1964)/Featurettes/Commentary And Archives.mkv")"
|
|
[ "$(dirname "$(dirname "$d")")" = "$LIB/Marnie (1964)" ]
|
|
}
|
|
|
|
@test "a folder inside a movie folder is bonus even when its name is misspelled" {
|
|
plan
|
|
local d; d="$(dst_of "$LIB/Marnie (1964)/Exrtas/Marnie - A Closer Look.mkv")"
|
|
[ "$(dirname "$(dirname "$d")")" = "$LIB/Marnie (1964)" ]
|
|
}
|
|
|
|
@test "a subtitle follows the film of the folder it sits in" {
|
|
plan
|
|
local d; d="$(dst_of "$LIB/Alien (1979)/subs/3_Eng.srt")"
|
|
[ "$(dirname "$(dirname "$d")")" = "$LIB/Alien (1979)" ]
|
|
}
|
|
|
|
@test "macOS sidecars are parked in one pile, never inside a movie folder" {
|
|
plan
|
|
local d; d="$(dst_of "$LIB/Alien (1979)/._Alien.1979.1080p.BluRay.x265.10bit.mkv")"
|
|
[[ "$(dirname "$d")" == "$LIB"/_* ]]
|
|
d="$(dst_of "$LIB/.DS_Store")"
|
|
[[ "$(dirname "$d")" == "$LIB"/_* ]]
|
|
grep -q "AppleDouble" "$EXC"
|
|
}
|
|
|
|
@test "a file whose name is a number becomes a review line, not a folder named 1" {
|
|
plan
|
|
run awk -F'\t' '$1=="mkdir"{print $3}' "$OUT"
|
|
[[ "$output" != "$LIB/1"$'\n'* ]] && [[ "$output" != "$LIB/1" ]]
|
|
[[ "$output" != *"$LIB/1/"* ]]
|
|
grep -q "UNNAMED" "$EXC"
|
|
}
|
|
|
|
@test "a box set in the movie library is quarantined, not filed as movies" {
|
|
plan
|
|
local d
|
|
d="$(dst_of "$LIB/Monk 2002 Seasons 1 to 8/Season 1/Monk - S01E01 - Mr Monk and the Rival.mkv")"
|
|
[[ "$d" == "$LIB"/_TV\ needs\ a\ TV\ library\ -\ monk* ]]
|
|
d="$(dst_of "$LIB/Big Little Lies Season 1/Big Little Lies S01E01.mp4")"
|
|
[[ "$d" == "$LIB"/_TV\ needs\ a\ TV\ library\ -\ big\ little\ lies* ]]
|
|
# one series is one pile, whatever its episodes are called
|
|
run python3 -c '
|
|
import re, sys
|
|
pile = {}
|
|
for line in open(sys.argv[1]):
|
|
f = line.rstrip("\n").split("\t")
|
|
if len(f) > 2 and f[0] == "move":
|
|
m = re.search(r"/_TV needs a TV library - ([^/]*)/", f[2] + "/")
|
|
if m:
|
|
pile[m.group(1)] = pile.get(m.group(1), 0) + 1
|
|
for k in sorted(pile):
|
|
print(f"{k}={pile[k]}")
|
|
' "$OUT"
|
|
[ "${#lines[@]}" -eq 2 ]
|
|
[[ "${lines[0]}" == big\ little\ lies*=5 ]]
|
|
[[ "${lines[1]}" == monk*=5 ]]
|
|
}
|
|
|
|
@test "a film named Part 2 is not called a series" {
|
|
plan
|
|
[ "$(dst_of "$LIB/The Godfather Part 2 (1974)/The.Godfather.Part.2.1974.1080p.BluRay.mkv")" \
|
|
= "$LIB/The Godfather Part 2 (1974)/The Godfather Part 2 (1974).mkv" ]
|
|
}
|
|
|
|
@test "the TV mover files a box set into the TV library in Jellyfin shape" {
|
|
tv_plan
|
|
tv_apply
|
|
[ -e "$TVLIB/Monk (2002)/Season 01/Monk - S01E01 - Mr Monk and the Rival.mkv" ]
|
|
[ -e "$TVLIB/Big Little Lies/Season 01/Big Little Lies S01E01.mp4" ]
|
|
[ ! -e "$LIB/Monk 2002 Seasons 1 to 8/Season 1/Monk - S01E01 - Mr Monk and the Rival.mkv" ]
|
|
# films are not the TV mover's business
|
|
[ -e "$LIB/Alien (1979)/Alien.1979.1080p.BluRay.x265.10bit.mkv" ]
|
|
}
|
|
|
|
@test "the TV mover leaves a film whose name only looks like an episode" {
|
|
tv_plan
|
|
tv_apply
|
|
local gf="$LIB/The Godfather Part 2 (1974)/The.Godfather.Part.2.1974.1080p.BluRay.mkv"
|
|
[ "$(tv_dst_of "$gf")" = "" ]
|
|
[ -e "$gf" ]
|
|
grep -q "Godfather.*not a series" "$BATS_TEST_TMPDIR/tv.out"
|
|
}
|
|
|
|
@test "the TV mover keeps an episode's own subtitle with the episode" {
|
|
at "$LIB/Monk 2002 Seasons 1 to 8/Season 1/Monk - S01E01 - Mr Monk and the Rival.srt" 40000
|
|
tv_plan
|
|
tv_apply
|
|
[ -e "$TVLIB/Monk (2002)/Season 01/Monk - S01E01 - Mr Monk and the Rival.srt" ]
|
|
}
|
|
|
|
@test "a second TV move finds nothing left to move" {
|
|
tv_plan
|
|
tv_apply
|
|
tv_plan
|
|
grep -q "0 moves" "$BATS_TEST_TMPDIR/tv.out"
|
|
}
|
|
|
|
@test "a show whose own name carries the year is not given it twice" {
|
|
local d="$LIB/Leah Remini Scientology and the Aftermath (2016)/Season 1"
|
|
for e in 01 02 03 04 05; do
|
|
at "$d/Leah Remini Scientology and the Aftermath (2016) - S01E$e - Episode.mkv" 3000000
|
|
done
|
|
tv_plan
|
|
tv_apply
|
|
[ -e "$TVLIB/Leah Remini Scientology and the Aftermath (2016)/Season 01/Leah Remini Scientology and the Aftermath (2016) - S01E01 - Episode.mkv" ]
|
|
[ ! -e "$TVLIB/Leah Remini Scientology and the Aftermath (2016) (2016)" ]
|
|
}
|
|
|
|
@test "an all-lowercase series name is written the way the library writes names" {
|
|
for e in 01 02 03 04 05; do
|
|
at "$LIB/downton abbey 2010/Season 1/downton abbey - s01e$e - episode title.mkv" 3000000
|
|
done
|
|
tv_plan
|
|
tv_apply
|
|
# the FOLDER is titled; the file keeps the name the library already holds
|
|
[ -e "$TVLIB/Downton Abbey (2010)/Season 01/downton abbey - s01e01 - episode title.mkv" ]
|
|
}
|
|
|
|
@test "the mover leaves an episode the TV library already holds" {
|
|
local src="$LIB/Monk 2002 Seasons 1 to 8/Season 1/Monk - S01E01 - Mr Monk and the Rival.mkv"
|
|
for e in 01 02 03 04 05; do
|
|
at "$TVLIB/Monk (2002)/Season 01/Monk - S01E$e - Mr Monk and the Rival.mkv" 3000000
|
|
done
|
|
tv_plan
|
|
[ -z "$(tv_dst_of "$src")" ]
|
|
[ -e "$src" ]
|
|
grep -q "Monk.*already in" "$BATS_TEST_TMPDIR/tv.out"
|
|
}
|
|
|
|
@test "an episode the TV library holds in another encode is still moved" {
|
|
for e in 01 02 03 04 05; do
|
|
at "$TVLIB/Big Little Lies/Season 01/Big Little Lies S01E$e 1080p.mp4" 4000000
|
|
done
|
|
tv_plan
|
|
tv_apply
|
|
[ -e "$TVLIB/Big Little Lies/Season 01/Big Little Lies S01E01.mp4" ]
|
|
[ -e "$TVLIB/Big Little Lies/Season 01/Big Little Lies S01E01 1080p.mp4" ]
|
|
}
|
|
|
|
@test "every file in the census is placed or kept — nothing is dropped" {
|
|
plan
|
|
python3 - "$ENUM" "$OUT" <<'PY'
|
|
import sys
|
|
enum = {l.rstrip("\n").split("\t")[2] for l in open(sys.argv[1])}
|
|
placed = set()
|
|
for line in open(sys.argv[2]):
|
|
f = line.rstrip("\n").split("\t")
|
|
if f[0] in ("move", "keep"):
|
|
placed.add(f[1])
|
|
missing = enum - placed
|
|
assert not missing, f"{len(missing)} census files with no plan row: {sorted(missing)[:3]}"
|
|
PY
|
|
grep -q "left-unmoved=0" "$PLANOUT"
|
|
grep -q "double-claim=0" "$PLANOUT"
|
|
grep -q "occupied=0" "$PLANOUT"
|
|
grep -q "bad-names=0" "$PLANOUT"
|
|
grep -q "unplanned=0" "$EXC"
|
|
}
|
|
|
|
# ---------------------------------------------------------------- executor
|
|
|
|
# apply_plan — plan, then apply for real; the journal goes in the tmpdir.
|
|
apply_plan() {
|
|
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
|
# `/` is always a mountpoint, so the overlay note behaves as it does on a
|
|
# host that runs the metadata view.
|
|
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" \
|
|
--journal "$JOURNAL" --metadata-view / --apply
|
|
}
|
|
|
|
@test "a run without --apply says what it would do and touches nothing" {
|
|
plan
|
|
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
|
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"--apply"* ]]
|
|
[ -e "$LIB/Alien (1979)/Alien.1979.1080p.BluRay.x265.10bit.mkv" ]
|
|
[ ! -e "$LIB/Alien (1979)/Alien (1979).mkv" ]
|
|
# a dry run may not even build the folders: an operator must be able to
|
|
# run it against the live library with no --apply and change nothing
|
|
[ ! -d "$LIB/V-H-S-2 (2013)" ]
|
|
[ ! -e "$JOURNAL" ]
|
|
}
|
|
|
|
@test "apply moves every planned file, keeps its bytes, and journals each move" {
|
|
plan
|
|
apply_plan
|
|
[ "$status" -eq 0 ]
|
|
[ -e "$LIB/Alien (1979)/Alien (1979).mkv" ]
|
|
[ ! -e "$LIB/Alien (1979)/Alien.1979.1080p.BluRay.x265.10bit.mkv" ]
|
|
local planned moved
|
|
planned=$(awk -F'\t' '$1=="move"' "$OUT" | wc -l)
|
|
moved=$(python3 -c 'import json,sys
|
|
print(sum(1 for l in open(sys.argv[1]) if json.loads(l).get("op") == "move"))' "$JOURNAL")
|
|
[ "$moved" -eq "$planned" ]
|
|
# The media servers' sidecars live in the metadata overlay and did not follow
|
|
# these moves, so the run names that cleanup at the moment it is needed.
|
|
[[ "$output" == *"media-view.sh reap"* ]]
|
|
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
|
|
[ "$status" -eq 0 ]
|
|
}
|
|
|
|
# The overlay note is only true where the metadata view is really mounted. On a
|
|
# host whose media servers bind the array directly (dankserver today: both
|
|
# quadlets mount /mnt/entropy, /var/lib/media-meta does not exist) the sidecars
|
|
# moved with the media, and "next: run reap" is an order that cannot be carried
|
|
# out. --metadata-view is how the caller names that path; the default is the
|
|
# host's view path, and the note rides only when it is a mountpoint.
|
|
@test "an apply with no mounted metadata view does not order a reap" {
|
|
plan
|
|
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
|
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"moved"* ]]
|
|
[[ "$output" != *"media-view.sh reap"* ]]
|
|
}
|
|
|
|
@test "a rollback with no mounted metadata view does not order a reap" {
|
|
plan
|
|
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
|
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
run python3 "$APPLY" rollback --journal "$JOURNAL" --root "$LIB" --apply
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"rolled back"* ]]
|
|
[[ "$output" != *"media-view.sh reap"* ]]
|
|
}
|
|
|
|
@test "a target already holding a different file stops the whole run" {
|
|
plan
|
|
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
|
at "$LIB/Alien (1979)/Alien (1979).mkv" 7
|
|
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"occupied"* ]]
|
|
# nothing moved at all: the other films are still where the census saw them
|
|
[ -e "$LIB/Marnie (1964)/Marnie.1964.1080p.BluRay.x265.mkv" ]
|
|
[ ! -e "$JOURNAL" ]
|
|
}
|
|
|
|
@test "a plan row whose source has gone stops the run instead of inventing a move" {
|
|
plan
|
|
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
|
rm "$LIB/Marnie (1964)/Marnie.1964.1080p.BluRay.x265.mkv"
|
|
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"missing"* ]]
|
|
[ -e "$LIB/Alien (1979)/Alien.1979.1080p.BluRay.x265.10bit.mkv" ]
|
|
}
|
|
|
|
@test "a run interrupted between the journal write and the move finishes the job" {
|
|
plan
|
|
JOURNAL="$BATS_TEST_TMPDIR/journal.jsonl"
|
|
# The journal is written before each move, so an interruption leaves two
|
|
# states and only two: an intent whose move happened, and an intent whose
|
|
# move never ran. Both are simulated here by hand.
|
|
python3 - "$OUT" "$JOURNAL" <<'PY'
|
|
import json, os, sys
|
|
moves = [r for r in (l.rstrip("\n").split("\t") for l in open(sys.argv[1]))
|
|
if r[0] == "move"][:3]
|
|
with open(sys.argv[2], "w") as fh:
|
|
for i, r in enumerate(moves):
|
|
fh.write(json.dumps({"op": "move", "src": r[1], "dst": r[2],
|
|
"bytes": os.path.getsize(r[1])}) + "\n")
|
|
if i < 2: # the first two moves actually ran
|
|
os.makedirs(os.path.dirname(r[2]), exist_ok=True)
|
|
os.rename(r[1], r[2])
|
|
PY
|
|
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
|
|
[ "$status" -eq 0 ]
|
|
python3 - "$OUT" "$JOURNAL" <<'PY'
|
|
import json, os, sys
|
|
moves = [r for r in (l.rstrip("\n").split("\t") for l in open(sys.argv[1]))
|
|
if r[0] == "move"]
|
|
journal = [json.loads(l) for l in open(sys.argv[2])]
|
|
seen = [j["dst"] for j in journal]
|
|
assert len(seen) == len(set(seen)), "a destination was journalled twice"
|
|
for r in moves:
|
|
assert os.path.exists(r[2]) and not os.path.exists(r[1]), r[2]
|
|
PY
|
|
}
|
|
|
|
@test "rollback puts every file back where the census found it" {
|
|
plan
|
|
apply_plan
|
|
[ "$status" -eq 0 ]
|
|
run python3 "$APPLY" rollback --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
[ -e "$LIB/Alien (1979)/Alien.1979.1080p.BluRay.x265.10bit.mkv" ]
|
|
[ -e "$LIB/Marnie (1964)/Marnie.1964.1080p.BluRay.x265.mkv" ]
|
|
[ -e "$LIB/Alien (1979)/subs/3_Eng.srt" ]
|
|
[ ! -e "$LIB/Alien (1979)/Alien (1979).mkv" ]
|
|
# the folders the run created are empty afterwards and must not litter the
|
|
# library: this one never existed before the plan ran
|
|
[ -d "$LIB/VHS 2 (2013)" ]
|
|
[ ! -e "$LIB/V-H-S-2 (2013)" ]
|
|
}
|
|
|
|
@test "rollback refuses when a moved file has gone missing" {
|
|
plan
|
|
apply_plan
|
|
[ "$status" -eq 0 ]
|
|
rm "$LIB/Alien (1979)/Alien (1979).mkv"
|
|
run python3 "$APPLY" rollback --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"missing"* ]]
|
|
# the refusal is all-or-nothing: another film did not move back
|
|
[ -e "$LIB/Marnie (1964)/Marnie (1964).mkv" ]
|
|
}
|
|
|
|
# ---------------------------------------------------------------- sweep
|
|
#
|
|
# A relayout leaves the release folders it emptied standing at the library root:
|
|
# 1675 of them after the live run, each holding nothing but empty subfolders. The
|
|
# sweep pass removes only folders THIS RUN emptied — proven from the journal,
|
|
# never from "it looks empty" — so a folder someone made by hand is untouchable,
|
|
# and every removal rides the same rollback as the moves.
|
|
|
|
@test "sweep removes only the folders this run emptied" {
|
|
plan
|
|
apply_plan
|
|
mkdir -p "$LIB/A Empty Placeholder" # someone's folder, never a move source
|
|
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"swept"* ]]
|
|
[ ! -e "$LIB/AMERICAN POP" ]
|
|
[ ! -e "$LIB/VHS 2 (2013)" ]
|
|
[ ! -e "$LIB/Alien (1979)/Featurettes" ]
|
|
[ ! -e "$LIB/Alien (1979)/subs" ]
|
|
[ ! -e "$LIB/Marnie (1964)/Exrtas" ]
|
|
# a folder the run never moved a file out of survives, empty or not
|
|
[ -d "$LIB/A Empty Placeholder" ]
|
|
}
|
|
|
|
@test "a swept folder carries the empty junk folders inside it" {
|
|
mkdir -p "$LIB/AMERICAN POP/Screens" # screenshots, never a move source
|
|
plan
|
|
apply_plan
|
|
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"0 left in place"* ]]
|
|
[ ! -e "$LIB/AMERICAN POP/Screens" ]
|
|
[ ! -e "$LIB/AMERICAN POP" ]
|
|
}
|
|
|
|
@test "a sweep whose rmdir failed tries those folders again" {
|
|
plan
|
|
apply_plan
|
|
mkdir -p "$LIB/AMERICAN POP/Screens"
|
|
chmod 500 "$LIB/AMERICAN POP" # readable, but nothing can be removed
|
|
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"left in place"* ]]
|
|
[ -e "$LIB/AMERICAN POP/Screens" ]
|
|
chmod 700 "$LIB/AMERICAN POP" # the operator cleared the blocker
|
|
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" != *"left in place: "* ]]
|
|
[ ! -e "$LIB/AMERICAN POP" ]
|
|
}
|
|
|
|
@test "sweep without --apply reports the husks and changes nothing" {
|
|
plan
|
|
apply_plan
|
|
local before jlines
|
|
before=$(find "$LIB" -type d | wc -l)
|
|
jlines=$(wc -l < "$JOURNAL")
|
|
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL"
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"--apply"* ]]
|
|
[ "$(find "$LIB" -type d | wc -l)" -eq "$before" ]
|
|
[ "$(wc -l < "$JOURNAL")" -eq "$jlines" ]
|
|
[ -e "$LIB/AMERICAN POP" ]
|
|
}
|
|
|
|
@test "a folder that still holds content is never swept" {
|
|
plan
|
|
apply_plan
|
|
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
# the film folder, the folder the run created, and the piles all stay
|
|
[ -d "$LIB/Marnie (1964)" ]
|
|
[ -e "$LIB/Marnie (1964)/Marnie (1964).mkv" ]
|
|
[ -d "$LIB/V-H-S-2 (2013)" ]
|
|
[ -d "$LIB/_unsorted bonus/extras" ]
|
|
}
|
|
|
|
@test "rollback after a sweep brings the folders and the files back" {
|
|
plan
|
|
apply_plan
|
|
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
[ ! -e "$LIB/AMERICAN POP" ]
|
|
run python3 "$APPLY" rollback --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
[ -e "$LIB/AMERICAN POP/AMERICAN.POP.2023.1080p.WEB-DL.mkv" ]
|
|
[ -e "$LIB/VHS 2 (2013)/VHS.2.2013.1080p.BluRay.x265.mkv" ]
|
|
[ -e "$LIB/Alien (1979)/Featurettes/A Bug And A Bite - Making Alien.mkv" ]
|
|
[ -e "$LIB/Alien (1979)/subs/3_Eng.srt" ]
|
|
}
|
|
|
|
@test "a second sweep has nothing left to do and says so" {
|
|
plan
|
|
apply_plan
|
|
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
local before
|
|
before=$(find "$LIB" -type d | wc -l)
|
|
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"swept: 0"* ]]
|
|
[ "$(find "$LIB" -type d | wc -l)" -eq "$before" ]
|
|
}
|
|
|
|
@test "sweep never follows a symlink and never removes the folder it guards" {
|
|
plan
|
|
apply_plan
|
|
mkdir -p "$LIB/Elsewhere"
|
|
at "$LIB/Elsewhere/keep.txt" 10
|
|
ln -s "$LIB/Elsewhere" "$LIB/AMERICAN POP/pointer"
|
|
run python3 "$APPLY" sweep --root "$LIB" --journal "$JOURNAL" --apply
|
|
[ "$status" -eq 0 ]
|
|
# a link is content, not nothing: the husk it sits in survives untouched
|
|
[ -d "$LIB/AMERICAN POP" ]
|
|
[ -L "$LIB/AMERICAN POP/pointer" ]
|
|
[ -e "$LIB/Elsewhere/keep.txt" ]
|
|
}
|
|
|
|
@test "sweep refuses a journal that belongs to another library" {
|
|
plan
|
|
apply_plan
|
|
mkdir -p "$BATS_TEST_TMPDIR/Other"
|
|
run python3 "$APPLY" sweep --root "$BATS_TEST_TMPDIR/Other" \
|
|
--journal "$JOURNAL" --apply
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"outside"* ]]
|
|
[ -d "$LIB/AMERICAN POP" ]
|
|
}
|