Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
183a6d5442 | ||
|
|
6d274acc47 | ||
|
|
3344b51fbd | ||
|
|
e82c8b2d06 | ||
|
|
f7083f175d | ||
|
|
75b04dc313 | ||
|
|
2625ad1083 | ||
|
|
9c25b1cccb |
No files matched your search
@@ -156,6 +156,15 @@ Unit requirements:
|
||||
(`<host>/<svc>/quadlet/`), is installed by the deployer, and is covered by
|
||||
the quadlet test seam. The host-side unit is generated state; the repo file
|
||||
is the source of truth.
|
||||
- **Bind-source gate** — a unit whose `Volume=` source is a mount this host owns
|
||||
(the entropy array, the metadata view) carries
|
||||
`ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh <source>`.
|
||||
Measured 2026-10-31: in a **rootless** unit `RequiresMountsFor=` is inert (the
|
||||
user manager cannot order against the system's mount unit), and
|
||||
`ConditionPathIsMountPoint=` is fail-*dead* — an unmet condition logs `skipped,
|
||||
unmet condition check` with `NRestarts=0` and no start attempt ever follows.
|
||||
Bind the array before it is mounted and the container sees the empty pre-mount
|
||||
directory as truth; a media scan then deletes the library it cannot see.
|
||||
|
||||
Before declaring a service deployed: `systemctl --user is-enabled <name>` →
|
||||
`enabled`, `is-active` → `active`, and the container answers its health check.
|
||||
|
||||
@@ -36,6 +36,9 @@ Volume=/home/dank/Containers/copyparty/cfg:/cfg
|
||||
PublishPort=3923:3923
|
||||
|
||||
[Service]
|
||||
# /w is the array: serving the pre-mount directory instead of the volume shows an
|
||||
# empty share and nothing complains, so the mount has to answer first.
|
||||
ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /mnt/entropy
|
||||
EnvironmentFile=/home/dank/Containers/copyparty/env
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# The io controller is what makes IOWeight= mean anything, and systemd does not
|
||||
# enable it below `user.slice` by default: with io accounting off, `io` never
|
||||
# appears in `user.slice/cgroup.subtree_control`, so no rootless container can
|
||||
# have an `io.weight` file and `IOWeight=` in a quadlet is silently inert.
|
||||
# Measured on Dankserver 2026-10-31: setting this property flipped
|
||||
# `user.slice/cgroup.subtree_control` from `cpu memory pids` to `cpu io memory
|
||||
# pids` immediately, with nothing restarted.
|
||||
#
|
||||
# This is half the fix; the other half is delegating io to user@1000.service
|
||||
# (user@1000.service.d/io-delegate.conf). Both must be in place before a reboot,
|
||||
# because the delegation is read when the user manager starts. Law and the
|
||||
# verification command: [[Quadlet Lifecycle]].
|
||||
[Slice]
|
||||
IOAccounting=yes
|
||||
@@ -0,0 +1,12 @@
|
||||
# The system manager gives the user manager cpu/memory/pids by default (systemd
|
||||
# 259: `systemctl show user@1000.service -p DelegateControllers` → `cpu memory
|
||||
# pids`). Without `io` in that set no `io.weight` file can exist anywhere below
|
||||
# `user.slice`, so `IOWeight=` in a rootless quadlet is silently inert — the same
|
||||
# class of trap as `RequiresMountsFor=`. Granting the controller here is what
|
||||
# makes the media servers' I/O share real.
|
||||
#
|
||||
# Inert until `user@1000.service` starts again, i.e. the next reboot (restarting
|
||||
# it would kill every rootless container on the host). Verify after the reboot
|
||||
# with the check in [[Quadlet Lifecycle]].
|
||||
[Service]
|
||||
DelegateControllers=cpu memory pids io
|
||||
@@ -43,11 +43,10 @@ PublishPort=1900:1900/udp
|
||||
Volume=/home/dank/Containers/jellyfin/config:/config
|
||||
Volume=/home/dank/Containers/jellyfin/cache:/cache
|
||||
# The view, writable: Jellyfin is the metadata writer (movie.nfo / tvshow.nfo /
|
||||
# episode.nfo + artwork) and those bytes belong on the SSD. The condition is the
|
||||
# fail-closed guard — starting against a plain, unmounted view directory would let
|
||||
# a scan conclude the library is empty.
|
||||
# episode.nfo + artwork) and those bytes belong on the SSD. The gate in [Service]
|
||||
# is the guard: started against a plain, unmounted view directory a scan decides
|
||||
# the library is empty and deletes what it cannot see.
|
||||
Volume=/var/lib/media-meta/view:/data
|
||||
UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view
|
||||
AddDevice=/dev/dri
|
||||
GroupAdd=keep-groups
|
||||
HealthCmd=curl -f http://localhost:8096/health
|
||||
@@ -56,8 +55,19 @@ HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
|
||||
[Service]
|
||||
# Refuse to start until the bind source is a real mount — see the header of
|
||||
# scripts/wait-for-mount.sh for the blind-boot incident this prevents.
|
||||
ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /var/lib/media-meta/view /mnt/entropy
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
# A library scan reads the array for minutes on end. Weighted LOW (default is
|
||||
# 100) so that when the array is contended a scan loses to playback and to every
|
||||
# other container this user runs. Proportional share, not a cap: it changes
|
||||
# nothing while the disk is idle. 10 is a convention — the requirement is only
|
||||
# that it sit far below 100 — and it is inert until user@1000.service is
|
||||
# delegated the io controller ([[Quadlet Lifecycle]]).
|
||||
IOWeight=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -16,12 +16,21 @@ Volume=/var/lib/plexmediaserver:/config
|
||||
# to the overlay view re-points Plex at the sidecars without touching its DB.
|
||||
# Read-only: Plex consumes the NFO files Jellyfin writes and writes no metadata.
|
||||
Volume=/var/lib/media-meta/view:/mnt/entropy:ro
|
||||
UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view
|
||||
PublishPort=32400:32400
|
||||
|
||||
[Service]
|
||||
# Refuse to start until the bind source is a real mount: Plex blind is how paths
|
||||
# get marked missing in its own database. Both the view and the array under it
|
||||
# must answer — scripts/wait-for-mount.sh names the 2026-10-31 incident.
|
||||
ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /var/lib/media-meta/view /mnt/entropy
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
# See the jellyfin unit for the reasoning: a section scan gets a LOW weight
|
||||
# (default 100) so a walk of 50k files loses the array to playback and to every
|
||||
# other container under this user. Inert until user@1000.service holds the io
|
||||
# controller ([[Quadlet Lifecycle]]).
|
||||
IOWeight=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1208,14 +1208,47 @@ ensure_media_view() {
|
||||
# array below stays read-only and every sidecar Jellyfin writes lands on the
|
||||
# NVMe.
|
||||
# Raise it BEFORE either container starts — a start against a bare view
|
||||
# directory would let a scan clear the library database. The units also carry
|
||||
# ConditionPathIsMountPoint, so a missing view skips the start rather than
|
||||
# damaging anything. A relayout must be followed by `media-view.sh reap`
|
||||
# directory would let a scan clear the library database. Each unit gates on
|
||||
# scripts/wait-for-mount.sh, which waits and then refuses the start. A
|
||||
# relayout must be followed by `media-view.sh reap`
|
||||
# (see the Media Relayout page), which this is not.
|
||||
run_privileged bash scripts/media-view.sh install
|
||||
bash scripts/media-view.sh verify
|
||||
}
|
||||
|
||||
ensure_user_io_delegate() {
|
||||
# The media servers ask for an I/O share (IOWeight= in their quadlets) so a
|
||||
# library scan cannot starve playback or the array's other reader. Two system
|
||||
# drop-ins are what make that request real, and both are host state, so the
|
||||
# deployer places them:
|
||||
# user.slice.d/io-accounting.conf io enters user.slice's subtree
|
||||
# user@1000.service.d/io-delegate.conf the user manager gets the controller
|
||||
# Without the first, `io` never appears in user.slice/cgroup.subtree_control,
|
||||
# so no container under it can have an io.weight file at all; without the
|
||||
# second the user manager never asks for the controller. systemd hands
|
||||
# user@.service cpu/memory/pids by default and drops a line it cannot honour
|
||||
# without a word — IOWeight= on its own is inert config.
|
||||
local src dst
|
||||
for src in \
|
||||
dankserver/host/systemd/user.slice.d/io-accounting.conf \
|
||||
dankserver/host/systemd/user@1000.service.d/io-delegate.conf; do
|
||||
dst=/etc/systemd/system/${src#dankserver/host/systemd/}
|
||||
run_privileged mkdir -p "$(dirname "$dst")"
|
||||
if ! run_privileged cmp -s "$src" "$dst"; then
|
||||
run_privileged install -m 644 "$src" "$dst"
|
||||
run_privileged systemctl daemon-reload
|
||||
fi
|
||||
done
|
||||
# Accounting lands immediately; the delegation is read when user@1000.service
|
||||
# starts, and restarting that unit would kill every rootless container on this
|
||||
# host. So the share becomes real at the next reboot — say so out loud rather
|
||||
# than let a quadlet line read as live config.
|
||||
if ! systemctl show user@1000.service -p DelegateControllers | grep -qw io; then
|
||||
echo "NOTE: io is not delegated to user@1000.service yet - IOWeight= is"
|
||||
echo "NOTE: inert until the next reboot (see [[Quadlet Lifecycle]])"
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_provider_keys() {
|
||||
# Jellyfin's metadata-provider keys live in the container-owned plugin
|
||||
# configuration under the data root, never in git; scripts/jellyfin-providers.sh
|
||||
@@ -1237,6 +1270,7 @@ ensure_provider_keys() {
|
||||
|
||||
deploy_jellyfin() {
|
||||
ensure_media_view
|
||||
ensure_user_io_delegate
|
||||
ensure_provider_keys
|
||||
cp "dankserver/jellyfin/quadlet/jellyfin.container" "$quadlet_unit_dir/"
|
||||
systemctl --user daemon-reload
|
||||
@@ -1326,6 +1360,7 @@ deploy_copyparty() {
|
||||
|
||||
deploy_plex() {
|
||||
ensure_media_view
|
||||
ensure_user_io_delegate
|
||||
cp "dankserver/plex/quadlet/plex.container" "$quadlet_unit_dir/"
|
||||
systemctl --user daemon-reload
|
||||
if ! systemctl --user is-active -q plex.service; then
|
||||
|
||||
+68
-9
@@ -59,10 +59,18 @@ def journal_default(root):
|
||||
|
||||
|
||||
class Journal:
|
||||
"""Append-only JSONL of intents. Writes are fsynced before the tool acts."""
|
||||
"""Append-only JSONL of intents. Writes are fsynced before the tool acts.
|
||||
|
||||
def __init__(self, path, writing):
|
||||
``batch`` is how many intents may be written before one fsync covers them.
|
||||
The rule the file exists for is unchanged — an intent is on disk before the
|
||||
move it authorises runs — so the caller must call ``sync()`` before it
|
||||
performs a batch's renames; only the number of flushes falls. 1 = a flush
|
||||
per intent (the original behaviour).
|
||||
"""
|
||||
|
||||
def __init__(self, path, writing, batch=1):
|
||||
self.path, self.writing = path, writing
|
||||
self.batch = max(1, int(batch))
|
||||
self.records = []
|
||||
if os.path.exists(path):
|
||||
for line in open(path):
|
||||
@@ -70,11 +78,15 @@ class Journal:
|
||||
if line:
|
||||
self.records.append(json.loads(line))
|
||||
self._fh = None
|
||||
self._unsynced = 0
|
||||
# (src, dst) index: already() is called three times per move, so an
|
||||
# any() over every record written so far made a 58k-move pass quadratic.
|
||||
self._intents = {(r.get("src"), r.get("dst"))
|
||||
for r in self.records if r.get("op") == "move"}
|
||||
|
||||
def already(self, dst, src):
|
||||
"""An intent for this exact (src, dst) was written by an earlier run."""
|
||||
return any(r.get("op") == "move" and r.get("dst") == dst
|
||||
and r.get("src") == src for r in self.records)
|
||||
return (src, dst) in self._intents
|
||||
|
||||
@property
|
||||
def rolled_back(self):
|
||||
@@ -82,16 +94,28 @@ class Journal:
|
||||
|
||||
def record(self, **fields):
|
||||
self.records.append(fields)
|
||||
if fields.get("op") == "move":
|
||||
self._intents.add((fields.get("src"), fields.get("dst")))
|
||||
if not self.writing:
|
||||
return
|
||||
if self._fh is None:
|
||||
self._fh = open(self.path, "a")
|
||||
self._fh.write(json.dumps(fields) + "\n")
|
||||
self._fh.flush()
|
||||
self._unsynced += 1
|
||||
if self._unsynced >= self.batch:
|
||||
self.sync()
|
||||
|
||||
def sync(self):
|
||||
"""Make every intent written so far durable. No-op when nothing pending."""
|
||||
if not self.writing or self._fh is None or not self._unsynced:
|
||||
return
|
||||
os.fsync(self._fh.fileno())
|
||||
self._unsynced = 0
|
||||
|
||||
def close(self):
|
||||
if self._fh:
|
||||
self.sync()
|
||||
self._fh.close()
|
||||
|
||||
|
||||
@@ -177,14 +201,30 @@ def do_apply(args, root, journal, mkdirs, keeps, moves):
|
||||
print(f" would move {src} -> {dst}")
|
||||
return 0
|
||||
|
||||
batch = max(1, args.batch)
|
||||
made = {r.get("dst") for r in journal.records if r.get("op") == "mkdir"}
|
||||
dirs_new = 0
|
||||
queued = []
|
||||
|
||||
def drain_dirs():
|
||||
"""Create the queued folders only after their intents are durable."""
|
||||
nonlocal dirs_new
|
||||
if not queued:
|
||||
return
|
||||
journal.sync()
|
||||
for d in queued:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
dirs_new += 1
|
||||
queued.clear()
|
||||
|
||||
for d in sorted(mkdirs, key=lambda q: q.count(os.sep)):
|
||||
if d in made or os.path.isdir(d):
|
||||
continue
|
||||
journal.record(op="mkdir", dst=d)
|
||||
os.makedirs(d, exist_ok=True)
|
||||
dirs_new += 1
|
||||
queued.append(d)
|
||||
if len(queued) >= batch:
|
||||
drain_dirs()
|
||||
drain_dirs()
|
||||
|
||||
# same-filesystem gate: rename cannot cross a device, and a silent copy of a
|
||||
# library this size is not an option anyone would choose.
|
||||
@@ -201,6 +241,19 @@ def do_apply(args, root, journal, mkdirs, keeps, moves):
|
||||
return 2
|
||||
|
||||
moved = done = 0
|
||||
pending = []
|
||||
|
||||
def drain_moves():
|
||||
"""Perform the queued renames only after their intents are on disk."""
|
||||
nonlocal moved
|
||||
if not pending:
|
||||
return
|
||||
journal.sync()
|
||||
for s, d in pending:
|
||||
os.rename(s, d)
|
||||
moved += 1
|
||||
pending.clear()
|
||||
|
||||
for src, dst, _kind in moves:
|
||||
if journal.already(dst, src):
|
||||
if os.path.exists(dst) and not os.path.exists(src):
|
||||
@@ -213,8 +266,10 @@ def do_apply(args, root, journal, mkdirs, keeps, moves):
|
||||
st = os.stat(src)
|
||||
journal.record(op="move", src=src, dst=dst, bytes=st.st_size, mode=st.st_mode,
|
||||
mtime=st.st_mtime, ino=st.st_ino, dev=st.st_dev)
|
||||
os.rename(src, dst)
|
||||
moved += 1
|
||||
pending.append((src, dst))
|
||||
if len(pending) >= batch:
|
||||
drain_moves()
|
||||
drain_moves()
|
||||
print(f"applied: {moved} moved, {done} already done in an earlier run,"
|
||||
f" {dirs_new} folders created, {len(keeps)} files stayed put")
|
||||
print(f"journal: {journal.path} — undo with: media-apply.py rollback"
|
||||
@@ -418,13 +473,17 @@ def main():
|
||||
ap.add_argument("--metadata-view", default=METADATA_VIEW,
|
||||
help="overlay the media servers read through; the sidecar"
|
||||
" note prints only when this path is a mountpoint")
|
||||
ap.add_argument("--batch", type=int, default=64,
|
||||
help="journal intents per fsync (1 = flush before every move)."
|
||||
" An intent is still durable before the move it authorises;"
|
||||
" only the number of flushes falls")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = os.path.abspath(args.root)
|
||||
if not os.path.isdir(root):
|
||||
raise Refuse(f"no library at {root}")
|
||||
journal_path = args.journal or journal_default(root)
|
||||
journal = Journal(journal_path, args.apply)
|
||||
journal = Journal(journal_path, args.apply, batch=args.batch)
|
||||
try:
|
||||
if args.cmd in ("rollback", "verify", "sweep") and not os.path.exists(journal_path):
|
||||
raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
|
||||
|
||||
+33
-10
@@ -168,12 +168,16 @@ SEASON_META = re.compile(r"^season\.nfo$", re.I)
|
||||
FOLDER_IMAGE = re.compile(r"^folder\.(jpg|jpeg|png|webp|bmp)$", re.I)
|
||||
SEASON_DIR_RE = re.compile(r"(?i)^\s*(?:season[\s._-]*(\d{1,3})|s(\d{1,2}))\s*$")
|
||||
BAD_CHARS = re.compile(r'[<>:"/|?*]')
|
||||
# SxxEyy with an optional second episode, plus the 1x04 form. The second number
|
||||
# is guarded so a resolution never reads as an episode: "S01E01-1080p" is ONE
|
||||
# episode, and the longest-match backtracking of \d{1,3} is what the (?![\dpP])
|
||||
# after it closes off.
|
||||
# SxxEyy with an optional second episode, plus the 1x04 form. Both episode numbers
|
||||
# are guarded so a longer digit run never reads as an episode. A resolution glued
|
||||
# on stays legal — "S01E01-1080p" is ONE episode — while "S03E2019" and
|
||||
# "S04E47370355" (both real names on the array, from a box set whose provider rows
|
||||
# say the same junk) state no episode at all: the run is an airdate or a pack
|
||||
# index, and the first three digits of it would steal E201 and E473 from the
|
||||
# episodes that own them. No marker means the file is numbered by its provider
|
||||
# row, or by tvdb's title, or it parks.
|
||||
EP_RANGE = re.compile(r"(?i)(?<![a-z0-9])s(?:eason[\s._-]*)?(\d{1,2})[\s._-]*"
|
||||
r"e(?:p(?:isode)[\s._-]*)?(\d{1,3})"
|
||||
r"e(?:p(?:isode)[\s._-]*)?(\d{1,3})(?![\dpP])"
|
||||
r"(?:[\s._-]*(?:-|to[\s._-]*|ep)[\s._-]*(?:s\d{1,2}[\s._-]*)?e?"
|
||||
r"(\d{1,3})(?![\dpP]))?")
|
||||
NX_RE = re.compile(r"(?i)(?<![a-z0-9])(\d{1,2})x(\d{2,3})(?![a-z0-9])")
|
||||
@@ -201,6 +205,11 @@ EXTRA_KIND = {"trailer": "trailers", "tmbd": "trailers", "intro": "other",
|
||||
"deleted": "deleted scenes", "other": "other", "epilogue": "other",
|
||||
"opening": "other", "featurette": "featurettes", "short": "shorts"}
|
||||
SHORT_MAX = 45 * 60 # over this, a file with no episode evidence is film-shaped
|
||||
# An episode number above this is not an episode number: the array's box sets
|
||||
# carry 2019 (an airdate fragment) and 47370355 (a pack index) in the column the
|
||||
# metadata server recorded as the match. tvdb's episode list is the only door on
|
||||
# such a row; nothing files at E1000 or beyond.
|
||||
EP_MAX = 999
|
||||
|
||||
|
||||
def rel(p):
|
||||
@@ -748,10 +757,11 @@ for r in video:
|
||||
else:
|
||||
season = fs if fs is not None else num(t["season"])
|
||||
ep = leading_index(r["base"]) or num(t["episode"])
|
||||
if not 0 <= season <= 200 or ep <= 0:
|
||||
if not 0 <= season <= 200 or not 1 <= ep <= EP_MAX:
|
||||
# Season 0 IS a season: specials live in Season 00 by Jellyfin's own
|
||||
# contract. A season that is a YEAR is the date matcher's doing, and
|
||||
# the only authority on it is tvdb's episode list for the series.
|
||||
# contract. A season that is a YEAR is the date matcher's doing and an
|
||||
# episode above EP_MAX is a box set's airdate or pack index; neither is
|
||||
# a number, and the only authority on either is tvdb's episode list.
|
||||
recovered = tvdb_number(t)
|
||||
if not recovered:
|
||||
unknown.append(
|
||||
@@ -1199,7 +1209,9 @@ def write_exceptions():
|
||||
f"{n} S{'/'.join(f'{x:02d}' for x in ss)}"
|
||||
for n, ss in clash[:300]) + "\n")
|
||||
map_ids = {e["id"] for e in MAP.values()}
|
||||
unseen = sorted(f for f in MAP if f not in MAP_SEEN)
|
||||
on_array = {rel(p).split("/")[0] for p in held}
|
||||
stalled = sorted(f for f in MAP if f not in MAP_SEEN and f in on_array)
|
||||
unseen = sorted(f for f in MAP if f not in MAP_SEEN and f not in on_array)
|
||||
unanswered = sorted({f"{identity(s)[0]} ({s['tvdb']})"
|
||||
for s in canon.values()
|
||||
if s.get("tvdb") in map_ids and not s.get("canon")})
|
||||
@@ -1211,9 +1223,20 @@ def write_exceptions():
|
||||
" and re-plan, because a rename is idempotent.\n")
|
||||
fh.write(" " + "; ".join(
|
||||
f"{f} -> {MAP[f]['id']}" for f in sorted(MAP_SEEN)) + "\n")
|
||||
if stalled:
|
||||
fh.write(f"\n{len(stalled):>6} map rows name a folder on the array that\n")
|
||||
fh.write(" never became a series: every file in it parked, so the id\n"
|
||||
" had nothing to name. The map names a RECORD — it numbers\n"
|
||||
" no episodes and owns no bonus material, so read the park\n"
|
||||
" reasons above this line: a folder of openings, endings and\n"
|
||||
" promos with no show to claim them is the usual shape.\n")
|
||||
fh.write(" " + "; ".join(
|
||||
f"{f} -> {MAP[f]['id']}" for f in stalled[:300]) + "\n")
|
||||
if unseen:
|
||||
fh.write(f"\n{len(unseen):>6} map rows name no folder on the array: the\n")
|
||||
fh.write(" pack moved, or the row carries a name the array never held.\n")
|
||||
fh.write(" pack moved (into the review pile counts: the map reads\n"
|
||||
" top-level folders only), or the row carries a name the\n"
|
||||
" array never held.\n")
|
||||
fh.write(" " + "; ".join(unseen[:300]) + "\n")
|
||||
if MAP_CLASH:
|
||||
fh.write(f"\n{len(MAP_CLASH):>6} folders carry a provider tvdb id the map\n")
|
||||
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env bash
|
||||
# wait-for-mount.sh — refuse a container start until what it binds is mounted.
|
||||
#
|
||||
# Why this exists (live 2026-10-31, Dankserver): a rootless quadlet cannot order
|
||||
# itself against a SYSTEM mount unit. `RequiresMountsFor=/mnt/entropy` in a user
|
||||
# unit resolves to nothing and is inert, so when the array's mount finished a
|
||||
# second after podman resolved the bind, Jellyfin and Plex started against the
|
||||
# pre-mount directory. Their library root looked EMPTY and Jellyfin began
|
||||
# deleting metadata by the thousand (`Removing item, Type: "Season"`). A blind
|
||||
# bind is not a cosmetic fault; it is a metadata-deleting machine.
|
||||
#
|
||||
# A `ConditionPathIsMountPoint=` is NOT the fix, measured on the same host: an
|
||||
# unmet condition logs "skipped, unmet condition check", NRestarts=0, and no
|
||||
# start attempt ever follows — the unit stays dead until a human starts it.
|
||||
# That is fail-dead. This script WAITS (the mount usually lands within seconds)
|
||||
# and only then refuses, so `Restart=always` carries the unit across the race.
|
||||
#
|
||||
# usage: wait-for-mount.sh [-t SECONDS] MOUNTPOINT [MOUNTPOINT...]
|
||||
# exit 0 every path is a mountpoint
|
||||
# exit 1 a path is still not a mountpoint after the window (named on stderr)
|
||||
# exit 2 usage error
|
||||
set -euo pipefail
|
||||
|
||||
usage() { printf 'usage: %s [-t SECONDS] MOUNTPOINT [MOUNTPOINT...]\n' "$0"; }
|
||||
|
||||
window=90
|
||||
while getopts ":t:h" opt; do
|
||||
case "$opt" in
|
||||
t) window=$OPTARG ;;
|
||||
h) usage; exit 0 ;;
|
||||
*) usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND - 1))
|
||||
|
||||
[ "$#" -ge 1 ] || { usage >&2; exit 2; }
|
||||
case "$window" in
|
||||
'' | *[!0-9]*) printf 'refused: -t wants whole seconds, got %s\n' "$window" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
is_mount() { findmnt -M "$1" -rno TARGET >/dev/null 2>&1; }
|
||||
|
||||
report() {
|
||||
local why=$1; shift
|
||||
printf '%s:' "$why"
|
||||
for p in "$@"; do printf ' %s' "$p"; done
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
deadline=$(( $(date +%s) + window ))
|
||||
warned=0
|
||||
while :; do
|
||||
missing=()
|
||||
for path in "$@"; do
|
||||
is_mount "$path" || missing+=("$path")
|
||||
done
|
||||
[ "${#missing[@]}" -eq 0 ] && break
|
||||
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
report "refused: still not mounted after ${window}s" "${missing[@]}" >&2
|
||||
printf 'refusing to start: a container bind-mounted here would see the empty\n' >&2
|
||||
printf 'directory underneath and conclude the library is empty — which is how a\n' >&2
|
||||
printf 'scan begins deleting metadata it cannot find.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$warned" -eq 0 ]; then
|
||||
report "waiting for mount(s) this unit depends on" "${missing[@]}"
|
||||
warned=1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
[ "$warned" -eq 0 ] || printf 'mounted; proceeding.\n'
|
||||
exit 0
|
||||
@@ -100,3 +100,12 @@ aborts with `DEPLOY INCOMPLETE: e2e gate FAILED` when it does not pass
|
||||
hand-off list (`HANDOFFS`) instead of a single host. The guard contract is
|
||||
untouched: challenge cleared, no authentik `uidField`, status < 500, and every
|
||||
host NOT in the list still has to land on its own vhost.
|
||||
7. **The runner's resolver is part of the suite.** A 2026-10-31 deploy of one
|
||||
Dankserver quadlet failed the gate with 21 reds spread over SSO, homepage RBAC
|
||||
and every vhost crawl — precisely the specs that navigate to an absolute
|
||||
`*.franksdank.cloud` name, while every spec on the relative `baseURL` stayed
|
||||
green. The error was `net::ERR_NAME_NOT_RESOLVED`: the workstation had lost its
|
||||
wired link, and on the fallback its resolver was no longer pinned to Pi-hole.
|
||||
Nothing on the fleet was wrong — the same suite re-run minutes later passed
|
||||
37/37. Before a mass red gets blamed on the guard chain, run
|
||||
`resolvectl query grafana.franksdank.cloud` on the box that runs the gate.
|
||||
@@ -167,6 +167,43 @@ Gate: `tests/quadlet_image_pins.bats`. Where upstream publishes no version
|
||||
tags (copyparty-ac stops at 1.19.15), pin `:latest@sha256:` and name the
|
||||
version the config label carries in the comment beside it.
|
||||
|
||||
## 4b. Resource control needs host permission (measured 2026-10-31, Dankserver)
|
||||
|
||||
`IOWeight=` in a rootless quadlet does nothing on its own, and nothing says so.
|
||||
The system manager delegates `cpu memory pids` to `user@.service` and **not** `io`;
|
||||
systemd drops a resource line it cannot honour **without a word** — the generator
|
||||
still emits `IOWeight=10`, the container still starts and answers its health check,
|
||||
and the cgroup simply has no `io.weight` file. Same family as
|
||||
`RequiresMountsFor=` (inert in the user manager) and `ConditionPathIsMountPoint=`
|
||||
(fail-dead): the line is accepted, the unit is happy, the setting is dead.
|
||||
|
||||
Two system drop-ins make it real, both placed by `ensure_user_io_delegate` in
|
||||
`scripts/deploy-remote.sh` (repo copies under `dankserver/host/systemd/`):
|
||||
|
||||
| File | Effect |
|
||||
|---|---|
|
||||
| `user.slice.d/io-accounting.conf` → `[Slice] IOAccounting=yes` | puts `io` into `user.slice/cgroup.subtree_control`; lands immediately, no restart |
|
||||
| `user@1000.service.d/io-delegate.conf` → `[Service] DelegateControllers=cpu memory pids io` | the user manager can only request controllers it was delegated; read when it starts, so it lands on **the next reboot** (restarting it kills every rootless container on the host) |
|
||||
|
||||
Read the chain in this order — each line is the failure mode of the one after it:
|
||||
|
||||
```bash
|
||||
cat /sys/fs/cgroup/user.slice/cgroup.subtree_control # wants: io
|
||||
systemctl show user@1000.service -p DelegateControllers # wants: io
|
||||
cat /sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/app.slice/\<unit\>.service/libpod-payload-*/io.weight
|
||||
```
|
||||
|
||||
`io` missing from the first → the accounting drop-in is absent. Missing from the
|
||||
second → the host has not rebooted since the delegation landed (the deployer prints
|
||||
`NOTE: io is not delegated …` for exactly this). No file at the third → that unit
|
||||
carries no weight.
|
||||
|
||||
Two limits that survive the fix: a container's weight covers **everything** in its
|
||||
cgroup, so a library scan and a playback stream share one weight; and the comparison
|
||||
is made *inside* `user.slice`, so a rootless unit's weight cannot push it below a
|
||||
root unit (`frank-archon`). Shaping that pairing is the weight of `user.slice`
|
||||
itself — a fleet decision, not a unit line.
|
||||
|
||||
## 5. Rootless Quadlet log retrieval
|
||||
|
||||
Step 0 — establish ownership: `systemctl --user status <name>.service`
|
||||
|
||||
@@ -46,10 +46,11 @@ view really is a mountpoint.
|
||||
| Installer | `scripts/media-view.sh` — `plan` / `install` / `verify` / `status` / `reap` |
|
||||
| Written by | Jellyfin only (Plex binds read-only); nothing else in the fleet touches it |
|
||||
|
||||
Both consumers guard the view with
|
||||
`UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view`, and
|
||||
`scripts/deploy.sh dankserver --service jellyfin|plex` calls `media-view.sh
|
||||
install` + `verify` before it restarts either container.
|
||||
Both consumers gate their start on the view with
|
||||
`ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh` (it waits
|
||||
for the mount, then refuses), and `scripts/deploy.sh dankserver --service
|
||||
jellyfin|plex` calls `media-view.sh install` + `verify` before it restarts
|
||||
either container.
|
||||
|
||||
| Consumer | Bind | Mode | Why that path |
|
||||
|---|---|---|---|
|
||||
@@ -80,11 +81,20 @@ the defaults are the host values above.
|
||||
refuse via `lsblk -dno ROTA` if the upper base lives on the array: an overlay
|
||||
whose upper is the spinners is the original problem with extra steps, and it
|
||||
silently turns every metadata write back into a seek.
|
||||
- **Never mount the view over an unmounted lower.** `RequiresMountsFor=/mnt/entropy`
|
||||
plus the container `ConditionPathIsMountPoint` guards: an empty `lowerdir`
|
||||
presents an *empty library* as truth, and a media scan would then clear the
|
||||
database of everything. systemd does not create a mount point, so `install`
|
||||
creates `view` itself; without it the mount simply fails.
|
||||
- **A container that binds the array must refuse to start until the mount
|
||||
answers.** Live 2026-10-31: the array's mount and the media containers raced at
|
||||
boot, podman resolved the bind against the pre-mount directory, and both
|
||||
servers came up blind — Jellyfin read an empty library and began
|
||||
`Removing item, Type: "Season"` by the thousand. Two dependencies people reach
|
||||
for do not work here, both measured on that host: `RequiresMountsFor=` in a
|
||||
**rootless** unit is inert (the user manager cannot order against the system's
|
||||
`mnt-entropy.mount`), and `ConditionPathIsMountPoint=` is fail-*dead* — an
|
||||
unmet condition logs `skipped, unmet condition check` with `NRestarts=0` and no
|
||||
start attempt ever follows, so the unit waits for a human. The working guard is
|
||||
`scripts/wait-for-mount.sh` as `ExecStartPre`: it waits out the race, refuses by
|
||||
name after its window, and `Restart=always` carries the unit from there.
|
||||
systemd does not create a mount point, so `install` creates `view` itself;
|
||||
without it the mount simply fails.
|
||||
- **Never delete through the view.** A delete writes an overlayfs whiteout
|
||||
(`.wh.<name>`) in the upper layer, which hides the array's copy without
|
||||
removing it — the file reappears the moment the view is unmounted, and the
|
||||
@@ -107,7 +117,8 @@ the defaults are the host values above.
|
||||
|
||||
| Symptom | Meaning |
|
||||
|---|---|
|
||||
| `jellyfin.service` / `plex.service` show `ConditionPathIsMountPoint=... was not met` and stay inactive | The view is not mounted. Do not start them anyway: `scripts/media-view.sh install`, then `verify`. |
|
||||
| `jellyfin.service` / `plex.service` fail with `refused: still not mounted after 90s: …` | The view or the array is not mounted. Do not start them anyway: `scripts/media-view.sh install`, then `verify`. |
|
||||
| Servers answer, library looks empty, Jellyfin logs `Removing item, Type: "Season"` by the thousand | Blind bind — the container started before the mount and bound the pre-mount directory. `podman exec jellyfin ls /data` against `ls /mnt/entropy` is the whole diagnosis; restarting the container re-resolves the bind. |
|
||||
| Jellyfin log: `Read-only file system : '/data/Movies/…/movie.nfo'` | The container is binding the raw array, not the view — the quadlet drifted back or the view is gone. `media-view.sh status`. |
|
||||
| A folder appears in a media server with artwork/`.nfo` but no video | Ghost metadata from a relayout. `media-view.sh reap`. |
|
||||
| `verify` prints `refused: …/view is not a mountpoint` | A plain directory is sitting where the overlay belongs. That is the fail-closed guard working; `install` on the box. |
|
||||
|
||||
@@ -81,6 +81,13 @@ An apply also leaves the release folders it emptied standing where they were —
|
||||
hundreds of them at the library root — which is what `sweep` removes, dry-run
|
||||
first, from the same journal.
|
||||
|
||||
`--batch N` (default 64) makes the journal durable per batch instead of per
|
||||
move: N intents are written and fsynced, then those N rename. The rule a crash
|
||||
relies on — no rename runs before its intent is on disk — is unchanged, so an
|
||||
interrupted apply still replays the batch and resumes with nothing lost. Batch 1
|
||||
is the old behaviour; the only reason to name it is debugging, because on the
|
||||
spinning array it spends most of a large apply's wall time in fsyncs.
|
||||
|
||||
After it lands, both servers need a scan or the change is invisible. [[Plex]] and
|
||||
[[Jellyfin]] own the exact requests; two laws hold at this step: `204` is
|
||||
acceptance and anything else queued nothing — re-issue the identical request
|
||||
@@ -228,7 +235,10 @@ scripts/media-apply.py apply --plan /mnt/entropy/media-relayout-shows2-plan.tsv
|
||||
|
||||
Evidence, strongest first — the order the planner believes things:
|
||||
|
||||
- the **episode marker the file's own name states** (`S01E02-E03`, `1x04`);
|
||||
- the **episode marker the file's own name states** (`S01E02-E03`, `1x04`); a digit
|
||||
run longer than three at that position is NOT a number — `S03E2019` and
|
||||
`S04E47370355` are one box set's airdate fragment and pack index — so such a file
|
||||
is renumbered by tvdb's title or parks, and is never truncated down to fit;
|
||||
- the **`Season NN` folder it sits in and the number its name leads with inside
|
||||
it**: `Season 19/01 Charles Barkley, Nirvana.avi` is S19E01;
|
||||
- the **provider row**: which episode the file IS, and the numbering of a file
|
||||
|
||||
@@ -77,7 +77,7 @@ Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them.
|
||||
| Port 53 `bind: permission denied` crash loop after a reboot | sysctl conflict: the value must be 53 at boot, and remember `/etc/sysctl.conf` is applied **after** all of `sysctl.d/`. Redeploy pihole — `deploy_pihole` reinstalls the drop-in, neutralizes conflicting drop-ins, and comments out a stale `/etc/sysctl.conf` line |
|
||||
| Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`), and FTL serves it. Use the FQDN, or add an `/etc/hosts` entry on the client |
|
||||
| Every `franksdank.cloud` name is NXDOMAIN for LAN clients while external names still answer | FTL holds no local records: `getent hosts <name>` on Dankclaw comes back empty → `podman exec pihole pihole restartdns reload` (the 2026-10-19 pitfall under **Records**) |
|
||||
| One client gets nothing while every other client is fine | FTL rate-limiting that client: `podman exec pihole grep Rate-limiting /var/log/pihole/FTL.log`. A `10.0.2.x` client address is a pasta container's DNS forwarder, not a machine |
|
||||
| One client gets nothing while every other client is fine | FTL rate-limiting that client: `podman exec pihole grep Rate-limiting /var/log/pihole/FTL.log`. A `10.0.2.x` client address is a pasta container's DNS forwarder, not a machine — and **every** rootless container on that host shares the one address, so a single chatty container throttles DNS for all of them (measured 2026-10-31: 8 throttle events in a day, all `10.0.2.100`) |
|
||||
|
||||
## See Also
|
||||
|
||||
|
||||
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
# Seam: a container that binds the entropy array must refuse to start until the
|
||||
# thing it binds is actually mounted.
|
||||
#
|
||||
# Live 2026-10-31, Dankserver: the array's systemd mount STARTED eight seconds
|
||||
# before the media containers were created, but a rootless quadlet cannot order
|
||||
# itself against a SYSTEM mount unit — `RequiresMountsFor=/mnt/entropy` resolves
|
||||
# to nothing in the user manager and is simply inert. So podman resolved the bind
|
||||
# against the pre-mount directory (which held one stale subdirectory) and both
|
||||
# Jellyfin and Plex came up blind: their library root looked empty, Jellyfin
|
||||
# logged thousands of `Removing item, Type: "Season"`, and the array sat at 80 %
|
||||
# util doing nothing useful until the host went down. A blind bind is not a
|
||||
# cosmetic fault — it is a metadata-deleting machine.
|
||||
#
|
||||
# The measured fix is `ExecStartPre=<repo>/scripts/wait-for-mount.sh <source>`:
|
||||
# it WAITS for the mount and then refuses, so `Restart=always` carries the unit
|
||||
# across a slow mount. A `ConditionPathIsMountPoint=` was measured on the same
|
||||
# host and is the wrong tool: an unmet condition logs `skipped, unmet condition
|
||||
# check` with NRestarts=0 and never a start attempt — the unit stays dead until
|
||||
# a human starts it. That is fail-dead, not fail-closed.
|
||||
#
|
||||
# Tests assert only observable behavior of the gate (exit status and the name it
|
||||
# refuses by) and the structural law over the quadlets; never the internals.
|
||||
|
||||
load helpers
|
||||
|
||||
GATE="$REPO_ROOT/scripts/wait-for-mount.sh"
|
||||
|
||||
@test "the gate accepts a path that really is a mountpoint" {
|
||||
run "$GATE" -t 2 /proc
|
||||
[ "$status" -eq 0 ] || { printf '%s\n' "$output" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "a plain directory is refused, and refused BY NAME" {
|
||||
dir="$BATS_TEST_TMPDIR/not-a-mount"
|
||||
mkdir "$dir"
|
||||
run "$GATE" -t 1 "$dir"
|
||||
[ "$status" -ne 0 ] || { printf 'accepted a plain directory\n' >&2; return 1; }
|
||||
[[ "$output" == *"$dir"* ]] || { printf 'refusal did not name the path: %s\n' "$output" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "a path that does not exist is refused by name" {
|
||||
run "$GATE" -t 1 "$BATS_TEST_TMPDIR/does/not/exist"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"$BATS_TEST_TMPDIR/does/not/exist"* ]]
|
||||
}
|
||||
|
||||
@test "the gate holds the deadline instead of deciding on one look" {
|
||||
# A single check that failed at the wrong instant is the boot race itself:
|
||||
# the mount lands a second later and the unit is dead anyway. So the window
|
||||
# must be spent waiting, and that is observable from outside as elapsed time.
|
||||
dir="$BATS_TEST_TMPDIR/never"
|
||||
mkdir "$dir"
|
||||
start=$SECONDS
|
||||
run "$GATE" -t 4 "$dir"
|
||||
elapsed=$((SECONDS - start))
|
||||
[ "$status" -ne 0 ]
|
||||
[ "$elapsed" -ge 4 ] || { printf 'gave up after %ss, window was 4s\n' "$elapsed" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "the gate accepts a mount that arrives during its window" {
|
||||
dir="$BATS_TEST_TMPDIR/later"
|
||||
mkdir "$dir"
|
||||
if ! mount --bind /proc "$dir" 2>/dev/null; then
|
||||
skip "creating a mount needs privileges this box does not grant"
|
||||
fi
|
||||
run "$GATE" -t 4 "$dir"
|
||||
umount "$dir" 2>/dev/null || sudo -n umount "$dir" 2>/dev/null || true
|
||||
[ "$status" -eq 0 ] || { printf 'gate did not wait for a mount that arrived: %s\n' "$output" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "no unit without the gate binds the array or the metadata view" {
|
||||
bad=""
|
||||
while IFS= read -r unit; do
|
||||
sources=$(sed -n 's/^Volume=\([^:]*\):.*/\1/p' "$unit" \
|
||||
| grep -E '^(/mnt/entropy|/var/lib/media-meta/view)$' | sort -u)
|
||||
[ -n "$sources" ] || continue
|
||||
for src in $sources; do
|
||||
grep -q "ExecStartPre=.*wait-for-mount.sh.* $src\b" "$unit" \
|
||||
|| bad="$bad$unit binds $src with no start gate"$'\n'
|
||||
done
|
||||
done < <(find "$REPO_ROOT"/dankserver "$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro \
|
||||
-path '*/quadlet/*.container' -not -path '*.AGENTTRASH*')
|
||||
[ -z "$bad" ] || { printf 'blind binds:\n%s' "$bad" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "no gated unit keeps the fail-dead ConditionPathIsMountPoint" {
|
||||
bad=""
|
||||
while IFS= read -r unit; do
|
||||
grep -q "wait-for-mount.sh" "$unit" || continue
|
||||
grep -q "ConditionPathIsMountPoint" "$unit" \
|
||||
&& bad="$bad$unit: a gate and an unmet-condition skip cannot coexist"$'\n'
|
||||
done < <(find "$REPO_ROOT"/dankserver "$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro \
|
||||
-path '*/quadlet/*.container' -not -path '*.AGENTTRASH*')
|
||||
[ -z "$bad" ] || { printf 'fail-dead conditions:\n%s' "$bad" >&2; return 1; }
|
||||
}
|
||||
@@ -106,3 +106,24 @@ load helpers
|
||||
# env file, not an Environment= line.
|
||||
! grep -qE '^Environment=CUSTOM_(USER|PASSWORD)=' "$unit" || return 1
|
||||
}
|
||||
|
||||
@test "a quadlet that asks for an I/O share ships the host permission that makes it real" {
|
||||
# IOWeight= is silently inert unless the system manager both enables io for
|
||||
# user.slice and delegates it to the user manager: systemd drops a resource
|
||||
# line it cannot honour without a word and the container still starts healthy.
|
||||
# So an in-repo weight is only honest together with both drop-ins AND with a
|
||||
# deployer that places them. Measured 2026-10-31; verification chain in
|
||||
# Architecture/Quadlet Lifecycle 4b.
|
||||
if grep -lq '^IOWeight=' "$REPO_ROOT"/dankserver/*/quadlet/*.container; then
|
||||
asked=1
|
||||
else
|
||||
skip "no dankserver quadlet asks for an I/O share"
|
||||
fi
|
||||
grep -q '^IOAccounting=yes$' \
|
||||
"$REPO_ROOT/dankserver/host/systemd/user.slice.d/io-accounting.conf" || return 1
|
||||
grep -q '^DelegateControllers=.* io' \
|
||||
"$REPO_ROOT/dankserver/host/systemd/user@1000.service.d/io-delegate.conf" || return 1
|
||||
for f in user.slice.d/io-accounting.conf 'user@1000.service.d/io-delegate.conf'; do
|
||||
grep -q "dankserver/host/systemd/$f" "$REPO_ROOT/scripts/deploy-remote.sh" || return 1
|
||||
done
|
||||
}
|
||||
@@ -61,7 +61,10 @@ UNIT="$REPO_ROOT/dankserver/host/systemd/media-meta-view.mount"
|
||||
@test "both media servers bind the view and refuse to start without it" {
|
||||
for unit in "$JELLY" "$PLEX"; do
|
||||
grep -q "^Volume=$MOUNT:" "$unit" || return 1
|
||||
grep -q "^UnitConfigItem=ConditionPathIsMountPoint=$MOUNT$" "$unit" || return 1
|
||||
# The refusal is a start GATE, not a Condition: measured on the host, a
|
||||
# unit whose Condition goes unmet is skipped forever (NRestarts=0, no
|
||||
# start attempt) — fail-dead. The gate waits, then refuses.
|
||||
grep -q "^ExecStartPre=.*wait-for-mount.sh .*$MOUNT" "$unit" || return 1
|
||||
# The raw array must no longer be a container source: that bind is what
|
||||
# made sidecar sharing impossible (read-only) and metadata slow (seeks).
|
||||
! grep -q '^Volume=/mnt/entropy' "$unit"
|
||||
|
||||
@@ -494,6 +494,65 @@ for r in moves:
|
||||
PY
|
||||
}
|
||||
|
||||
# The live pass paid an fsync per move on a spinning array (63 074 of them) and
|
||||
# the run took 11 minutes for 58 278 renames. --batch N writes N intents, syncs
|
||||
# once, then performs those N moves — the durability rule is unchanged (no move
|
||||
# runs before its intent is on disk), only the number of flushes falls.
|
||||
@test "a batched apply journals every move and lands the same tree" {
|
||||
plan
|
||||
JOURNAL="$BATS_TEST_TMPDIR/batched.jsonl"
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" \
|
||||
--batch 4 --apply
|
||||
[ "$status" -eq 0 ]
|
||||
local planned journalled
|
||||
planned=$(awk -F'\t' '$1=="move"' "$OUT" | wc -l)
|
||||
journalled=$(grep -c '"op": "move"' "$JOURNAL")
|
||||
[ "$journalled" -eq "$planned" ]
|
||||
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
|
||||
[ "$status" -eq 0 ]
|
||||
python3 - "$OUT" <<'PY'
|
||||
import os, sys
|
||||
moves = [r for r in (l.rstrip("\n").split("\t") for l in open(sys.argv[1]))
|
||||
if r[0] == "move"]
|
||||
assert moves, "fixture planned no moves"
|
||||
for r in moves:
|
||||
assert os.path.exists(r[2]), f"target missing: {r[2]}"
|
||||
assert not os.path.exists(r[1]), f"source left behind: {r[1]}"
|
||||
PY
|
||||
}
|
||||
|
||||
@test "an apply stopped mid-batch resumes without losing or double-journalling a move" {
|
||||
plan
|
||||
JOURNAL="$BATS_TEST_TMPDIR/interrupted-batch.jsonl"
|
||||
python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" \
|
||||
--batch 2 --apply > "$BATS_TEST_TMPDIR/first.log" 2>&1 &
|
||||
local pid=$! i
|
||||
# stop the run the moment a batch of intents is on disk (it may also have
|
||||
# finished already — both outcomes must resume cleanly)
|
||||
for i in $(seq 1 200); do
|
||||
[ -f "$JOURNAL" ] && [ "$(grep -c '"op": "move"' "$JOURNAL" 2>/dev/null || true)" -ge 2 ] && break
|
||||
sleep 0.02
|
||||
done
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
wait "$pid" 2>/dev/null || true
|
||||
|
||||
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --batch 2 --apply
|
||||
[ "$status" -eq 0 ]
|
||||
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
|
||||
[ "$status" -eq 0 ]
|
||||
python3 - "$OUT" "$JOURNAL" <<'PY'
|
||||
import json, os, sys
|
||||
moves = [r for r in (l.rstrip("\n").split("\t") for l in open(sys.argv[1]))
|
||||
if r[0] == "move"]
|
||||
journal = [json.loads(l) for l in open(sys.argv[2])]
|
||||
dsts = [j["dst"] for j in journal if j.get("op") == "move"]
|
||||
assert len(dsts) == len(set(dsts)), "a destination was journalled twice"
|
||||
assert len(dsts) == len(moves), f"journalled {len(dsts)} of {len(moves)} moves"
|
||||
for r in moves:
|
||||
assert os.path.exists(r[2]) and not os.path.exists(r[1]), r[2]
|
||||
PY
|
||||
}
|
||||
|
||||
@test "rollback puts every file back where the census found it" {
|
||||
plan
|
||||
apply_plan
|
||||
|
||||
@@ -513,6 +513,46 @@ PY
|
||||
[[ "$output" == *"Best Of The Examples.mkv"* ]]
|
||||
}
|
||||
|
||||
|
||||
# A release pack can carry a number that is not an episode at all: two files of
|
||||
# the Attack on Titan box set on the array are named S03E2019 and S04E47370355,
|
||||
# and the provider rows the server recorded for them say the same junk. Reading
|
||||
# either as "the first three digits" files the box set under E201 and E473 and
|
||||
# steals those episode slots from their real owners, so a run too long to be an
|
||||
# episode is no marker and no number: the file is renumbered from tvdb's title
|
||||
# when the series' id answers it, and parks when nothing answers.
|
||||
@test "an episode number longer than three digits never files as an episode" {
|
||||
file "$LIB/Junk Numbers/Attack Pack S03E2019.mkv" 5000
|
||||
title_row "$LIB/Junk Numbers/Attack Pack S03E2019.mkv" "Attack Pack" 2013 3 2019
|
||||
plan_park
|
||||
run grep -c -- "- S03E201 -" "$OUT"
|
||||
[ "$output" = "0" ]
|
||||
[[ "$(dst "$LIB/Junk Numbers/Attack Pack S03E2019.mkv")" == "$OUTSIDE/"* ]]
|
||||
grep -q "a provider row of season 3 episode 2019" "$EXC"
|
||||
}
|
||||
|
||||
@test "a provider episode too long to be one is renumbered by the title tvdb states" {
|
||||
printf '{"tvdb:series:4242": %s, "tvdb:episodes:4242": %s}\n' \
|
||||
'{"name": "Attack Pack", "year": "2013"}' \
|
||||
'{"3/7": "A Title tvdb Holds"}' > "$CACHE"
|
||||
file "$LIB/Junk Numbers/Attack Pack S03E2019.mkv" 5000
|
||||
title_row "$LIB/Junk Numbers/Attack Pack S03E2019.mkv" \
|
||||
"Attack Pack" 2013 3 2019 "" "" "A Title tvdb Holds" "4242"
|
||||
plan
|
||||
want="$LIB/Attack Pack (2013) [tvdbid-4242]/Season 03"
|
||||
want+="/Attack Pack (2013) - S03E07 - A Title tvdb Holds.mkv"
|
||||
[ "$(dst "$LIB/Junk Numbers/Attack Pack S03E2019.mkv")" = "$want" ]
|
||||
}
|
||||
|
||||
@test "a resolution glued to the marker still reads as one episode" {
|
||||
file "$LIB/Junk Numbers/Glued Mark S01E02-1080p.mkv" 5000
|
||||
title_row "$LIB/Junk Numbers/Glued Mark S01E02-1080p.mkv" \
|
||||
"The Examples" 2010 1 2 "" "" "Second Look"
|
||||
plan
|
||||
[ "$(dst "$LIB/Junk Numbers/Glued Mark S01E02-1080p.mkv")" \
|
||||
= "$LIB/The Examples (2010)/Season 01/The Examples (2010) - S01E02 - Second Look.mkv" ]
|
||||
}
|
||||
|
||||
# Anime numbering is the one place a tvdb title can be confidently wrong: a pack
|
||||
# numbered by its release group still names S05E01, and tvdb answers that slot with
|
||||
# a different episode. A season the authority does not hold, or holds fewer of than
|
||||
@@ -655,6 +695,20 @@ plan_map() { # the plan, with the operator's map in han
|
||||
[ "$status" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "a map row whose folder is on the array but never became a series says so" {
|
||||
# The array holds three folders exactly like this one: a credits/extras pack
|
||||
# the operator mapped to a record, whose every file is bonus material with no
|
||||
# series to own it. The folder IS on the array, and telling the operator the
|
||||
# pack moved sends them hunting for a folder that never left.
|
||||
printf 'Credit Pack\t114801\tthe record tvdb names\n' > "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
file "$LIB/Credit Pack/01. Extras/NCOP 01.mkv" 1200
|
||||
plan_map "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
grep -q "bonus material with no series to own it" "$EXC"
|
||||
grep -q "never became a series" "$EXC"
|
||||
run grep "no folder on the array" "$EXC"
|
||||
[ "$status" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "a map line whose id is not a number stops the plan" {
|
||||
printf 'Black Clover 001\tnotanid\ttypo\n' > "$BATS_TEST_TMPDIR/ids.tsv"
|
||||
file "$LIB/Black Clover 001/Season 01/Black.Clover.001.mkv" 5000
|
||||
|
||||
Reference in new issue
Block a user