Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f7083f175d |
No files matched your search
@@ -156,15 +156,6 @@ Unit requirements:
|
|||||||
(`<host>/<svc>/quadlet/`), is installed by the deployer, and is covered by
|
(`<host>/<svc>/quadlet/`), is installed by the deployer, and is covered by
|
||||||
the quadlet test seam. The host-side unit is generated state; the repo file
|
the quadlet test seam. The host-side unit is generated state; the repo file
|
||||||
is the source of truth.
|
is the source of truth.
|
||||||
- **Bind-source gate** — a unit whose `Volume=` source is a mount this host owns
|
|
||||||
(the entropy array, the metadata view) carries
|
|
||||||
`ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh <source>`.
|
|
||||||
Measured 2026-10-31: in a **rootless** unit `RequiresMountsFor=` is inert (the
|
|
||||||
user manager cannot order against the system's mount unit), and
|
|
||||||
`ConditionPathIsMountPoint=` is fail-*dead* — an unmet condition logs `skipped,
|
|
||||||
unmet condition check` with `NRestarts=0` and no start attempt ever follows.
|
|
||||||
Bind the array before it is mounted and the container sees the empty pre-mount
|
|
||||||
directory as truth; a media scan then deletes the library it cannot see.
|
|
||||||
|
|
||||||
Before declaring a service deployed: `systemctl --user is-enabled <name>` →
|
Before declaring a service deployed: `systemctl --user is-enabled <name>` →
|
||||||
`enabled`, `is-active` → `active`, and the container answers its health check.
|
`enabled`, `is-active` → `active`, and the container answers its health check.
|
||||||
|
|||||||
@@ -36,9 +36,6 @@ Volume=/home/dank/Containers/copyparty/cfg:/cfg
|
|||||||
PublishPort=3923:3923
|
PublishPort=3923:3923
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
# /w is the array: serving the pre-mount directory instead of the volume shows an
|
|
||||||
# empty share and nothing complains, so the mount has to answer first.
|
|
||||||
ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /mnt/entropy
|
|
||||||
EnvironmentFile=/home/dank/Containers/copyparty/env
|
EnvironmentFile=/home/dank/Containers/copyparty/env
|
||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=5
|
RestartSec=5
|
||||||
|
|||||||
@@ -43,10 +43,11 @@ PublishPort=1900:1900/udp
|
|||||||
Volume=/home/dank/Containers/jellyfin/config:/config
|
Volume=/home/dank/Containers/jellyfin/config:/config
|
||||||
Volume=/home/dank/Containers/jellyfin/cache:/cache
|
Volume=/home/dank/Containers/jellyfin/cache:/cache
|
||||||
# The view, writable: Jellyfin is the metadata writer (movie.nfo / tvshow.nfo /
|
# The view, writable: Jellyfin is the metadata writer (movie.nfo / tvshow.nfo /
|
||||||
# episode.nfo + artwork) and those bytes belong on the SSD. The gate in [Service]
|
# episode.nfo + artwork) and those bytes belong on the SSD. The condition is the
|
||||||
# is the guard: started against a plain, unmounted view directory a scan decides
|
# fail-closed guard — starting against a plain, unmounted view directory would let
|
||||||
# the library is empty and deletes what it cannot see.
|
# a scan conclude the library is empty.
|
||||||
Volume=/var/lib/media-meta/view:/data
|
Volume=/var/lib/media-meta/view:/data
|
||||||
|
UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view
|
||||||
AddDevice=/dev/dri
|
AddDevice=/dev/dri
|
||||||
GroupAdd=keep-groups
|
GroupAdd=keep-groups
|
||||||
HealthCmd=curl -f http://localhost:8096/health
|
HealthCmd=curl -f http://localhost:8096/health
|
||||||
@@ -55,9 +56,6 @@ HealthTimeout=10s
|
|||||||
HealthRetries=3
|
HealthRetries=3
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
# Refuse to start until the bind source is a real mount — see the header of
|
|
||||||
# scripts/wait-for-mount.sh for the blind-boot incident this prevents.
|
|
||||||
ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /var/lib/media-meta/view /mnt/entropy
|
|
||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=5
|
RestartSec=5
|
||||||
|
|
||||||
|
|||||||
@@ -16,13 +16,10 @@ Volume=/var/lib/plexmediaserver:/config
|
|||||||
# to the overlay view re-points Plex at the sidecars without touching its DB.
|
# to the overlay view re-points Plex at the sidecars without touching its DB.
|
||||||
# Read-only: Plex consumes the NFO files Jellyfin writes and writes no metadata.
|
# Read-only: Plex consumes the NFO files Jellyfin writes and writes no metadata.
|
||||||
Volume=/var/lib/media-meta/view:/mnt/entropy:ro
|
Volume=/var/lib/media-meta/view:/mnt/entropy:ro
|
||||||
|
UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view
|
||||||
PublishPort=32400:32400
|
PublishPort=32400:32400
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
# Refuse to start until the bind source is a real mount: Plex blind is how paths
|
|
||||||
# get marked missing in its own database. Both the view and the array under it
|
|
||||||
# must answer — scripts/wait-for-mount.sh names the 2026-10-31 incident.
|
|
||||||
ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /var/lib/media-meta/view /mnt/entropy
|
|
||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=5
|
RestartSec=5
|
||||||
|
|
||||||
|
|||||||
+68
-9
@@ -59,10 +59,18 @@ def journal_default(root):
|
|||||||
|
|
||||||
|
|
||||||
class Journal:
|
class Journal:
|
||||||
"""Append-only JSONL of intents. Writes are fsynced before the tool acts."""
|
"""Append-only JSONL of intents. Writes are fsynced before the tool acts.
|
||||||
|
|
||||||
def __init__(self, path, writing):
|
``batch`` is how many intents may be written before one fsync covers them.
|
||||||
|
The rule the file exists for is unchanged — an intent is on disk before the
|
||||||
|
move it authorises runs — so the caller must call ``sync()`` before it
|
||||||
|
performs a batch's renames; only the number of flushes falls. 1 = a flush
|
||||||
|
per intent (the original behaviour).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, path, writing, batch=1):
|
||||||
self.path, self.writing = path, writing
|
self.path, self.writing = path, writing
|
||||||
|
self.batch = max(1, int(batch))
|
||||||
self.records = []
|
self.records = []
|
||||||
if os.path.exists(path):
|
if os.path.exists(path):
|
||||||
for line in open(path):
|
for line in open(path):
|
||||||
@@ -70,11 +78,15 @@ class Journal:
|
|||||||
if line:
|
if line:
|
||||||
self.records.append(json.loads(line))
|
self.records.append(json.loads(line))
|
||||||
self._fh = None
|
self._fh = None
|
||||||
|
self._unsynced = 0
|
||||||
|
# (src, dst) index: already() is called three times per move, so an
|
||||||
|
# any() over every record written so far made a 58k-move pass quadratic.
|
||||||
|
self._intents = {(r.get("src"), r.get("dst"))
|
||||||
|
for r in self.records if r.get("op") == "move"}
|
||||||
|
|
||||||
def already(self, dst, src):
|
def already(self, dst, src):
|
||||||
"""An intent for this exact (src, dst) was written by an earlier run."""
|
"""An intent for this exact (src, dst) was written by an earlier run."""
|
||||||
return any(r.get("op") == "move" and r.get("dst") == dst
|
return (src, dst) in self._intents
|
||||||
and r.get("src") == src for r in self.records)
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def rolled_back(self):
|
def rolled_back(self):
|
||||||
@@ -82,16 +94,28 @@ class Journal:
|
|||||||
|
|
||||||
def record(self, **fields):
|
def record(self, **fields):
|
||||||
self.records.append(fields)
|
self.records.append(fields)
|
||||||
|
if fields.get("op") == "move":
|
||||||
|
self._intents.add((fields.get("src"), fields.get("dst")))
|
||||||
if not self.writing:
|
if not self.writing:
|
||||||
return
|
return
|
||||||
if self._fh is None:
|
if self._fh is None:
|
||||||
self._fh = open(self.path, "a")
|
self._fh = open(self.path, "a")
|
||||||
self._fh.write(json.dumps(fields) + "\n")
|
self._fh.write(json.dumps(fields) + "\n")
|
||||||
self._fh.flush()
|
self._fh.flush()
|
||||||
|
self._unsynced += 1
|
||||||
|
if self._unsynced >= self.batch:
|
||||||
|
self.sync()
|
||||||
|
|
||||||
|
def sync(self):
|
||||||
|
"""Make every intent written so far durable. No-op when nothing pending."""
|
||||||
|
if not self.writing or self._fh is None or not self._unsynced:
|
||||||
|
return
|
||||||
os.fsync(self._fh.fileno())
|
os.fsync(self._fh.fileno())
|
||||||
|
self._unsynced = 0
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
if self._fh:
|
if self._fh:
|
||||||
|
self.sync()
|
||||||
self._fh.close()
|
self._fh.close()
|
||||||
|
|
||||||
|
|
||||||
@@ -177,14 +201,30 @@ def do_apply(args, root, journal, mkdirs, keeps, moves):
|
|||||||
print(f" would move {src} -> {dst}")
|
print(f" would move {src} -> {dst}")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
batch = max(1, args.batch)
|
||||||
made = {r.get("dst") for r in journal.records if r.get("op") == "mkdir"}
|
made = {r.get("dst") for r in journal.records if r.get("op") == "mkdir"}
|
||||||
dirs_new = 0
|
dirs_new = 0
|
||||||
|
queued = []
|
||||||
|
|
||||||
|
def drain_dirs():
|
||||||
|
"""Create the queued folders only after their intents are durable."""
|
||||||
|
nonlocal dirs_new
|
||||||
|
if not queued:
|
||||||
|
return
|
||||||
|
journal.sync()
|
||||||
|
for d in queued:
|
||||||
|
os.makedirs(d, exist_ok=True)
|
||||||
|
dirs_new += 1
|
||||||
|
queued.clear()
|
||||||
|
|
||||||
for d in sorted(mkdirs, key=lambda q: q.count(os.sep)):
|
for d in sorted(mkdirs, key=lambda q: q.count(os.sep)):
|
||||||
if d in made or os.path.isdir(d):
|
if d in made or os.path.isdir(d):
|
||||||
continue
|
continue
|
||||||
journal.record(op="mkdir", dst=d)
|
journal.record(op="mkdir", dst=d)
|
||||||
os.makedirs(d, exist_ok=True)
|
queued.append(d)
|
||||||
dirs_new += 1
|
if len(queued) >= batch:
|
||||||
|
drain_dirs()
|
||||||
|
drain_dirs()
|
||||||
|
|
||||||
# same-filesystem gate: rename cannot cross a device, and a silent copy of a
|
# same-filesystem gate: rename cannot cross a device, and a silent copy of a
|
||||||
# library this size is not an option anyone would choose.
|
# library this size is not an option anyone would choose.
|
||||||
@@ -201,6 +241,19 @@ def do_apply(args, root, journal, mkdirs, keeps, moves):
|
|||||||
return 2
|
return 2
|
||||||
|
|
||||||
moved = done = 0
|
moved = done = 0
|
||||||
|
pending = []
|
||||||
|
|
||||||
|
def drain_moves():
|
||||||
|
"""Perform the queued renames only after their intents are on disk."""
|
||||||
|
nonlocal moved
|
||||||
|
if not pending:
|
||||||
|
return
|
||||||
|
journal.sync()
|
||||||
|
for s, d in pending:
|
||||||
|
os.rename(s, d)
|
||||||
|
moved += 1
|
||||||
|
pending.clear()
|
||||||
|
|
||||||
for src, dst, _kind in moves:
|
for src, dst, _kind in moves:
|
||||||
if journal.already(dst, src):
|
if journal.already(dst, src):
|
||||||
if os.path.exists(dst) and not os.path.exists(src):
|
if os.path.exists(dst) and not os.path.exists(src):
|
||||||
@@ -213,8 +266,10 @@ def do_apply(args, root, journal, mkdirs, keeps, moves):
|
|||||||
st = os.stat(src)
|
st = os.stat(src)
|
||||||
journal.record(op="move", src=src, dst=dst, bytes=st.st_size, mode=st.st_mode,
|
journal.record(op="move", src=src, dst=dst, bytes=st.st_size, mode=st.st_mode,
|
||||||
mtime=st.st_mtime, ino=st.st_ino, dev=st.st_dev)
|
mtime=st.st_mtime, ino=st.st_ino, dev=st.st_dev)
|
||||||
os.rename(src, dst)
|
pending.append((src, dst))
|
||||||
moved += 1
|
if len(pending) >= batch:
|
||||||
|
drain_moves()
|
||||||
|
drain_moves()
|
||||||
print(f"applied: {moved} moved, {done} already done in an earlier run,"
|
print(f"applied: {moved} moved, {done} already done in an earlier run,"
|
||||||
f" {dirs_new} folders created, {len(keeps)} files stayed put")
|
f" {dirs_new} folders created, {len(keeps)} files stayed put")
|
||||||
print(f"journal: {journal.path} — undo with: media-apply.py rollback"
|
print(f"journal: {journal.path} — undo with: media-apply.py rollback"
|
||||||
@@ -418,13 +473,17 @@ def main():
|
|||||||
ap.add_argument("--metadata-view", default=METADATA_VIEW,
|
ap.add_argument("--metadata-view", default=METADATA_VIEW,
|
||||||
help="overlay the media servers read through; the sidecar"
|
help="overlay the media servers read through; the sidecar"
|
||||||
" note prints only when this path is a mountpoint")
|
" note prints only when this path is a mountpoint")
|
||||||
|
ap.add_argument("--batch", type=int, default=64,
|
||||||
|
help="journal intents per fsync (1 = flush before every move)."
|
||||||
|
" An intent is still durable before the move it authorises;"
|
||||||
|
" only the number of flushes falls")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
root = os.path.abspath(args.root)
|
root = os.path.abspath(args.root)
|
||||||
if not os.path.isdir(root):
|
if not os.path.isdir(root):
|
||||||
raise Refuse(f"no library at {root}")
|
raise Refuse(f"no library at {root}")
|
||||||
journal_path = args.journal or journal_default(root)
|
journal_path = args.journal or journal_default(root)
|
||||||
journal = Journal(journal_path, args.apply)
|
journal = Journal(journal_path, args.apply, batch=args.batch)
|
||||||
try:
|
try:
|
||||||
if args.cmd in ("rollback", "verify", "sweep") and not os.path.exists(journal_path):
|
if args.cmd in ("rollback", "verify", "sweep") and not os.path.exists(journal_path):
|
||||||
raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
|
raise Refuse(f"no journal at {journal_path} — nothing to {args.cmd}")
|
||||||
|
|||||||
@@ -1,74 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# wait-for-mount.sh — refuse a container start until what it binds is mounted.
|
|
||||||
#
|
|
||||||
# Why this exists (live 2026-10-31, Dankserver): a rootless quadlet cannot order
|
|
||||||
# itself against a SYSTEM mount unit. `RequiresMountsFor=/mnt/entropy` in a user
|
|
||||||
# unit resolves to nothing and is inert, so when the array's mount finished a
|
|
||||||
# second after podman resolved the bind, Jellyfin and Plex started against the
|
|
||||||
# pre-mount directory. Their library root looked EMPTY and Jellyfin began
|
|
||||||
# deleting metadata by the thousand (`Removing item, Type: "Season"`). A blind
|
|
||||||
# bind is not a cosmetic fault; it is a metadata-deleting machine.
|
|
||||||
#
|
|
||||||
# A `ConditionPathIsMountPoint=` is NOT the fix, measured on the same host: an
|
|
||||||
# unmet condition logs "skipped, unmet condition check", NRestarts=0, and no
|
|
||||||
# start attempt ever follows — the unit stays dead until a human starts it.
|
|
||||||
# That is fail-dead. This script WAITS (the mount usually lands within seconds)
|
|
||||||
# and only then refuses, so `Restart=always` carries the unit across the race.
|
|
||||||
#
|
|
||||||
# usage: wait-for-mount.sh [-t SECONDS] MOUNTPOINT [MOUNTPOINT...]
|
|
||||||
# exit 0 every path is a mountpoint
|
|
||||||
# exit 1 a path is still not a mountpoint after the window (named on stderr)
|
|
||||||
# exit 2 usage error
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
usage() { printf 'usage: %s [-t SECONDS] MOUNTPOINT [MOUNTPOINT...]\n' "$0"; }
|
|
||||||
|
|
||||||
window=90
|
|
||||||
while getopts ":t:h" opt; do
|
|
||||||
case "$opt" in
|
|
||||||
t) window=$OPTARG ;;
|
|
||||||
h) usage; exit 0 ;;
|
|
||||||
*) usage >&2; exit 2 ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
shift $((OPTIND - 1))
|
|
||||||
|
|
||||||
[ "$#" -ge 1 ] || { usage >&2; exit 2; }
|
|
||||||
case "$window" in
|
|
||||||
'' | *[!0-9]*) printf 'refused: -t wants whole seconds, got %s\n' "$window" >&2; exit 2 ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
is_mount() { findmnt -M "$1" -rno TARGET >/dev/null 2>&1; }
|
|
||||||
|
|
||||||
report() {
|
|
||||||
local why=$1; shift
|
|
||||||
printf '%s:' "$why"
|
|
||||||
for p in "$@"; do printf ' %s' "$p"; done
|
|
||||||
printf '\n'
|
|
||||||
}
|
|
||||||
|
|
||||||
deadline=$(( $(date +%s) + window ))
|
|
||||||
warned=0
|
|
||||||
while :; do
|
|
||||||
missing=()
|
|
||||||
for path in "$@"; do
|
|
||||||
is_mount "$path" || missing+=("$path")
|
|
||||||
done
|
|
||||||
[ "${#missing[@]}" -eq 0 ] && break
|
|
||||||
|
|
||||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
|
||||||
report "refused: still not mounted after ${window}s" "${missing[@]}" >&2
|
|
||||||
printf 'refusing to start: a container bind-mounted here would see the empty\n' >&2
|
|
||||||
printf 'directory underneath and conclude the library is empty — which is how a\n' >&2
|
|
||||||
printf 'scan begins deleting metadata it cannot find.\n' >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [ "$warned" -eq 0 ]; then
|
|
||||||
report "waiting for mount(s) this unit depends on" "${missing[@]}"
|
|
||||||
warned=1
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
|
|
||||||
[ "$warned" -eq 0 ] || printf 'mounted; proceeding.\n'
|
|
||||||
exit 0
|
|
||||||
@@ -46,11 +46,10 @@ view really is a mountpoint.
|
|||||||
| Installer | `scripts/media-view.sh` — `plan` / `install` / `verify` / `status` / `reap` |
|
| Installer | `scripts/media-view.sh` — `plan` / `install` / `verify` / `status` / `reap` |
|
||||||
| Written by | Jellyfin only (Plex binds read-only); nothing else in the fleet touches it |
|
| Written by | Jellyfin only (Plex binds read-only); nothing else in the fleet touches it |
|
||||||
|
|
||||||
Both consumers gate their start on the view with
|
Both consumers guard the view with
|
||||||
`ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh` (it waits
|
`UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view`, and
|
||||||
for the mount, then refuses), and `scripts/deploy.sh dankserver --service
|
`scripts/deploy.sh dankserver --service jellyfin|plex` calls `media-view.sh
|
||||||
jellyfin|plex` calls `media-view.sh install` + `verify` before it restarts
|
install` + `verify` before it restarts either container.
|
||||||
either container.
|
|
||||||
|
|
||||||
| Consumer | Bind | Mode | Why that path |
|
| Consumer | Bind | Mode | Why that path |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
@@ -81,20 +80,11 @@ the defaults are the host values above.
|
|||||||
refuse via `lsblk -dno ROTA` if the upper base lives on the array: an overlay
|
refuse via `lsblk -dno ROTA` if the upper base lives on the array: an overlay
|
||||||
whose upper is the spinners is the original problem with extra steps, and it
|
whose upper is the spinners is the original problem with extra steps, and it
|
||||||
silently turns every metadata write back into a seek.
|
silently turns every metadata write back into a seek.
|
||||||
- **A container that binds the array must refuse to start until the mount
|
- **Never mount the view over an unmounted lower.** `RequiresMountsFor=/mnt/entropy`
|
||||||
answers.** Live 2026-10-31: the array's mount and the media containers raced at
|
plus the container `ConditionPathIsMountPoint` guards: an empty `lowerdir`
|
||||||
boot, podman resolved the bind against the pre-mount directory, and both
|
presents an *empty library* as truth, and a media scan would then clear the
|
||||||
servers came up blind — Jellyfin read an empty library and began
|
database of everything. systemd does not create a mount point, so `install`
|
||||||
`Removing item, Type: "Season"` by the thousand. Two dependencies people reach
|
creates `view` itself; without it the mount simply fails.
|
||||||
for do not work here, both measured on that host: `RequiresMountsFor=` in a
|
|
||||||
**rootless** unit is inert (the user manager cannot order against the system's
|
|
||||||
`mnt-entropy.mount`), and `ConditionPathIsMountPoint=` is fail-*dead* — an
|
|
||||||
unmet condition logs `skipped, unmet condition check` with `NRestarts=0` and no
|
|
||||||
start attempt ever follows, so the unit waits for a human. The working guard is
|
|
||||||
`scripts/wait-for-mount.sh` as `ExecStartPre`: it waits out the race, refuses by
|
|
||||||
name after its window, and `Restart=always` carries the unit from there.
|
|
||||||
systemd does not create a mount point, so `install` creates `view` itself;
|
|
||||||
without it the mount simply fails.
|
|
||||||
- **Never delete through the view.** A delete writes an overlayfs whiteout
|
- **Never delete through the view.** A delete writes an overlayfs whiteout
|
||||||
(`.wh.<name>`) in the upper layer, which hides the array's copy without
|
(`.wh.<name>`) in the upper layer, which hides the array's copy without
|
||||||
removing it — the file reappears the moment the view is unmounted, and the
|
removing it — the file reappears the moment the view is unmounted, and the
|
||||||
@@ -117,8 +107,7 @@ the defaults are the host values above.
|
|||||||
|
|
||||||
| Symptom | Meaning |
|
| Symptom | Meaning |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `jellyfin.service` / `plex.service` fail with `refused: still not mounted after 90s: …` | The view or the array is not mounted. Do not start them anyway: `scripts/media-view.sh install`, then `verify`. |
|
| `jellyfin.service` / `plex.service` show `ConditionPathIsMountPoint=... was not met` and stay inactive | The view is not mounted. Do not start them anyway: `scripts/media-view.sh install`, then `verify`. |
|
||||||
| Servers answer, library looks empty, Jellyfin logs `Removing item, Type: "Season"` by the thousand | Blind bind — the container started before the mount and bound the pre-mount directory. `podman exec jellyfin ls /data` against `ls /mnt/entropy` is the whole diagnosis; restarting the container re-resolves the bind. |
|
|
||||||
| Jellyfin log: `Read-only file system : '/data/Movies/…/movie.nfo'` | The container is binding the raw array, not the view — the quadlet drifted back or the view is gone. `media-view.sh status`. |
|
| Jellyfin log: `Read-only file system : '/data/Movies/…/movie.nfo'` | The container is binding the raw array, not the view — the quadlet drifted back or the view is gone. `media-view.sh status`. |
|
||||||
| A folder appears in a media server with artwork/`.nfo` but no video | Ghost metadata from a relayout. `media-view.sh reap`. |
|
| A folder appears in a media server with artwork/`.nfo` but no video | Ghost metadata from a relayout. `media-view.sh reap`. |
|
||||||
| `verify` prints `refused: …/view is not a mountpoint` | A plain directory is sitting where the overlay belongs. That is the fail-closed guard working; `install` on the box. |
|
| `verify` prints `refused: …/view is not a mountpoint` | A plain directory is sitting where the overlay belongs. That is the fail-closed guard working; `install` on the box. |
|
||||||
|
|||||||
@@ -81,6 +81,13 @@ An apply also leaves the release folders it emptied standing where they were —
|
|||||||
hundreds of them at the library root — which is what `sweep` removes, dry-run
|
hundreds of them at the library root — which is what `sweep` removes, dry-run
|
||||||
first, from the same journal.
|
first, from the same journal.
|
||||||
|
|
||||||
|
`--batch N` (default 64) makes the journal durable per batch instead of per
|
||||||
|
move: N intents are written and fsynced, then those N rename. The rule a crash
|
||||||
|
relies on — no rename runs before its intent is on disk — is unchanged, so an
|
||||||
|
interrupted apply still replays the batch and resumes with nothing lost. Batch 1
|
||||||
|
is the old behaviour; the only reason to name it is debugging, because on the
|
||||||
|
spinning array it spends most of a large apply's wall time in fsyncs.
|
||||||
|
|
||||||
After it lands, both servers need a scan or the change is invisible. [[Plex]] and
|
After it lands, both servers need a scan or the change is invisible. [[Plex]] and
|
||||||
[[Jellyfin]] own the exact requests; two laws hold at this step: `204` is
|
[[Jellyfin]] own the exact requests; two laws hold at this step: `204` is
|
||||||
acceptance and anything else queued nothing — re-issue the identical request
|
acceptance and anything else queued nothing — re-issue the identical request
|
||||||
|
|||||||
@@ -1,96 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Seam: a container that binds the entropy array must refuse to start until the
|
|
||||||
# thing it binds is actually mounted.
|
|
||||||
#
|
|
||||||
# Live 2026-10-31, Dankserver: the array's systemd mount STARTED eight seconds
|
|
||||||
# before the media containers were created, but a rootless quadlet cannot order
|
|
||||||
# itself against a SYSTEM mount unit — `RequiresMountsFor=/mnt/entropy` resolves
|
|
||||||
# to nothing in the user manager and is simply inert. So podman resolved the bind
|
|
||||||
# against the pre-mount directory (which held one stale subdirectory) and both
|
|
||||||
# Jellyfin and Plex came up blind: their library root looked empty, Jellyfin
|
|
||||||
# logged thousands of `Removing item, Type: "Season"`, and the array sat at 80 %
|
|
||||||
# util doing nothing useful until the host went down. A blind bind is not a
|
|
||||||
# cosmetic fault — it is a metadata-deleting machine.
|
|
||||||
#
|
|
||||||
# The measured fix is `ExecStartPre=<repo>/scripts/wait-for-mount.sh <source>`:
|
|
||||||
# it WAITS for the mount and then refuses, so `Restart=always` carries the unit
|
|
||||||
# across a slow mount. A `ConditionPathIsMountPoint=` was measured on the same
|
|
||||||
# host and is the wrong tool: an unmet condition logs `skipped, unmet condition
|
|
||||||
# check` with NRestarts=0 and never a start attempt — the unit stays dead until
|
|
||||||
# a human starts it. That is fail-dead, not fail-closed.
|
|
||||||
#
|
|
||||||
# Tests assert only observable behavior of the gate (exit status and the name it
|
|
||||||
# refuses by) and the structural law over the quadlets; never the internals.
|
|
||||||
|
|
||||||
load helpers
|
|
||||||
|
|
||||||
GATE="$REPO_ROOT/scripts/wait-for-mount.sh"
|
|
||||||
|
|
||||||
@test "the gate accepts a path that really is a mountpoint" {
|
|
||||||
run "$GATE" -t 2 /proc
|
|
||||||
[ "$status" -eq 0 ] || { printf '%s\n' "$output" >&2; return 1; }
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "a plain directory is refused, and refused BY NAME" {
|
|
||||||
dir="$BATS_TEST_TMPDIR/not-a-mount"
|
|
||||||
mkdir "$dir"
|
|
||||||
run "$GATE" -t 1 "$dir"
|
|
||||||
[ "$status" -ne 0 ] || { printf 'accepted a plain directory\n' >&2; return 1; }
|
|
||||||
[[ "$output" == *"$dir"* ]] || { printf 'refusal did not name the path: %s\n' "$output" >&2; return 1; }
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "a path that does not exist is refused by name" {
|
|
||||||
run "$GATE" -t 1 "$BATS_TEST_TMPDIR/does/not/exist"
|
|
||||||
[ "$status" -ne 0 ]
|
|
||||||
[[ "$output" == *"$BATS_TEST_TMPDIR/does/not/exist"* ]]
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "the gate holds the deadline instead of deciding on one look" {
|
|
||||||
# A single check that failed at the wrong instant is the boot race itself:
|
|
||||||
# the mount lands a second later and the unit is dead anyway. So the window
|
|
||||||
# must be spent waiting, and that is observable from outside as elapsed time.
|
|
||||||
dir="$BATS_TEST_TMPDIR/never"
|
|
||||||
mkdir "$dir"
|
|
||||||
start=$SECONDS
|
|
||||||
run "$GATE" -t 4 "$dir"
|
|
||||||
elapsed=$((SECONDS - start))
|
|
||||||
[ "$status" -ne 0 ]
|
|
||||||
[ "$elapsed" -ge 4 ] || { printf 'gave up after %ss, window was 4s\n' "$elapsed" >&2; return 1; }
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "the gate accepts a mount that arrives during its window" {
|
|
||||||
dir="$BATS_TEST_TMPDIR/later"
|
|
||||||
mkdir "$dir"
|
|
||||||
if ! mount --bind /proc "$dir" 2>/dev/null; then
|
|
||||||
skip "creating a mount needs privileges this box does not grant"
|
|
||||||
fi
|
|
||||||
run "$GATE" -t 4 "$dir"
|
|
||||||
umount "$dir" 2>/dev/null || sudo -n umount "$dir" 2>/dev/null || true
|
|
||||||
[ "$status" -eq 0 ] || { printf 'gate did not wait for a mount that arrived: %s\n' "$output" >&2; return 1; }
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "no unit without the gate binds the array or the metadata view" {
|
|
||||||
bad=""
|
|
||||||
while IFS= read -r unit; do
|
|
||||||
sources=$(sed -n 's/^Volume=\([^:]*\):.*/\1/p' "$unit" \
|
|
||||||
| grep -E '^(/mnt/entropy|/var/lib/media-meta/view)$' | sort -u)
|
|
||||||
[ -n "$sources" ] || continue
|
|
||||||
for src in $sources; do
|
|
||||||
grep -q "ExecStartPre=.*wait-for-mount.sh.* $src\b" "$unit" \
|
|
||||||
|| bad="$bad$unit binds $src with no start gate"$'\n'
|
|
||||||
done
|
|
||||||
done < <(find "$REPO_ROOT"/dankserver "$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro \
|
|
||||||
-path '*/quadlet/*.container' -not -path '*.AGENTTRASH*')
|
|
||||||
[ -z "$bad" ] || { printf 'blind binds:\n%s' "$bad" >&2; return 1; }
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "no gated unit keeps the fail-dead ConditionPathIsMountPoint" {
|
|
||||||
bad=""
|
|
||||||
while IFS= read -r unit; do
|
|
||||||
grep -q "wait-for-mount.sh" "$unit" || continue
|
|
||||||
grep -q "ConditionPathIsMountPoint" "$unit" \
|
|
||||||
&& bad="$bad$unit: a gate and an unmet-condition skip cannot coexist"$'\n'
|
|
||||||
done < <(find "$REPO_ROOT"/dankserver "$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro \
|
|
||||||
-path '*/quadlet/*.container' -not -path '*.AGENTTRASH*')
|
|
||||||
[ -z "$bad" ] || { printf 'fail-dead conditions:\n%s' "$bad" >&2; return 1; }
|
|
||||||
}
|
|
||||||
@@ -61,10 +61,7 @@ UNIT="$REPO_ROOT/dankserver/host/systemd/media-meta-view.mount"
|
|||||||
@test "both media servers bind the view and refuse to start without it" {
|
@test "both media servers bind the view and refuse to start without it" {
|
||||||
for unit in "$JELLY" "$PLEX"; do
|
for unit in "$JELLY" "$PLEX"; do
|
||||||
grep -q "^Volume=$MOUNT:" "$unit" || return 1
|
grep -q "^Volume=$MOUNT:" "$unit" || return 1
|
||||||
# The refusal is a start GATE, not a Condition: measured on the host, a
|
grep -q "^UnitConfigItem=ConditionPathIsMountPoint=$MOUNT$" "$unit" || return 1
|
||||||
# unit whose Condition goes unmet is skipped forever (NRestarts=0, no
|
|
||||||
# start attempt) — fail-dead. The gate waits, then refuses.
|
|
||||||
grep -q "^ExecStartPre=.*wait-for-mount.sh .*$MOUNT" "$unit" || return 1
|
|
||||||
# The raw array must no longer be a container source: that bind is what
|
# The raw array must no longer be a container source: that bind is what
|
||||||
# made sidecar sharing impossible (read-only) and metadata slow (seeks).
|
# made sidecar sharing impossible (read-only) and metadata slow (seeks).
|
||||||
! grep -q '^Volume=/mnt/entropy' "$unit"
|
! grep -q '^Volume=/mnt/entropy' "$unit"
|
||||||
|
|||||||
@@ -494,6 +494,65 @@ for r in moves:
|
|||||||
PY
|
PY
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# The live pass paid an fsync per move on a spinning array (63 074 of them) and
|
||||||
|
# the run took 11 minutes for 58 278 renames. --batch N writes N intents, syncs
|
||||||
|
# once, then performs those N moves — the durability rule is unchanged (no move
|
||||||
|
# runs before its intent is on disk), only the number of flushes falls.
|
||||||
|
@test "a batched apply journals every move and lands the same tree" {
|
||||||
|
plan
|
||||||
|
JOURNAL="$BATS_TEST_TMPDIR/batched.jsonl"
|
||||||
|
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" \
|
||||||
|
--batch 4 --apply
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
local planned journalled
|
||||||
|
planned=$(awk -F'\t' '$1=="move"' "$OUT" | wc -l)
|
||||||
|
journalled=$(grep -c '"op": "move"' "$JOURNAL")
|
||||||
|
[ "$journalled" -eq "$planned" ]
|
||||||
|
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
python3 - "$OUT" <<'PY'
|
||||||
|
import os, sys
|
||||||
|
moves = [r for r in (l.rstrip("\n").split("\t") for l in open(sys.argv[1]))
|
||||||
|
if r[0] == "move"]
|
||||||
|
assert moves, "fixture planned no moves"
|
||||||
|
for r in moves:
|
||||||
|
assert os.path.exists(r[2]), f"target missing: {r[2]}"
|
||||||
|
assert not os.path.exists(r[1]), f"source left behind: {r[1]}"
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "an apply stopped mid-batch resumes without losing or double-journalling a move" {
|
||||||
|
plan
|
||||||
|
JOURNAL="$BATS_TEST_TMPDIR/interrupted-batch.jsonl"
|
||||||
|
python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" \
|
||||||
|
--batch 2 --apply > "$BATS_TEST_TMPDIR/first.log" 2>&1 &
|
||||||
|
local pid=$! i
|
||||||
|
# stop the run the moment a batch of intents is on disk (it may also have
|
||||||
|
# finished already — both outcomes must resume cleanly)
|
||||||
|
for i in $(seq 1 200); do
|
||||||
|
[ -f "$JOURNAL" ] && [ "$(grep -c '"op": "move"' "$JOURNAL" 2>/dev/null || true)" -ge 2 ] && break
|
||||||
|
sleep 0.02
|
||||||
|
done
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
wait "$pid" 2>/dev/null || true
|
||||||
|
|
||||||
|
run python3 "$APPLY" apply --plan "$OUT" --root "$LIB" --journal "$JOURNAL" --batch 2 --apply
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
run python3 "$APPLY" verify --plan "$OUT" --root "$LIB" --journal "$JOURNAL"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
python3 - "$OUT" "$JOURNAL" <<'PY'
|
||||||
|
import json, os, sys
|
||||||
|
moves = [r for r in (l.rstrip("\n").split("\t") for l in open(sys.argv[1]))
|
||||||
|
if r[0] == "move"]
|
||||||
|
journal = [json.loads(l) for l in open(sys.argv[2])]
|
||||||
|
dsts = [j["dst"] for j in journal if j.get("op") == "move"]
|
||||||
|
assert len(dsts) == len(set(dsts)), "a destination was journalled twice"
|
||||||
|
assert len(dsts) == len(moves), f"journalled {len(dsts)} of {len(moves)} moves"
|
||||||
|
for r in moves:
|
||||||
|
assert os.path.exists(r[2]) and not os.path.exists(r[1]), r[2]
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
@test "rollback puts every file back where the census found it" {
|
@test "rollback puts every file back where the census found it" {
|
||||||
plan
|
plan
|
||||||
apply_plan
|
apply_plan
|
||||||
|
|||||||
Reference in new issue
Block a user