fix(fleet): refuse a container start until the mount it binds has answered
Dankserver rebooted with the array's mount and the media containers racing. A rootless quadlet cannot order itself against the system's mnt-entropy.mount, so `RequiresMountsFor=/mnt/entropy` is inert in the user manager: podman resolved the bind against the pre-mount directory, Jellyfin and Plex came up blind, and Jellyfin read an empty library and began `Removing item, Type: "Season"` by the thousand. Copyparty had no guard at all. `scripts/wait-for-mount.sh` is the gate: it waits for every named mountpoint, then refuses by name. `ConditionPathIsMountPoint=` is NOT the fix — measured on the host, an unmet condition logs `skipped, unmet condition check` with NRestarts=0 and no start attempt ever follows, which is fail-dead, not fail-closed. Jellyfin and Plex swap the condition for the gate; Copyparty gets one; the law is recorded in AGENTS.md and the view page. Verified on the host: the 5.7.0 quadlet generator accepts ExecStartPre in [Service] (exit 0, all three units generated, gate present), the gate waits for a mount that appears mid-window and proceeds, and it refuses a plain directory after its window naming the path. tests/media_overlay.bats pinned the old mechanism by name; its intent — both media servers refuse to start without the view — now rides the gate.
This commit is contained in:
1 parent
75b04dc313
commit
e82c8b2d06
8 files changed
+217
-16
No files matched your search
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
# Seam: a container that binds the entropy array must refuse to start until the
|
||||
# thing it binds is actually mounted.
|
||||
#
|
||||
# Live 2026-10-31, Dankserver: the array's systemd mount STARTED eight seconds
|
||||
# before the media containers were created, but a rootless quadlet cannot order
|
||||
# itself against a SYSTEM mount unit — `RequiresMountsFor=/mnt/entropy` resolves
|
||||
# to nothing in the user manager and is simply inert. So podman resolved the bind
|
||||
# against the pre-mount directory (which held one stale subdirectory) and both
|
||||
# Jellyfin and Plex came up blind: their library root looked empty, Jellyfin
|
||||
# logged thousands of `Removing item, Type: "Season"`, and the array sat at 80 %
|
||||
# util doing nothing useful until the host went down. A blind bind is not a
|
||||
# cosmetic fault — it is a metadata-deleting machine.
|
||||
#
|
||||
# The measured fix is `ExecStartPre=<repo>/scripts/wait-for-mount.sh <source>`:
|
||||
# it WAITS for the mount and then refuses, so `Restart=always` carries the unit
|
||||
# across a slow mount. A `ConditionPathIsMountPoint=` was measured on the same
|
||||
# host and is the wrong tool: an unmet condition logs `skipped, unmet condition
|
||||
# check` with NRestarts=0 and never a start attempt — the unit stays dead until
|
||||
# a human starts it. That is fail-dead, not fail-closed.
|
||||
#
|
||||
# Tests assert only observable behavior of the gate (exit status and the name it
|
||||
# refuses by) and the structural law over the quadlets; never the internals.
|
||||
|
||||
load helpers
|
||||
|
||||
GATE="$REPO_ROOT/scripts/wait-for-mount.sh"
|
||||
|
||||
@test "the gate accepts a path that really is a mountpoint" {
|
||||
run "$GATE" -t 2 /proc
|
||||
[ "$status" -eq 0 ] || { printf '%s\n' "$output" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "a plain directory is refused, and refused BY NAME" {
|
||||
dir="$BATS_TEST_TMPDIR/not-a-mount"
|
||||
mkdir "$dir"
|
||||
run "$GATE" -t 1 "$dir"
|
||||
[ "$status" -ne 0 ] || { printf 'accepted a plain directory\n' >&2; return 1; }
|
||||
[[ "$output" == *"$dir"* ]] || { printf 'refusal did not name the path: %s\n' "$output" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "a path that does not exist is refused by name" {
|
||||
run "$GATE" -t 1 "$BATS_TEST_TMPDIR/does/not/exist"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"$BATS_TEST_TMPDIR/does/not/exist"* ]]
|
||||
}
|
||||
|
||||
@test "the gate holds the deadline instead of deciding on one look" {
|
||||
# A single check that failed at the wrong instant is the boot race itself:
|
||||
# the mount lands a second later and the unit is dead anyway. So the window
|
||||
# must be spent waiting, and that is observable from outside as elapsed time.
|
||||
dir="$BATS_TEST_TMPDIR/never"
|
||||
mkdir "$dir"
|
||||
start=$SECONDS
|
||||
run "$GATE" -t 4 "$dir"
|
||||
elapsed=$((SECONDS - start))
|
||||
[ "$status" -ne 0 ]
|
||||
[ "$elapsed" -ge 4 ] || { printf 'gave up after %ss, window was 4s\n' "$elapsed" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "the gate accepts a mount that arrives during its window" {
|
||||
dir="$BATS_TEST_TMPDIR/later"
|
||||
mkdir "$dir"
|
||||
if ! mount --bind /proc "$dir" 2>/dev/null; then
|
||||
skip "creating a mount needs privileges this box does not grant"
|
||||
fi
|
||||
run "$GATE" -t 4 "$dir"
|
||||
umount "$dir" 2>/dev/null || sudo -n umount "$dir" 2>/dev/null || true
|
||||
[ "$status" -eq 0 ] || { printf 'gate did not wait for a mount that arrived: %s\n' "$output" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "no unit without the gate binds the array or the metadata view" {
|
||||
bad=""
|
||||
while IFS= read -r unit; do
|
||||
sources=$(sed -n 's/^Volume=\([^:]*\):.*/\1/p' "$unit" \
|
||||
| grep -E '^(/mnt/entropy|/var/lib/media-meta/view)$' | sort -u)
|
||||
[ -n "$sources" ] || continue
|
||||
for src in $sources; do
|
||||
grep -q "ExecStartPre=.*wait-for-mount.sh.* $src\b" "$unit" \
|
||||
|| bad="$bad$unit binds $src with no start gate"$'\n'
|
||||
done
|
||||
done < <(find "$REPO_ROOT"/dankserver "$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro \
|
||||
-path '*/quadlet/*.container' -not -path '*.AGENTTRASH*')
|
||||
[ -z "$bad" ] || { printf 'blind binds:\n%s' "$bad" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "no gated unit keeps the fail-dead ConditionPathIsMountPoint" {
|
||||
bad=""
|
||||
while IFS= read -r unit; do
|
||||
grep -q "wait-for-mount.sh" "$unit" || continue
|
||||
grep -q "ConditionPathIsMountPoint" "$unit" \
|
||||
&& bad="$bad$unit: a gate and an unmet-condition skip cannot coexist"$'\n'
|
||||
done < <(find "$REPO_ROOT"/dankserver "$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro \
|
||||
-path '*/quadlet/*.container' -not -path '*.AGENTTRASH*')
|
||||
[ -z "$bad" ] || { printf 'fail-dead conditions:\n%s' "$bad" >&2; return 1; }
|
||||
}
|
||||
Reference in new issue
Block a user