From e82c8b2d06575cf572724288ad7a2962d314b707 Mon Sep 17 00:00:00 2001 From: dank Date: Sat, 31 Oct 2026 18:17:02 -0700 Subject: [PATCH] fix(fleet): refuse a container start until the mount it binds has answered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dankserver rebooted with the array's mount and the media containers racing. A rootless quadlet cannot order itself against the system's mnt-entropy.mount, so `RequiresMountsFor=/mnt/entropy` is inert in the user manager: podman resolved the bind against the pre-mount directory, Jellyfin and Plex came up blind, and Jellyfin read an empty library and began `Removing item, Type: "Season"` by the thousand. Copyparty had no guard at all. `scripts/wait-for-mount.sh` is the gate: it waits for every named mountpoint, then refuses by name. `ConditionPathIsMountPoint=` is NOT the fix — measured on the host, an unmet condition logs `skipped, unmet condition check` with NRestarts=0 and no start attempt ever follows, which is fail-dead, not fail-closed. Jellyfin and Plex swap the condition for the gate; Copyparty gets one; the law is recorded in AGENTS.md and the view page. Verified on the host: the 5.7.0 quadlet generator accepts ExecStartPre in [Service] (exit 0, all three units generated, gate present), the gate waits for a mount that appears mid-window and proceeds, and it refuses a plain directory after its window naming the path. tests/media_overlay.bats pinned the old mechanism by name; its intent — both media servers refuse to start without the view — now rides the gate. --- AGENTS.md | 9 ++ .../copyparty/quadlet/copyparty.container | 3 + .../jellyfin/quadlet/jellyfin.container | 10 +- dankserver/plex/quadlet/plex.container | 5 +- scripts/wait-for-mount.sh | 74 ++++++++++++++ .../Dankserver/Media Metadata View.md | 31 ++++-- tests/array_mount_gate.bats | 96 +++++++++++++++++++ tests/media_overlay.bats | 5 +- 8 files changed, 217 insertions(+), 16 deletions(-) create mode 100755 scripts/wait-for-mount.sh create mode 100755 tests/array_mount_gate.bats diff --git a/AGENTS.md b/AGENTS.md index 0206de6..363e88e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -156,6 +156,15 @@ Unit requirements: (`//quadlet/`), is installed by the deployer, and is covered by the quadlet test seam. The host-side unit is generated state; the repo file is the source of truth. +- **Bind-source gate** — a unit whose `Volume=` source is a mount this host owns + (the entropy array, the metadata view) carries + `ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh `. + Measured 2026-10-31: in a **rootless** unit `RequiresMountsFor=` is inert (the + user manager cannot order against the system's mount unit), and + `ConditionPathIsMountPoint=` is fail-*dead* — an unmet condition logs `skipped, + unmet condition check` with `NRestarts=0` and no start attempt ever follows. + Bind the array before it is mounted and the container sees the empty pre-mount + directory as truth; a media scan then deletes the library it cannot see. Before declaring a service deployed: `systemctl --user is-enabled ` → `enabled`, `is-active` → `active`, and the container answers its health check. diff --git a/dankserver/copyparty/quadlet/copyparty.container b/dankserver/copyparty/quadlet/copyparty.container index 76a31b0..08386d5 100644 --- a/dankserver/copyparty/quadlet/copyparty.container +++ b/dankserver/copyparty/quadlet/copyparty.container @@ -36,6 +36,9 @@ Volume=/home/dank/Containers/copyparty/cfg:/cfg PublishPort=3923:3923 [Service] +# /w is the array: serving the pre-mount directory instead of the volume shows an +# empty share and nothing complains, so the mount has to answer first. +ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /mnt/entropy EnvironmentFile=/home/dank/Containers/copyparty/env Restart=always RestartSec=5 diff --git a/dankserver/jellyfin/quadlet/jellyfin.container b/dankserver/jellyfin/quadlet/jellyfin.container index 4152131..65f45b9 100644 --- a/dankserver/jellyfin/quadlet/jellyfin.container +++ b/dankserver/jellyfin/quadlet/jellyfin.container @@ -43,11 +43,10 @@ PublishPort=1900:1900/udp Volume=/home/dank/Containers/jellyfin/config:/config Volume=/home/dank/Containers/jellyfin/cache:/cache # The view, writable: Jellyfin is the metadata writer (movie.nfo / tvshow.nfo / -# episode.nfo + artwork) and those bytes belong on the SSD. The condition is the -# fail-closed guard — starting against a plain, unmounted view directory would let -# a scan conclude the library is empty. +# episode.nfo + artwork) and those bytes belong on the SSD. The gate in [Service] +# is the guard: started against a plain, unmounted view directory a scan decides +# the library is empty and deletes what it cannot see. Volume=/var/lib/media-meta/view:/data -UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view AddDevice=/dev/dri GroupAdd=keep-groups HealthCmd=curl -f http://localhost:8096/health @@ -56,6 +55,9 @@ HealthTimeout=10s HealthRetries=3 [Service] +# Refuse to start until the bind source is a real mount — see the header of +# scripts/wait-for-mount.sh for the blind-boot incident this prevents. +ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /var/lib/media-meta/view /mnt/entropy Restart=always RestartSec=5 diff --git a/dankserver/plex/quadlet/plex.container b/dankserver/plex/quadlet/plex.container index ca6354e..8e93cfa 100644 --- a/dankserver/plex/quadlet/plex.container +++ b/dankserver/plex/quadlet/plex.container @@ -16,10 +16,13 @@ Volume=/var/lib/plexmediaserver:/config # to the overlay view re-points Plex at the sidecars without touching its DB. # Read-only: Plex consumes the NFO files Jellyfin writes and writes no metadata. Volume=/var/lib/media-meta/view:/mnt/entropy:ro -UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view PublishPort=32400:32400 [Service] +# Refuse to start until the bind source is a real mount: Plex blind is how paths +# get marked missing in its own database. Both the view and the array under it +# must answer — scripts/wait-for-mount.sh names the 2026-10-31 incident. +ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /var/lib/media-meta/view /mnt/entropy Restart=always RestartSec=5 diff --git a/scripts/wait-for-mount.sh b/scripts/wait-for-mount.sh new file mode 100755 index 0000000..fc77343 --- /dev/null +++ b/scripts/wait-for-mount.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# wait-for-mount.sh — refuse a container start until what it binds is mounted. +# +# Why this exists (live 2026-10-31, Dankserver): a rootless quadlet cannot order +# itself against a SYSTEM mount unit. `RequiresMountsFor=/mnt/entropy` in a user +# unit resolves to nothing and is inert, so when the array's mount finished a +# second after podman resolved the bind, Jellyfin and Plex started against the +# pre-mount directory. Their library root looked EMPTY and Jellyfin began +# deleting metadata by the thousand (`Removing item, Type: "Season"`). A blind +# bind is not a cosmetic fault; it is a metadata-deleting machine. +# +# A `ConditionPathIsMountPoint=` is NOT the fix, measured on the same host: an +# unmet condition logs "skipped, unmet condition check", NRestarts=0, and no +# start attempt ever follows — the unit stays dead until a human starts it. +# That is fail-dead. This script WAITS (the mount usually lands within seconds) +# and only then refuses, so `Restart=always` carries the unit across the race. +# +# usage: wait-for-mount.sh [-t SECONDS] MOUNTPOINT [MOUNTPOINT...] +# exit 0 every path is a mountpoint +# exit 1 a path is still not a mountpoint after the window (named on stderr) +# exit 2 usage error +set -euo pipefail + +usage() { printf 'usage: %s [-t SECONDS] MOUNTPOINT [MOUNTPOINT...]\n' "$0"; } + +window=90 +while getopts ":t:h" opt; do + case "$opt" in + t) window=$OPTARG ;; + h) usage; exit 0 ;; + *) usage >&2; exit 2 ;; + esac +done +shift $((OPTIND - 1)) + +[ "$#" -ge 1 ] || { usage >&2; exit 2; } +case "$window" in + '' | *[!0-9]*) printf 'refused: -t wants whole seconds, got %s\n' "$window" >&2; exit 2 ;; +esac + +is_mount() { findmnt -M "$1" -rno TARGET >/dev/null 2>&1; } + +report() { + local why=$1; shift + printf '%s:' "$why" + for p in "$@"; do printf ' %s' "$p"; done + printf '\n' +} + +deadline=$(( $(date +%s) + window )) +warned=0 +while :; do + missing=() + for path in "$@"; do + is_mount "$path" || missing+=("$path") + done + [ "${#missing[@]}" -eq 0 ] && break + + if [ "$(date +%s)" -ge "$deadline" ]; then + report "refused: still not mounted after ${window}s" "${missing[@]}" >&2 + printf 'refusing to start: a container bind-mounted here would see the empty\n' >&2 + printf 'directory underneath and conclude the library is empty — which is how a\n' >&2 + printf 'scan begins deleting metadata it cannot find.\n' >&2 + exit 1 + fi + if [ "$warned" -eq 0 ]; then + report "waiting for mount(s) this unit depends on" "${missing[@]}" + warned=1 + fi + sleep 1 +done + +[ "$warned" -eq 0 ] || printf 'mounted; proceeding.\n' +exit 0 diff --git a/serverAppConfigs-docs/Dankserver/Media Metadata View.md b/serverAppConfigs-docs/Dankserver/Media Metadata View.md index 71aceb2..3a06509 100644 --- a/serverAppConfigs-docs/Dankserver/Media Metadata View.md +++ b/serverAppConfigs-docs/Dankserver/Media Metadata View.md @@ -46,10 +46,11 @@ view really is a mountpoint. | Installer | `scripts/media-view.sh` — `plan` / `install` / `verify` / `status` / `reap` | | Written by | Jellyfin only (Plex binds read-only); nothing else in the fleet touches it | -Both consumers guard the view with -`UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view`, and -`scripts/deploy.sh dankserver --service jellyfin|plex` calls `media-view.sh -install` + `verify` before it restarts either container. +Both consumers gate their start on the view with +`ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh` (it waits +for the mount, then refuses), and `scripts/deploy.sh dankserver --service +jellyfin|plex` calls `media-view.sh install` + `verify` before it restarts +either container. | Consumer | Bind | Mode | Why that path | |---|---|---|---| @@ -80,11 +81,20 @@ the defaults are the host values above. refuse via `lsblk -dno ROTA` if the upper base lives on the array: an overlay whose upper is the spinners is the original problem with extra steps, and it silently turns every metadata write back into a seek. -- **Never mount the view over an unmounted lower.** `RequiresMountsFor=/mnt/entropy` - plus the container `ConditionPathIsMountPoint` guards: an empty `lowerdir` - presents an *empty library* as truth, and a media scan would then clear the - database of everything. systemd does not create a mount point, so `install` - creates `view` itself; without it the mount simply fails. +- **A container that binds the array must refuse to start until the mount + answers.** Live 2026-10-31: the array's mount and the media containers raced at + boot, podman resolved the bind against the pre-mount directory, and both + servers came up blind — Jellyfin read an empty library and began + `Removing item, Type: "Season"` by the thousand. Two dependencies people reach + for do not work here, both measured on that host: `RequiresMountsFor=` in a + **rootless** unit is inert (the user manager cannot order against the system's + `mnt-entropy.mount`), and `ConditionPathIsMountPoint=` is fail-*dead* — an + unmet condition logs `skipped, unmet condition check` with `NRestarts=0` and no + start attempt ever follows, so the unit waits for a human. The working guard is + `scripts/wait-for-mount.sh` as `ExecStartPre`: it waits out the race, refuses by + name after its window, and `Restart=always` carries the unit from there. + systemd does not create a mount point, so `install` creates `view` itself; + without it the mount simply fails. - **Never delete through the view.** A delete writes an overlayfs whiteout (`.wh.`) in the upper layer, which hides the array's copy without removing it — the file reappears the moment the view is unmounted, and the @@ -107,7 +117,8 @@ the defaults are the host values above. | Symptom | Meaning | |---|---| -| `jellyfin.service` / `plex.service` show `ConditionPathIsMountPoint=... was not met` and stay inactive | The view is not mounted. Do not start them anyway: `scripts/media-view.sh install`, then `verify`. | +| `jellyfin.service` / `plex.service` fail with `refused: still not mounted after 90s: …` | The view or the array is not mounted. Do not start them anyway: `scripts/media-view.sh install`, then `verify`. | +| Servers answer, library looks empty, Jellyfin logs `Removing item, Type: "Season"` by the thousand | Blind bind — the container started before the mount and bound the pre-mount directory. `podman exec jellyfin ls /data` against `ls /mnt/entropy` is the whole diagnosis; restarting the container re-resolves the bind. | | Jellyfin log: `Read-only file system : '/data/Movies/…/movie.nfo'` | The container is binding the raw array, not the view — the quadlet drifted back or the view is gone. `media-view.sh status`. | | A folder appears in a media server with artwork/`.nfo` but no video | Ghost metadata from a relayout. `media-view.sh reap`. | | `verify` prints `refused: …/view is not a mountpoint` | A plain directory is sitting where the overlay belongs. That is the fail-closed guard working; `install` on the box. | diff --git a/tests/array_mount_gate.bats b/tests/array_mount_gate.bats new file mode 100755 index 0000000..c44e745 --- /dev/null +++ b/tests/array_mount_gate.bats @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Seam: a container that binds the entropy array must refuse to start until the +# thing it binds is actually mounted. +# +# Live 2026-10-31, Dankserver: the array's systemd mount STARTED eight seconds +# before the media containers were created, but a rootless quadlet cannot order +# itself against a SYSTEM mount unit — `RequiresMountsFor=/mnt/entropy` resolves +# to nothing in the user manager and is simply inert. So podman resolved the bind +# against the pre-mount directory (which held one stale subdirectory) and both +# Jellyfin and Plex came up blind: their library root looked empty, Jellyfin +# logged thousands of `Removing item, Type: "Season"`, and the array sat at 80 % +# util doing nothing useful until the host went down. A blind bind is not a +# cosmetic fault — it is a metadata-deleting machine. +# +# The measured fix is `ExecStartPre=/scripts/wait-for-mount.sh `: +# it WAITS for the mount and then refuses, so `Restart=always` carries the unit +# across a slow mount. A `ConditionPathIsMountPoint=` was measured on the same +# host and is the wrong tool: an unmet condition logs `skipped, unmet condition +# check` with NRestarts=0 and never a start attempt — the unit stays dead until +# a human starts it. That is fail-dead, not fail-closed. +# +# Tests assert only observable behavior of the gate (exit status and the name it +# refuses by) and the structural law over the quadlets; never the internals. + +load helpers + +GATE="$REPO_ROOT/scripts/wait-for-mount.sh" + +@test "the gate accepts a path that really is a mountpoint" { + run "$GATE" -t 2 /proc + [ "$status" -eq 0 ] || { printf '%s\n' "$output" >&2; return 1; } +} + +@test "a plain directory is refused, and refused BY NAME" { + dir="$BATS_TEST_TMPDIR/not-a-mount" + mkdir "$dir" + run "$GATE" -t 1 "$dir" + [ "$status" -ne 0 ] || { printf 'accepted a plain directory\n' >&2; return 1; } + [[ "$output" == *"$dir"* ]] || { printf 'refusal did not name the path: %s\n' "$output" >&2; return 1; } +} + +@test "a path that does not exist is refused by name" { + run "$GATE" -t 1 "$BATS_TEST_TMPDIR/does/not/exist" + [ "$status" -ne 0 ] + [[ "$output" == *"$BATS_TEST_TMPDIR/does/not/exist"* ]] +} + +@test "the gate holds the deadline instead of deciding on one look" { + # A single check that failed at the wrong instant is the boot race itself: + # the mount lands a second later and the unit is dead anyway. So the window + # must be spent waiting, and that is observable from outside as elapsed time. + dir="$BATS_TEST_TMPDIR/never" + mkdir "$dir" + start=$SECONDS + run "$GATE" -t 4 "$dir" + elapsed=$((SECONDS - start)) + [ "$status" -ne 0 ] + [ "$elapsed" -ge 4 ] || { printf 'gave up after %ss, window was 4s\n' "$elapsed" >&2; return 1; } +} + +@test "the gate accepts a mount that arrives during its window" { + dir="$BATS_TEST_TMPDIR/later" + mkdir "$dir" + if ! mount --bind /proc "$dir" 2>/dev/null; then + skip "creating a mount needs privileges this box does not grant" + fi + run "$GATE" -t 4 "$dir" + umount "$dir" 2>/dev/null || sudo -n umount "$dir" 2>/dev/null || true + [ "$status" -eq 0 ] || { printf 'gate did not wait for a mount that arrived: %s\n' "$output" >&2; return 1; } +} + +@test "no unit without the gate binds the array or the metadata view" { + bad="" + while IFS= read -r unit; do + sources=$(sed -n 's/^Volume=\([^:]*\):.*/\1/p' "$unit" \ + | grep -E '^(/mnt/entropy|/var/lib/media-meta/view)$' | sort -u) + [ -n "$sources" ] || continue + for src in $sources; do + grep -q "ExecStartPre=.*wait-for-mount.sh.* $src\b" "$unit" \ + || bad="$bad$unit binds $src with no start gate"$'\n' + done + done < <(find "$REPO_ROOT"/dankserver "$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro \ + -path '*/quadlet/*.container' -not -path '*.AGENTTRASH*') + [ -z "$bad" ] || { printf 'blind binds:\n%s' "$bad" >&2; return 1; } +} + +@test "no gated unit keeps the fail-dead ConditionPathIsMountPoint" { + bad="" + while IFS= read -r unit; do + grep -q "wait-for-mount.sh" "$unit" || continue + grep -q "ConditionPathIsMountPoint" "$unit" \ + && bad="$bad$unit: a gate and an unmet-condition skip cannot coexist"$'\n' + done < <(find "$REPO_ROOT"/dankserver "$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro \ + -path '*/quadlet/*.container' -not -path '*.AGENTTRASH*') + [ -z "$bad" ] || { printf 'fail-dead conditions:\n%s' "$bad" >&2; return 1; } +} diff --git a/tests/media_overlay.bats b/tests/media_overlay.bats index ee6a3e1..1bf526f 100755 --- a/tests/media_overlay.bats +++ b/tests/media_overlay.bats @@ -61,7 +61,10 @@ UNIT="$REPO_ROOT/dankserver/host/systemd/media-meta-view.mount" @test "both media servers bind the view and refuse to start without it" { for unit in "$JELLY" "$PLEX"; do grep -q "^Volume=$MOUNT:" "$unit" || return 1 - grep -q "^UnitConfigItem=ConditionPathIsMountPoint=$MOUNT$" "$unit" || return 1 + # The refusal is a start GATE, not a Condition: measured on the host, a + # unit whose Condition goes unmet is skipped forever (NRestarts=0, no + # start attempt) — fail-dead. The gate waits, then refuses. + grep -q "^ExecStartPre=.*wait-for-mount.sh .*$MOUNT" "$unit" || return 1 # The raw array must no longer be a container source: that bind is what # made sidecar sharing impossible (read-only) and metadata slow (seeks). ! grep -q '^Volume=/mnt/entropy' "$unit"