Merge pull request 'fix(fleet): refuse a container start until the mount it binds has answered' (#109) from fix/array-mount-gate into main

AGENTS.md

@@ -156,6 +156,15 @@ Unit requirements:
   (`<host>/<svc>/quadlet/`), is installed by the deployer, and is covered by
   the quadlet test seam. The host-side unit is generated state; the repo file
   is the source of truth.
+- **Bind-source gate** — a unit whose `Volume=` source is a mount this host owns
+  (the entropy array, the metadata view) carries
+  `ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh <source>`.
+  Measured 2026-10-31: in a **rootless** unit `RequiresMountsFor=` is inert (the
+  user manager cannot order against the system's mount unit), and
+  `ConditionPathIsMountPoint=` is fail-*dead* — an unmet condition logs `skipped,
+  unmet condition check` with `NRestarts=0` and no start attempt ever follows.
+  Bind the array before it is mounted and the container sees the empty pre-mount
+  directory as truth; a media scan then deletes the library it cannot see.
 
 Before declaring a service deployed: `systemctl --user is-enabled <name>` →
 `enabled`, `is-active` → `active`, and the container answers its health check.

dankserver/copyparty/quadlet/copyparty.container

@@ -36,6 +36,9 @@ Volume=/home/dank/Containers/copyparty/cfg:/cfg
 PublishPort=3923:3923
 
 [Service]
+# /w is the array: serving the pre-mount directory instead of the volume shows an
+# empty share and nothing complains, so the mount has to answer first.
+ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /mnt/entropy
 EnvironmentFile=/home/dank/Containers/copyparty/env
 Restart=always
 RestartSec=5

dankserver/jellyfin/quadlet/jellyfin.container

@@ -43,11 +43,10 @@ PublishPort=1900:1900/udp
 Volume=/home/dank/Containers/jellyfin/config:/config
 Volume=/home/dank/Containers/jellyfin/cache:/cache
 # The view, writable: Jellyfin is the metadata writer (movie.nfo / tvshow.nfo /
-# episode.nfo + artwork) and those bytes belong on the SSD. The condition is the
-# fail-closed guard — starting against a plain, unmounted view directory would let
-# a scan conclude the library is empty.
+# episode.nfo + artwork) and those bytes belong on the SSD. The gate in [Service]
+# is the guard: started against a plain, unmounted view directory a scan decides
+# the library is empty and deletes what it cannot see.
 Volume=/var/lib/media-meta/view:/data
-UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view
 AddDevice=/dev/dri
 GroupAdd=keep-groups
 HealthCmd=curl -f http://localhost:8096/health
@@ -56,6 +55,9 @@ HealthTimeout=10s
 HealthRetries=3
 
 [Service]
+# Refuse to start until the bind source is a real mount — see the header of
+# scripts/wait-for-mount.sh for the blind-boot incident this prevents.
+ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /var/lib/media-meta/view /mnt/entropy
 Restart=always
 RestartSec=5
 

dankserver/plex/quadlet/plex.container

@@ -16,10 +16,13 @@ Volume=/var/lib/plexmediaserver:/config
 # to the overlay view re-points Plex at the sidecars without touching its DB.
 # Read-only: Plex consumes the NFO files Jellyfin writes and writes no metadata.
 Volume=/var/lib/media-meta/view:/mnt/entropy:ro
-UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view
 PublishPort=32400:32400
 
 [Service]
+# Refuse to start until the bind source is a real mount: Plex blind is how paths
+# get marked missing in its own database. Both the view and the array under it
+# must answer — scripts/wait-for-mount.sh names the 2026-10-31 incident.
+ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /var/lib/media-meta/view /mnt/entropy
 Restart=always
 RestartSec=5
 

scripts/wait-for-mount.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# wait-for-mount.sh — refuse a container start until what it binds is mounted.
+#
+# Why this exists (live 2026-10-31, Dankserver): a rootless quadlet cannot order
+# itself against a SYSTEM mount unit. `RequiresMountsFor=/mnt/entropy` in a user
+# unit resolves to nothing and is inert, so when the array's mount finished a
+# second after podman resolved the bind, Jellyfin and Plex started against the
+# pre-mount directory. Their library root looked EMPTY and Jellyfin began
+# deleting metadata by the thousand (`Removing item, Type: "Season"`). A blind
+# bind is not a cosmetic fault; it is a metadata-deleting machine.
+#
+# A `ConditionPathIsMountPoint=` is NOT the fix, measured on the same host: an
+# unmet condition logs "skipped, unmet condition check", NRestarts=0, and no
+# start attempt ever follows — the unit stays dead until a human starts it.
+# That is fail-dead. This script WAITS (the mount usually lands within seconds)
+# and only then refuses, so `Restart=always` carries the unit across the race.
+#
+# usage: wait-for-mount.sh [-t SECONDS] MOUNTPOINT [MOUNTPOINT...]
+#   exit 0  every path is a mountpoint
+#   exit 1  a path is still not a mountpoint after the window (named on stderr)
+#   exit 2  usage error
+set -euo pipefail
+
+usage() { printf 'usage: %s [-t SECONDS] MOUNTPOINT [MOUNTPOINT...]\n' "$0"; }
+
+window=90
+while getopts ":t:h" opt; do
+    case "$opt" in
+        t) window=$OPTARG ;;
+        h) usage; exit 0 ;;
+        *) usage >&2; exit 2 ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+[ "$#" -ge 1 ] || { usage >&2; exit 2; }
+case "$window" in
+    '' | *[!0-9]*) printf 'refused: -t wants whole seconds, got %s\n' "$window" >&2; exit 2 ;;
+esac
+
+is_mount() { findmnt -M "$1" -rno TARGET >/dev/null 2>&1; }
+
+report() {
+    local why=$1; shift
+    printf '%s:' "$why"
+    for p in "$@"; do printf ' %s' "$p"; done
+    printf '\n'
+}
+
+deadline=$(( $(date +%s) + window ))
+warned=0
+while :; do
+    missing=()
+    for path in "$@"; do
+        is_mount "$path" || missing+=("$path")
+    done
+    [ "${#missing[@]}" -eq 0 ] && break
+
+    if [ "$(date +%s)" -ge "$deadline" ]; then
+        report "refused: still not mounted after ${window}s" "${missing[@]}" >&2
+        printf 'refusing to start: a container bind-mounted here would see the empty\n' >&2
+        printf 'directory underneath and conclude the library is empty — which is how a\n' >&2
+        printf 'scan begins deleting metadata it cannot find.\n' >&2
+        exit 1
+    fi
+    if [ "$warned" -eq 0 ]; then
+        report "waiting for mount(s) this unit depends on" "${missing[@]}"
+        warned=1
+    fi
+    sleep 1
+done
+
+[ "$warned" -eq 0 ] || printf 'mounted; proceeding.\n'
+exit 0

serverAppConfigs-docs/Dankserver/Media Metadata View.md

@@ -46,10 +46,11 @@ view really is a mountpoint.
 | Installer | `scripts/media-view.sh` — `plan` / `install` / `verify` / `status` / `reap` |
 | Written by | Jellyfin only (Plex binds read-only); nothing else in the fleet touches it |
 
-Both consumers guard the view with
-`UnitConfigItem=ConditionPathIsMountPoint=/var/lib/media-meta/view`, and
-`scripts/deploy.sh dankserver --service jellyfin|plex` calls `media-view.sh
-install` + `verify` before it restarts either container.
+Both consumers gate their start on the view with
+`ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh` (it waits
+for the mount, then refuses), and `scripts/deploy.sh dankserver --service
+jellyfin|plex` calls `media-view.sh install` + `verify` before it restarts
+either container.
 
 | Consumer | Bind | Mode | Why that path |
 |---|---|---|---|
@@ -80,11 +81,20 @@ the defaults are the host values above.
   refuse via `lsblk -dno ROTA` if the upper base lives on the array: an overlay
   whose upper is the spinners is the original problem with extra steps, and it
   silently turns every metadata write back into a seek.
-- **Never mount the view over an unmounted lower.** `RequiresMountsFor=/mnt/entropy`
-  plus the container `ConditionPathIsMountPoint` guards: an empty `lowerdir`
-  presents an *empty library* as truth, and a media scan would then clear the
-  database of everything. systemd does not create a mount point, so `install`
-  creates `view` itself; without it the mount simply fails.
+- **A container that binds the array must refuse to start until the mount
+  answers.** Live 2026-10-31: the array's mount and the media containers raced at
+  boot, podman resolved the bind against the pre-mount directory, and both
+  servers came up blind — Jellyfin read an empty library and began
+  `Removing item, Type: "Season"` by the thousand. Two dependencies people reach
+  for do not work here, both measured on that host: `RequiresMountsFor=` in a
+  **rootless** unit is inert (the user manager cannot order against the system's
+  `mnt-entropy.mount`), and `ConditionPathIsMountPoint=` is fail-*dead* — an
+  unmet condition logs `skipped, unmet condition check` with `NRestarts=0` and no
+  start attempt ever follows, so the unit waits for a human. The working guard is
+  `scripts/wait-for-mount.sh` as `ExecStartPre`: it waits out the race, refuses by
+  name after its window, and `Restart=always` carries the unit from there.
+  systemd does not create a mount point, so `install` creates `view` itself;
+  without it the mount simply fails.
 - **Never delete through the view.** A delete writes an overlayfs whiteout
   (`.wh.<name>`) in the upper layer, which hides the array's copy without
   removing it — the file reappears the moment the view is unmounted, and the
@@ -107,7 +117,8 @@ the defaults are the host values above.
 
 | Symptom | Meaning |
 |---|---|
-| `jellyfin.service` / `plex.service` show `ConditionPathIsMountPoint=... was not met` and stay inactive | The view is not mounted. Do not start them anyway: `scripts/media-view.sh install`, then `verify`. |
+| `jellyfin.service` / `plex.service` fail with `refused: still not mounted after 90s: …` | The view or the array is not mounted. Do not start them anyway: `scripts/media-view.sh install`, then `verify`. |
+| Servers answer, library looks empty, Jellyfin logs `Removing item, Type: "Season"` by the thousand | Blind bind — the container started before the mount and bound the pre-mount directory. `podman exec jellyfin ls /data` against `ls /mnt/entropy` is the whole diagnosis; restarting the container re-resolves the bind. |
 | Jellyfin log: `Read-only file system : '/data/Movies/…/movie.nfo'` | The container is binding the raw array, not the view — the quadlet drifted back or the view is gone. `media-view.sh status`. |
 | A folder appears in a media server with artwork/`.nfo` but no video | Ghost metadata from a relayout. `media-view.sh reap`. |
 | `verify` prints `refused: …/view is not a mountpoint` | A plain directory is sitting where the overlay belongs. That is the fail-closed guard working; `install` on the box. |

tests/array_mount_gate.bats

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# Seam: a container that binds the entropy array must refuse to start until the
+# thing it binds is actually mounted.
+#
+# Live 2026-10-31, Dankserver: the array's systemd mount STARTED eight seconds
+# before the media containers were created, but a rootless quadlet cannot order
+# itself against a SYSTEM mount unit — `RequiresMountsFor=/mnt/entropy` resolves
+# to nothing in the user manager and is simply inert. So podman resolved the bind
+# against the pre-mount directory (which held one stale subdirectory) and both
+# Jellyfin and Plex came up blind: their library root looked empty, Jellyfin
+# logged thousands of `Removing item, Type: "Season"`, and the array sat at 80 %
+# util doing nothing useful until the host went down. A blind bind is not a
+# cosmetic fault — it is a metadata-deleting machine.
+#
+# The measured fix is `ExecStartPre=<repo>/scripts/wait-for-mount.sh <source>`:
+# it WAITS for the mount and then refuses, so `Restart=always` carries the unit
+# across a slow mount. A `ConditionPathIsMountPoint=` was measured on the same
+# host and is the wrong tool: an unmet condition logs `skipped, unmet condition
+# check` with NRestarts=0 and never a start attempt — the unit stays dead until
+# a human starts it. That is fail-dead, not fail-closed.
+#
+# Tests assert only observable behavior of the gate (exit status and the name it
+# refuses by) and the structural law over the quadlets; never the internals.
+
+load helpers
+
+GATE="$REPO_ROOT/scripts/wait-for-mount.sh"
+
+@test "the gate accepts a path that really is a mountpoint" {
+    run "$GATE" -t 2 /proc
+    [ "$status" -eq 0 ] || { printf '%s\n' "$output" >&2; return 1; }
+}
+
+@test "a plain directory is refused, and refused BY NAME" {
+    dir="$BATS_TEST_TMPDIR/not-a-mount"
+    mkdir "$dir"
+    run "$GATE" -t 1 "$dir"
+    [ "$status" -ne 0 ] || { printf 'accepted a plain directory\n' >&2; return 1; }
+    [[ "$output" == *"$dir"* ]] || { printf 'refusal did not name the path: %s\n' "$output" >&2; return 1; }
+}
+
+@test "a path that does not exist is refused by name" {
+    run "$GATE" -t 1 "$BATS_TEST_TMPDIR/does/not/exist"
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"$BATS_TEST_TMPDIR/does/not/exist"* ]]
+}
+
+@test "the gate holds the deadline instead of deciding on one look" {
+    # A single check that failed at the wrong instant is the boot race itself:
+    # the mount lands a second later and the unit is dead anyway. So the window
+    # must be spent waiting, and that is observable from outside as elapsed time.
+    dir="$BATS_TEST_TMPDIR/never"
+    mkdir "$dir"
+    start=$SECONDS
+    run "$GATE" -t 4 "$dir"
+    elapsed=$((SECONDS - start))
+    [ "$status" -ne 0 ]
+    [ "$elapsed" -ge 4 ] || { printf 'gave up after %ss, window was 4s\n' "$elapsed" >&2; return 1; }
+}
+
+@test "the gate accepts a mount that arrives during its window" {
+    dir="$BATS_TEST_TMPDIR/later"
+    mkdir "$dir"
+    if ! mount --bind /proc "$dir" 2>/dev/null; then
+        skip "creating a mount needs privileges this box does not grant"
+    fi
+    run "$GATE" -t 4 "$dir"
+    umount "$dir" 2>/dev/null || sudo -n umount "$dir" 2>/dev/null || true
+    [ "$status" -eq 0 ] || { printf 'gate did not wait for a mount that arrived: %s\n' "$output" >&2; return 1; }
+}
+
+@test "no unit without the gate binds the array or the metadata view" {
+    bad=""
+    while IFS= read -r unit; do
+        sources=$(sed -n 's/^Volume=\([^:]*\):.*/\1/p' "$unit" \
+            | grep -E '^(/mnt/entropy|/var/lib/media-meta/view)$' | sort -u)
+        [ -n "$sources" ] || continue
+        for src in $sources; do
+            grep -q "ExecStartPre=.*wait-for-mount.sh.* $src\b" "$unit" \
+                || bad="$bad$unit binds $src with no start gate"$'\n'
+        done
+    done < <(find "$REPO_ROOT"/dankserver "$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro \
+                 -path '*/quadlet/*.container' -not -path '*.AGENTTRASH*')
+    [ -z "$bad" ] || { printf 'blind binds:\n%s' "$bad" >&2; return 1; }
+}
+
+@test "no gated unit keeps the fail-dead ConditionPathIsMountPoint" {
+    bad=""
+    while IFS= read -r unit; do
+        grep -q "wait-for-mount.sh" "$unit" || continue
+        grep -q "ConditionPathIsMountPoint" "$unit" \
+            && bad="$bad$unit: a gate and an unmet-condition skip cannot coexist"$'\n'
+    done < <(find "$REPO_ROOT"/dankserver "$REPO_ROOT"/dankclaw "$REPO_ROOT"/frankbook-pro \
+                 -path '*/quadlet/*.container' -not -path '*.AGENTTRASH*')
+    [ -z "$bad" ] || { printf 'fail-dead conditions:\n%s' "$bad" >&2; return 1; }
+}

tests/media_overlay.bats

@@ -61,7 +61,10 @@ UNIT="$REPO_ROOT/dankserver/host/systemd/media-meta-view.mount"
 @test "both media servers bind the view and refuse to start without it" {
     for unit in "$JELLY" "$PLEX"; do
         grep -q "^Volume=$MOUNT:" "$unit" || return 1
-        grep -q "^UnitConfigItem=ConditionPathIsMountPoint=$MOUNT$" "$unit" || return 1
+        # The refusal is a start GATE, not a Condition: measured on the host, a
+        # unit whose Condition goes unmet is skipped forever (NRestarts=0, no
+        # start attempt) — fail-dead. The gate waits, then refuses.
+        grep -q "^ExecStartPre=.*wait-for-mount.sh .*$MOUNT" "$unit" || return 1
         # The raw array must no longer be a container source: that bind is what
         # made sidecar sharing impossible (read-only) and metadata slow (seeks).
         ! grep -q '^Volume=/mnt/entropy' "$unit"