scripts/media-verify.py

@@ -9,8 +9,13 @@ its destination, and NO row may still name its source — a source row is a scan
 that has not caught up, and it is the one signature of a library pointing at
 files that no longer exist.
 
-Two doors, both read-only, both without a lock while the server runs
-(`mode=ro&immutable=1`; there is no `sqlite3` binary on these hosts):
+Two doors, both read-only, both without a lock while the server runs (there is
+no `sqlite3` binary on these hosts). Open `mode=ro` PLAIN: both servers keep
+their libraries in WAL mode, and `immutable=1` — which claims the file never
+changes — makes sqlite skip the write-ahead log entirely, so every row the
+server has committed but not yet checkpointed reads as MISSING, and a read
+beside a running scanner can even fail as "database disk image is malformed".
+`immutable=1` is only the fallback, for a database whose `-shm` cannot be read.
 
   jellyfin  `BaseItems.Path`. The container binds the share at `/data`, so the
             database holds `/data/<library>/…` where the journal holds
@@ -22,9 +27,14 @@ Two doors, both read-only, both without a lock while the server runs
 
 Neither server holds a row for everything the journal renames: a `.log`, `.cue`
 or cover image has no row to land in, so its absence is not a fault. The report
-therefore breaks the missing rows down by extension, and `--ignore-ext` names
-the extensions this server does not index, which are then counted separately and
-kept out of the verdict.
+therefore breaks the missing rows down by extension, and `--ignore-ext` names the
+extensions this server does not index, which are then counted separately and kept
+out of the verdict. What is left is split by what the filesystem says, because
+the database alone cannot: a file still at its `src` is `REAL` (this pass's own
+damage), a file at neither path is `gone` (a later pass or the server renamed it),
+and a file sitting at its `dst` with no row naming it is simply something the
+server does not index. A row that still names `src` is none of these: it is the
+server's own stale pointer — a phantom — and it costs a re-scan, never a restore.
 
 Exit codes: 0 every move landed; 1 something is stale or missing (the details are
 printed); 2 the journal or the database is not usable at all.
@@ -92,10 +102,18 @@ def read_moves(path):
 def server_paths(db, kind):
     """Every path the server holds, or a refusal that names what is missing."""
     spec = KINDS[kind]
+    con = None
+    err = None
+    # Plain ro reads the WAL the server commits through; immutable is the last
+    # resort, because it cannot see a row the server has not checkpointed.
+    for opt in ("mode=ro", "mode=ro&immutable=1"):
         try:
-        con = sqlite3.connect(f"file:{db}?mode=ro&immutable=1", uri=True)
+            con = sqlite3.connect(f"file:{db}?{opt}", uri=True)
+            break
         except sqlite3.Error as e:
-        die(f"cannot open {db}: {e}")
+            err = e
+    if con is None:
+        die(f"cannot open {db}: {err}")
     with con:
         have = {r[1] for r in con.execute(f"pragma table_info({spec['table']})")}
         if not spec["columns"] <= have:
@@ -161,20 +179,37 @@ def main():
         if at_src:
             stale.append(src)
         if not at_dst:
-            missing.append(dst)
+            missing.append((src, dst))
+
+    # The database cannot tell a lost file from one it never indexes: both read
+    # as "no row at the destination". Stat both paths, so the report separates
+    # the classes the runbook names — `REAL` is the only one that is this pass's
+    # own damage, and `gone` is a later pass or the server's own rename.
+    on_disk, real, gone = [], [], []
+    for src, dst in missing:
+        if os.path.exists(dst):
+            on_disk.append(dst)
+        elif os.path.exists(src):
+            real.append(src)
+        else:
+            gone.append(dst)
 
     print(f"{args.db_kind}: {len(moves)} moves checked")
     print(f"landed: {landed}")
     print(f"still at src: {len(stale)}")
     print(f"missing from the DB: {len(missing)}")
+    print(f"  on disk at dst, no row names it: {len(on_disk)}")
+    print(f"  REAL — file still at src: {len(real)}")
+    print(f"  gone — at neither path: {len(gone)}")
     if ignored:
         print(f"not indexed (extension ignored): {ignored}")
     if missing:
         exts = Counter(os.path.splitext(d)[1].lower() or "(no extension)"
-                       for d in missing)
+                       for _, d in missing)
         print("missing by extension: "
               + ", ".join(f"{e} {n}" for e, n in sorted(exts.items())))
-    for label, items in (("stale", stale), ("missing", missing)):
+    for label, items in (("real", real), ("gone", gone), ("missing", on_disk),
+                         ("stale", stale)):
         for path in items[:args.show]:
             print(f"  {label}: {path}")
         if len(items) > args.show:

serverAppConfigs-docs/Dankserver/Plex.md

@@ -27,7 +27,7 @@ public traffic can be fronted by Caddy like every other vhost.
 | Runtime | podman quadlet, `plex.service` (user manager) — migrated from the native `plexmediaserver.service` OS-package unit (2026-09) |
 | Image | `lscr.io/linuxserver/plex:1.43.5.11029-6bf8af3f0-ls329` (pinned; verified == the registry's `latest` manifest digest at pin time: `sha256:8f655bad…e9e2334`) |
 | Data root | `/var/lib/plexmediaserver` bind-mounted **in place** as `/config` (RW) — the migration moved zero bytes |
-| Media | git: `/var/lib/media-meta/view` (the [[Media Metadata View]] overlay) bound at the same absolute path `/mnt/entropy` **read-only**; live today: `/mnt/entropy` itself, read-only — the view has never been installed. Either way every library path in the PMS database stays valid, zero re-scan |
+| Media | `/var/lib/media-meta/view` (the [[Media Metadata View]] overlay) bound at the same absolute path `/mnt/entropy` **read-only** — live since the 2026-10-31 cutover, so the PMS database's recorded paths keep resolving with zero re-scan |
 | Identity | fleet rootless model — container root maps to the `dank` user (1000:1000); the deployer chowns the data root on first cutover |
 | Port | `32400/tcp` published on `0.0.0.0` — zero client reconfiguration |
 | Network | `dankserver_appnet`, alias `plex`; Tautulli reaches it at `http://plex:32400` |
@@ -40,7 +40,7 @@ public traffic can be fronted by Caddy like every other vhost.
 The migration moved zero bytes: the container bind-mounts the native data
 root where it already lived, and the media archive re-mounts at
 `/mnt/entropy` inside the container — only the HOST side of that bind has moved
-(the array today; the overlay view in git has never been deployed), so every
+(the array until the 2026-10-31 view cutover, the overlay since), so every
 absolute library path in the
 PMS database keeps resolving. The one metadata change: rootless podman
 cannot map the real host uid 997, so the fleet model applies — container
@@ -87,6 +87,16 @@ root runs as the `dank` user, and the deployer adopts the data root with
    while the HTTP layer finishes loading; the first boot of a years-old
    library took ~10 minutes (2026-09 cutover). The deployer gate now allows
    600 s before failing; later boots are much faster (warm page cache).
+5. **`32400` answers TLS only to remote clients.** PMS requires secure
+   connections, so `curl http://192.168.0.235:32400/identity` from any other
+   machine closes without a single byte (curl exit 52) — that reads as a dead
+   server beside a healthy one, and it is how a post-cutover health check gets
+   misread. Probe with `curl -k https://192.168.0.235:32400/identity`. Two
+   consequences: `plex.franksdank.cloud` works **only** because its `transport
+   http` block carries `tls_insecure_skip_verify` (that option makes Caddy speak
+   TLS to PMS's self-signed cert — delete it and the vhost 502s), and the scan
+   block below rides `http://127.0.0.1` **inside** the container, where insecure
+   connections are allowed.
 
 ## Triggering a scan
 

tests/media_verify.bats

@@ -195,7 +195,7 @@ setup() {
         --share "$SHARE" --container /data
     [ "$status" -eq 1 ]
     [ "${lines[3]}" = "missing from the DB: 2" ]
-    [ "${lines[4]}" = "missing by extension: .log 1, .mkv 1" ]
+    [[ "$output" == *"missing by extension: .log 1, .mkv 1"* ]]
 }
 
 @test "--ignore-ext keeps a file the server never indexes out of the verdict" {
@@ -208,5 +208,75 @@ setup() {
     [ "$status" -eq 0 ]
     [ "${lines[1]}" = "landed: 1" ]
     [ "${lines[3]}" = "missing from the DB: 0" ]
-    [ "${lines[4]}" = "not indexed (extension ignored): 1" ]
+    [[ "$output" == *"not indexed (extension ignored): 1"* ]]
+}
+
+# wal_case <db> — a Jellyfin database whose NEW row lives only in the
+# write-ahead log: the server has committed the rename but not checkpointed it,
+# which is the normal state of a library a scan is writing. The holder
+# connection keeps the WAL from folding into the main file, and the verifier
+# runs as a real subprocess so it faces the same files Plex and Jellyfin leave.
+wal_case() {
+    python3 - "$JFDB" "$JOURNAL" "$SHARE" "$VERIFY" <<'PY'
+import sqlite3, subprocess, sys
+db, journal, share, verify = sys.argv[1:5]
+w = sqlite3.connect(db)
+w.execute("pragma journal_mode=wal")
+w.execute("pragma wal_autocheckpoint=0")
+w.execute("insert into BaseItems (Path, Type) values ('/data/Movies/Old.mkv', 'x')")
+w.commit()
+w.execute("pragma wal_checkpoint(truncate)")
+holder = sqlite3.connect(db)  # last-connection close would checkpoint
+w.execute("update BaseItems set Path='/data/Movies/Old/Old.mkv'")
+w.commit()
+r = subprocess.run([sys.executable, verify, "--journal", journal,
+                    "--db-kind", "jellyfin", "--db", db,
+                    "--share", share, "--container", "/data"],
+                   capture_output=True, text=True)
+print(r.stdout, end="")
+sys.exit(r.returncode)
+PY
+}
+
+@test "a row the server committed but has not checkpointed is proof, not a miss" {
+    move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
+    run wal_case
+    [ "$status" -eq 0 ]
+    [ "${lines[1]}" = "landed: 1" ]
+    [ "${lines[3]}" = "missing from the DB: 0" ]
+}
+
+# The DB alone cannot tell a lost file from a sidecar: both are "no row at the
+# destination". The tool therefore stats the two paths and separates the classes
+# the runbook names, so an operator never reads a `.pdf` as a lost film or,
+# worse, reads a real loss as a sidecar.
+@test "a file the database does not name that still sits at src is the pass's own damage" {
+    mkdir -p "$SHARE/Movies"
+    echo film > "$SHARE/Movies/Old.mkv"
+    move_row "$SHARE/Movies/Old.mkv" "$SHARE/Movies/Old/Old.mkv"
+
+    run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
+        --share "$SHARE" --container /data
+    [ "$status" -eq 1 ]
+    [ "${lines[3]}" = "missing from the DB: 1" ]
+    [ "${lines[4]}" = "  on disk at dst, no row names it: 0" ]
+    [ "${lines[5]}" = "  REAL — file still at src: 1" ]
+    [ "${lines[6]}" = "  gone — at neither path: 0" ]
+    [[ "$output" == *"real: $SHARE/Movies/Old.mkv"* ]]
+}
+
+@test "a file at neither path is gone, and a file at dst the server ignores is not a loss" {
+    move_row "$SHARE/Movies/Gone.mkv" "$SHARE/Movies/Gone/Gone.mkv"
+    mkdir -p "$SHARE/Movies/Sidecar"
+    echo cover > "$SHARE/Movies/Sidecar/cover.jpg"
+    move_row "$SHARE/Movies/Album/cover.jpg" "$SHARE/Movies/Sidecar/cover.jpg"
+
+    run python3 "$VERIFY" --journal "$JOURNAL" --db-kind jellyfin --db "$JFDB" \
+        --share "$SHARE" --container /data
+    [ "$status" -eq 1 ]
+    [ "${lines[3]}" = "missing from the DB: 2" ]
+    [ "${lines[4]}" = "  on disk at dst, no row names it: 1" ]
+    [ "${lines[5]}" = "  REAL — file still at src: 0" ]
+    [ "${lines[6]}" = "  gone — at neither path: 1" ]
+    [[ "$output" == *"gone: $SHARE/Movies/Gone/Gone.mkv"* ]]
 }