Compare commits
37
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fab17129e2 | ||
|
|
f46bf60939 | ||
|
|
b3eb9c6314 | ||
|
|
a3ddb4c65a | ||
|
|
b5173033b6 | ||
|
|
53e4a777b4 | ||
|
|
52d6da4f49 | ||
|
|
d138788411 | ||
|
|
6c169fc4ad | ||
|
|
e6a85176c4 | ||
|
|
9ff7ce6a12 | ||
|
|
f25be876fb | ||
|
|
bad78079d7 | ||
|
|
fee8121d47 | ||
|
|
1b5c11fb60 | ||
|
|
cae283047f | ||
|
|
51007ac604 | ||
|
|
f7c41966b0 | ||
|
|
25114ca622 | ||
|
|
a07f492b89 | ||
|
|
9961ed62c0 | ||
|
|
70be24e9d5 | ||
|
|
70aa40a24a | ||
|
|
8f2c3bb378 | ||
|
|
146c28da9f | ||
|
|
b82655d30a | ||
|
|
0161ab2aa8 | ||
|
|
085a5d570c | ||
|
|
9ca69d4e7e | ||
|
|
64b06b9426 | ||
|
|
e519f6b554 | ||
|
|
1e0247f29e | ||
|
|
71e7e94b94 | ||
|
|
24d793eb7d | ||
|
|
95375683d5 | ||
|
|
e7d7c6eea2 | ||
|
|
e1bf1d6e80 |
No files matched your search
@@ -231,7 +231,17 @@ Every Caddy-proxied service is in exactly one of three postures: behind the
|
|||||||
auth chain (Anubis :8923 → Authentik embedded outpost), **exempt by
|
auth chain (Anubis :8923 → Authentik embedded outpost), **exempt by
|
||||||
construction** because it authenticates itself (`auth.`, git smart-HTTP, Gitea,
|
construction** because it authenticates itself (`auth.`, git smart-HTTP, Gitea,
|
||||||
OpenWebUI, Jellyfin, Plex, Audiobookshelf, the *arr stack), or internal-only —
|
OpenWebUI, Jellyfin, Plex, Audiobookshelf, the *arr stack), or internal-only —
|
||||||
no public DNS name, no vhost. New services default to internal-only until guarded.
|
served to nobody off-LAN. New services default to internal-only until guarded.
|
||||||
|
|
||||||
|
**Internal-only is a source gate, not a missing DNS record.** Caddy answers
|
||||||
|
every Host it knows, and its per-name certificates are published in Certificate
|
||||||
|
Transparency logs, so a name with no Porkbun A record is still in a stranger's
|
||||||
|
reach (`crt.sh` lists it). A household-only service therefore carries an edge
|
||||||
|
gate — `@lan remote_ip 127.0.0.1/32 ::1/128 192.168.0.0/16 10.0.0.0/8
|
||||||
|
172.16.0.0/12`, `handle @lan { reverse_proxy … }`, `handle { respond 404 }` —
|
||||||
|
which is what the *arr stack and qBittorrent carry (2026-11-04: Frank's law,
|
||||||
|
"never leaving the home boxes"). A future household-only vhost gets the same
|
||||||
|
block, not merely an absent record.
|
||||||
|
|
||||||
The guard chain is a **browser-only device**: Anubis demands a JavaScript
|
The guard chain is a **browser-only device**: Anubis demands a JavaScript
|
||||||
proof-of-work and the outpost works from a session cookie, so any non-browser
|
proof-of-work and the outpost works from a session cookie, so any non-browser
|
||||||
@@ -278,7 +288,10 @@ archive.
|
|||||||
**Swap the model. Do not instrument it.** A payload change is: edit the trio
|
**Swap the model. Do not instrument it.** A payload change is: edit the trio
|
||||||
(env / serve script / unit) → `scripts/deploy.sh <host> --service <svc>` →
|
(env / serve script / unit) → `scripts/deploy.sh <host> --service <svc>` →
|
||||||
`/health` answers → one chat turn, one tool call, and one vision probe on an
|
`/health` answers → one chat turn, one tool call, and one vision probe on an
|
||||||
endpoint that serves images. That is the entire procedure.
|
endpoint that serves images → re-key that endpoint's OpenWebUI `model` row
|
||||||
|
(`meta.description` + `params.context.length`, API door in [[OpenWebUI]]). That is
|
||||||
|
the entire procedure — no deployer step touches `webui.db`, and a stale
|
||||||
|
`context.length` makes OpenWebUI advertise a window the endpoint does not grant.
|
||||||
|
|
||||||
- **No benchmarking unless Frank orders it.** No pre-swap baseline battery,
|
- **No benchmarking unless Frank orders it.** No pre-swap baseline battery,
|
||||||
no A/B arms, no decode/prefill ladders, no acceptance tables, no numbers
|
no A/B arms, no decode/prefill ladders, no acceptance tables, no numbers
|
||||||
|
|||||||
+42
-7
@@ -303,14 +303,31 @@ jelly.franksdank.cloud {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# --- *arr Stack (exempt by construction: own auth) ---
|
# --- *arr Stack + qBittorrent: LAN-ONLY at the edge (2026-11-04) -----------
|
||||||
|
# Frank's law: these never leave the home boxes. DNS secrecy is NOT a gate —
|
||||||
|
# Caddy issues one certificate per name and every one is in public CT logs
|
||||||
|
# (crt.sh lists prowlarr/radarr/lidarr/readarr/qbittorrent.franksdank.cloud),
|
||||||
|
# so a stranger can send the Host. The gate is the source address: a LAN client
|
||||||
|
# proxies through, anything off-LAN gets 404. `remote_ip` reads the real client
|
||||||
|
# address because the router DNATs the forward (no proxy hop in between).
|
||||||
prowlarr.franksdank.cloud {
|
prowlarr.franksdank.cloud {
|
||||||
reverse_proxy 192.168.0.112:9696
|
@lan remote_ip 127.0.0.1/32 ::1/128 192.168.0.0/16 10.0.0.0/8 172.16.0.0/12
|
||||||
|
handle @lan {
|
||||||
|
reverse_proxy 192.168.0.112:9696
|
||||||
|
}
|
||||||
|
handle {
|
||||||
|
respond 404
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
radarr.franksdank.cloud {
|
radarr.franksdank.cloud {
|
||||||
reverse_proxy 192.168.0.112:7878
|
@lan remote_ip 127.0.0.1/32 ::1/128 192.168.0.0/16 10.0.0.0/8 172.16.0.0/12
|
||||||
|
handle @lan {
|
||||||
|
reverse_proxy 192.168.0.112:7878
|
||||||
|
}
|
||||||
|
handle {
|
||||||
|
respond 404
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# --- Sonarr: internal-only by construction (retired 2026-09-04) -----------
|
# --- Sonarr: internal-only by construction (retired 2026-09-04) -----------
|
||||||
@@ -321,15 +338,33 @@ radarr.franksdank.cloud {
|
|||||||
# zero-auth UI to the internet via manual DNS — retired.
|
# zero-auth UI to the internet via manual DNS — retired.
|
||||||
|
|
||||||
lidarr.franksdank.cloud {
|
lidarr.franksdank.cloud {
|
||||||
reverse_proxy 192.168.0.112:8686
|
@lan remote_ip 127.0.0.1/32 ::1/128 192.168.0.0/16 10.0.0.0/8 172.16.0.0/12
|
||||||
|
handle @lan {
|
||||||
|
reverse_proxy 192.168.0.112:8686
|
||||||
|
}
|
||||||
|
handle {
|
||||||
|
respond 404
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
readarr.franksdank.cloud {
|
readarr.franksdank.cloud {
|
||||||
reverse_proxy 192.168.0.112:8787
|
@lan remote_ip 127.0.0.1/32 ::1/128 192.168.0.0/16 10.0.0.0/8 172.16.0.0/12
|
||||||
|
handle @lan {
|
||||||
|
reverse_proxy 192.168.0.112:8787
|
||||||
|
}
|
||||||
|
handle {
|
||||||
|
respond 404
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
qbittorrent.franksdank.cloud {
|
qbittorrent.franksdank.cloud {
|
||||||
reverse_proxy 192.168.0.112:8080
|
@lan remote_ip 127.0.0.1/32 ::1/128 192.168.0.0/16 10.0.0.0/8 172.16.0.0/12
|
||||||
|
handle @lan {
|
||||||
|
reverse_proxy 192.168.0.112:8080
|
||||||
|
}
|
||||||
|
handle {
|
||||||
|
respond 404
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# --- Internal listener: auth decision layer ---------------------------------
|
# --- Internal listener: auth decision layer ---------------------------------
|
||||||
|
|||||||
Executable
+204
@@ -0,0 +1,204 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""abs-post-reorg.py — drive the Audiobookshelf side of the 2026-11-04 relayout.
|
||||||
|
|
||||||
|
Runs ON Dankserver (the ABS host). Credentials come from the machine-local
|
||||||
|
file ~/.config/secrets/audiobookshelf and never enter argv; requests go
|
||||||
|
through urllib so neither the password nor the bearer token is world-readable
|
||||||
|
in /proc. The delete arm is the DB-row arm ONLY — `hard=1` is fs.remove() of
|
||||||
|
the item folder and must never reach this file (see the Audiobookshelf vault
|
||||||
|
page; after the move the covering paths are whole books).
|
||||||
|
|
||||||
|
Subcommands
|
||||||
|
covers --list LIST.tsv copy each ABS /metadata cover onto the book
|
||||||
|
folder ("srchost<TAB>dstarray" per line);
|
||||||
|
run BEFORE the mover so the cover rides the
|
||||||
|
book's move, or AFTER with dst already final
|
||||||
|
wipe [--apply] delete every library item DB row (progress
|
||||||
|
included — the campaign accepted losing the
|
||||||
|
11 progress rows); refuses without --apply
|
||||||
|
scan [--wait] trigger the library scan; --wait polls
|
||||||
|
isFetchingLibrary then prints the item total
|
||||||
|
status item total + isMissing census
|
||||||
|
|
||||||
|
Paging law (measured twice): page on `page` (0-based) + `limit`; `offset` is
|
||||||
|
ignored outright. Count unique ids; stop on `total`.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
BASE = "http://127.0.0.1:8000/audiobookshelf"
|
||||||
|
SECRET = os.environ.get("ABS_SECRET", "/home/dank/.config/secrets/audiobookshelf")
|
||||||
|
|
||||||
|
|
||||||
|
def cred():
|
||||||
|
kv = {}
|
||||||
|
with open(SECRET) as fh:
|
||||||
|
for line in fh:
|
||||||
|
if "=" in line:
|
||||||
|
k, _, v = line.strip().partition("=")
|
||||||
|
kv[k] = v
|
||||||
|
return kv.get("ABS_ADMIN_USER", "frank"), kv.get("ABS_ADMIN_PASSWORD")
|
||||||
|
|
||||||
|
|
||||||
|
def call(method, path, body=None, token=None, base=BASE):
|
||||||
|
data = json.dumps(body).encode() if body is not None else None
|
||||||
|
req = urllib.request.Request(base + path, data=data, method=method)
|
||||||
|
req.add_header("Content-Type", "application/json")
|
||||||
|
if token:
|
||||||
|
req.add_header("Authorization", f"Bearer {token}")
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as r:
|
||||||
|
raw = r.read().decode()
|
||||||
|
try:
|
||||||
|
return r.status, json.loads(raw)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return r.status, raw.strip()
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
return e.code, e.read().decode()[:200]
|
||||||
|
|
||||||
|
|
||||||
|
def login():
|
||||||
|
user, password = cred()
|
||||||
|
if not password:
|
||||||
|
sys.exit(f"refused: {SECRET} carries no ABS_ADMIN_PASSWORD")
|
||||||
|
code, body = call("POST", "/login", {"username": user, "password": password})
|
||||||
|
if code != 200:
|
||||||
|
sys.exit(f"login failed {code}: {body}")
|
||||||
|
return body["user"]["token"] # the token is inside user, not top level
|
||||||
|
|
||||||
|
|
||||||
|
def library_id(token, name="Audiobooks"):
|
||||||
|
code, libs = call("GET", "/api/libraries", token=token)
|
||||||
|
if code != 200:
|
||||||
|
sys.exit(f"libraries failed {code}: {libs}")
|
||||||
|
for lib in libs.get("libraries", []):
|
||||||
|
if lib["name"] == name:
|
||||||
|
return lib["id"]
|
||||||
|
sys.exit(f"no library named {name}")
|
||||||
|
|
||||||
|
|
||||||
|
def all_item_ids(token, lid):
|
||||||
|
ids, page = [], 0
|
||||||
|
code, one = call("GET", f"/api/libraries/{lid}/items?limit=1", token=token)
|
||||||
|
total = one.get("total", -1) if code == 200 else -1
|
||||||
|
while True:
|
||||||
|
code, body = call("GET", f"/api/libraries/{lid}/items?page={page}&limit=500",
|
||||||
|
token=token)
|
||||||
|
if code != 200:
|
||||||
|
sys.exit(f"items page {page} failed {code}")
|
||||||
|
results = body.get("results", [])
|
||||||
|
if not results:
|
||||||
|
break
|
||||||
|
ids += [it["id"] for it in results]
|
||||||
|
page += 1
|
||||||
|
if total >= 0 and page * 500 >= total:
|
||||||
|
break
|
||||||
|
return list(dict.fromkeys(ids)), total
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_covers(args):
|
||||||
|
copied = missing = exists = 0
|
||||||
|
for line in open(args.list):
|
||||||
|
src, _, dst = line.rstrip("\n").partition("\t")
|
||||||
|
if not dst:
|
||||||
|
continue
|
||||||
|
if os.path.exists(dst):
|
||||||
|
exists += 1
|
||||||
|
continue
|
||||||
|
if not os.path.exists(src):
|
||||||
|
missing += 1
|
||||||
|
continue
|
||||||
|
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||||
|
shutil.copy2(src, dst)
|
||||||
|
copied += 1
|
||||||
|
print(f"covers copied: {copied} already there: {exists} missing source: {missing}")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_wipe(args, token, lid):
|
||||||
|
ids, total = all_item_ids(token, lid)
|
||||||
|
print(f"library {lid}: {len(ids)} items (total={total})")
|
||||||
|
if not args.apply:
|
||||||
|
print("dry run: nothing deleted (add --apply)")
|
||||||
|
return
|
||||||
|
gone = 0
|
||||||
|
for i, iid in enumerate(ids):
|
||||||
|
code, _ = call("DELETE", f"/api/items/{iid}", token=token)
|
||||||
|
if code == 200:
|
||||||
|
gone += 1
|
||||||
|
elif i < 3:
|
||||||
|
print(f"delete {iid}: {code}")
|
||||||
|
if i % 500 == 499:
|
||||||
|
print(f" {i + 1}/{len(ids)} deleted")
|
||||||
|
print(f"deleted {gone}/{len(ids)} item rows")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_scan(args, token, lid):
|
||||||
|
code, body = call("POST", f"/api/libraries/{lid}/scan", token=token)
|
||||||
|
print(f"scan: {code} {body if code != 200 else 'accepted'}")
|
||||||
|
if not args.wait:
|
||||||
|
return
|
||||||
|
last = -1
|
||||||
|
while True:
|
||||||
|
time.sleep(30)
|
||||||
|
code, lib = call("GET", f"/api/libraries/{lid}", token=token)
|
||||||
|
code2, one = call("GET", f"/api/libraries/{lid}/items?limit=1", token=token)
|
||||||
|
total = one.get("total", -1) if code2 == 200 else -1
|
||||||
|
fetching = lib.get("isFetchingLibrary") if code == 200 else None
|
||||||
|
print(f" fetching={fetching} items={total}")
|
||||||
|
if fetching is False and total == last and total >= 0:
|
||||||
|
break
|
||||||
|
last = total
|
||||||
|
print(f"scan settled at {total} items")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_status(token, lid):
|
||||||
|
missing = 0
|
||||||
|
seen, page = set(), 0
|
||||||
|
while True:
|
||||||
|
code, body = call("GET", f"/api/libraries/{lid}/items?page={page}&limit=500",
|
||||||
|
token=token)
|
||||||
|
results = body.get("results", []) if code == 200 else []
|
||||||
|
if not results:
|
||||||
|
break
|
||||||
|
for it in results:
|
||||||
|
if it["id"] not in seen:
|
||||||
|
seen.add(it["id"])
|
||||||
|
if it.get("isMissing"):
|
||||||
|
missing += 1
|
||||||
|
page += 1
|
||||||
|
code, lib = call("GET", f"/api/libraries/{lid}", token=token)
|
||||||
|
print(f"items: {len(seen)} isMissing: {missing} "
|
||||||
|
f"pending tasks: {lib.get('pendingTasks') if code == 200 else '?'}")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||||
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||||
|
c = sub.add_parser("covers")
|
||||||
|
c.add_argument("--list", required=True)
|
||||||
|
w = sub.add_parser("wipe")
|
||||||
|
w.add_argument("--apply", action="store_true")
|
||||||
|
s = sub.add_parser("scan")
|
||||||
|
s.add_argument("--wait", action="store_true")
|
||||||
|
sub.add_parser("status")
|
||||||
|
args = ap.parse_args()
|
||||||
|
token = login()
|
||||||
|
lid = library_id(token)
|
||||||
|
if args.cmd == "covers":
|
||||||
|
cmd_covers(args)
|
||||||
|
elif args.cmd == "wipe":
|
||||||
|
cmd_wipe(args, token, lid)
|
||||||
|
elif args.cmd == "scan":
|
||||||
|
cmd_scan(args, token, lid)
|
||||||
|
else:
|
||||||
|
cmd_status(token, lid)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -323,11 +323,13 @@ copy_alloy_config() {
|
|||||||
# The repo config carries the token GRAFANA_HOST_LABEL; the installed copy
|
# The repo config carries the token GRAFANA_HOST_LABEL; the installed copy
|
||||||
# carries the host's name, so Loki streams say WHICH machine's journal a
|
# carries the host's name, so Loki streams say WHICH machine's journal a
|
||||||
# line came from without a second config file per host.
|
# line came from without a second config file per host.
|
||||||
# *_CONF_ROOT are test seams: the bats cases execute these copiers for
|
# *_CONF_ROOT and ALLOY_CONFIG_SRC are test seams: the bats cases execute
|
||||||
# real against a sandbox with a stubbed run_privileged.
|
# these copiers for real against a sandbox with a stubbed run_privileged;
|
||||||
|
# ALLOY_CONFIG_SRC lets the refusal case feed a deliberately broken config.
|
||||||
local root="${ALLOY_CONF_ROOT:-/opt/alloy}" tmp
|
local root="${ALLOY_CONF_ROOT:-/opt/alloy}" tmp
|
||||||
|
local src="${ALLOY_CONFIG_SRC:-shared/alloy/config.alloy}"
|
||||||
tmp="$(mktemp)"
|
tmp="$(mktemp)"
|
||||||
sed "s/GRAFANA_HOST_LABEL/$TARGET/g" shared/alloy/config.alloy > "$tmp"
|
sed "s/GRAFANA_HOST_LABEL/$TARGET/g" "$src" > "$tmp"
|
||||||
if ! grep -q "host = \"$TARGET\"," "$tmp" || grep -q "GRAFANA_HOST_LABEL" "$tmp"; then
|
if ! grep -q "host = \"$TARGET\"," "$tmp" || grep -q "GRAFANA_HOST_LABEL" "$tmp"; then
|
||||||
echo "alloy config: host-label substitution failed" >&2
|
echo "alloy config: host-label substitution failed" >&2
|
||||||
rm -f "$tmp"
|
rm -f "$tmp"
|
||||||
|
|||||||
Executable
+420
@@ -0,0 +1,420 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""media-ab-reorg.py — plan the Audiobooks dedupe + ABS-structure relayout.
|
||||||
|
|
||||||
|
2026-11-04 campaign. Turns the Audiobooks library into the layout Audiobookshelf
|
||||||
|
documents (https://audiobookshelf.org/docs/documentation/libraries/book-library/
|
||||||
|
directory-structure/):
|
||||||
|
|
||||||
|
{Author}/[{Series}/]{seq} - {year} - {Title} {Narrator}/…files…
|
||||||
|
|
||||||
|
The authority for every book's author/series/title/year/narrator is the ABS
|
||||||
|
database dump (--items, JSON lines: [relPath, isMissing, title, authors, series,
|
||||||
|
seq, year, narratorsJSON, coverPath, ebookFlag]); files are attached to their
|
||||||
|
book by longest-prefix match on the item's relPath, which is how ABS's own
|
||||||
|
grouping works. Folder names are then re-emitted so ABS's folder parser reads
|
||||||
|
the same metadata back off disk.
|
||||||
|
|
||||||
|
Three outputs, all consumable by scripts/media-apply.py (run everything with
|
||||||
|
--root /mnt/entropy; hold/intake/park live outside the library on purpose):
|
||||||
|
|
||||||
|
--plan mkdir/keep/move rows: dedupe losers -> --hold, ebook-only items
|
||||||
|
and comics -> --intake (the eBooks pile feeds Calibre next),
|
||||||
|
unowned media -> --park, everything else -> the new layout.
|
||||||
|
--covers rows "srchost \\t dstarray": covers ABS keeps in /metadata get a
|
||||||
|
copy beside the book (cp, it crosses filesystems); run this
|
||||||
|
BEFORE apply so the cover lands inside the moved folder.
|
||||||
|
--exceptions every judgement call: name collisions, byte-dups that pair two
|
||||||
|
different books, collapsed items, stray files.
|
||||||
|
|
||||||
|
Dedupe law: only CONFIRMED byte-identities (--dups from
|
||||||
|
media-ab-verifydups.py) remove a file. One copy survives per book: inside a
|
||||||
|
folder the cleanest-named file wins; across folders the item whose root matches
|
||||||
|
its author wins. Equal bytes under two different book identities keep both and
|
||||||
|
raise an exception — a trailer embedded in two books is not a duplicate.
|
||||||
|
|
||||||
|
Rerunnable: a second run against a moved library emits keep rows, not moves.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
AUDIO = {"m4b", "mp3", "m4a", "flac", "opus", "ogg", "oga", "mp4", "aac", "wma",
|
||||||
|
"aiff", "aif", "wav", "webm", "webma", "mka", "awb", "caf", "mpg",
|
||||||
|
"mpg2", "mpeg"}
|
||||||
|
EBOOK = {"epub", "pdf", "mobi", "azw3", "cbr", "cbz", "fb2"}
|
||||||
|
COMIC = {"cbr", "cbz"}
|
||||||
|
MEDIA = AUDIO | {"epub", "pdf", "mobi", "azw3", "cbr", "cbz"}
|
||||||
|
PLACEHOLDER_AUTHOR = {"unknown", "unknown artist", "various", "various artists",
|
||||||
|
"various authors", "n/a", ""}
|
||||||
|
BAD_CHARS = re.compile(r'[<>:"/\\|?*]')
|
||||||
|
DISC_DIR = re.compile(r"^(disc|cd|disk)\s*\d", re.I)
|
||||||
|
KEEP_HINT = re.compile(r"\s*\((?:2|3|copy|backup|duplicate)\)\s*$", re.I)
|
||||||
|
META_HOST = "/home/dank/Containers/audiobookshelf"
|
||||||
|
|
||||||
|
|
||||||
|
def clean(name):
|
||||||
|
name = BAD_CHARS.sub(" ", name or "")
|
||||||
|
name = re.sub(r"\s+", " ", name).strip().strip(".")
|
||||||
|
return name or "Untitled"
|
||||||
|
|
||||||
|
|
||||||
|
def clean_author(name):
|
||||||
|
"""DB author string -> folder name: one author, no 'zz' sort artifact."""
|
||||||
|
name = re.sub(r"^zz", "", (name or "").strip())
|
||||||
|
return clean(name)
|
||||||
|
|
||||||
|
|
||||||
|
def first_author(authors):
|
||||||
|
parts = [clean_author(p) for p in (authors or "").split("|")]
|
||||||
|
parts = [p for p in parts if p]
|
||||||
|
return parts[0] if parts else "Unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_narrators(blob):
|
||||||
|
try:
|
||||||
|
v = json.loads(blob) if blob else []
|
||||||
|
except (TypeError, json.JSONDecodeError):
|
||||||
|
v = []
|
||||||
|
if isinstance(v, str):
|
||||||
|
v = [v]
|
||||||
|
return [str(x).strip() for x in (v or []) if str(x).strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def core_title(title, author):
|
||||||
|
"""Title as the book folder should show it: release prefixes stripped."""
|
||||||
|
t = (title or "Untitled").strip()
|
||||||
|
a = (author or "").strip()
|
||||||
|
if a and t.lower().startswith(a.lower() + " -"):
|
||||||
|
t = t[len(a):].lstrip(" -")
|
||||||
|
return t.strip() or "Untitled"
|
||||||
|
|
||||||
|
|
||||||
|
def bookname(seq, year, title, narrators):
|
||||||
|
seq = (seq or "").strip()
|
||||||
|
seq = seq if re.fullmatch(r"\d+(\.\d+)?", seq) else ""
|
||||||
|
year = (year or "").strip()
|
||||||
|
year = year if re.fullmatch(r"(1[5-9]|20)\d\d", year) else ""
|
||||||
|
t = title
|
||||||
|
if seq:
|
||||||
|
# drop a sequence prefix already carried inside the title
|
||||||
|
t = re.sub(rf"^{re.escape(seq)}\s*[\-.]\s+", "", t)
|
||||||
|
if year:
|
||||||
|
t = re.sub(rf"^{year}\s*[\-.]\s+", "", t)
|
||||||
|
t = re.sub(rf"\s*\({year}\)\s*$", "", t)
|
||||||
|
name = t
|
||||||
|
if seq and year:
|
||||||
|
name = f"{seq} - {year} - {name}"
|
||||||
|
elif seq:
|
||||||
|
name = f"{seq} - {name}"
|
||||||
|
elif year:
|
||||||
|
name = f"{year} - {name}"
|
||||||
|
if len(narrators) == 1:
|
||||||
|
name = f"{name} {{{clean(narrators[0])}}}"
|
||||||
|
return clean(name)
|
||||||
|
|
||||||
|
|
||||||
|
class Campaign:
|
||||||
|
def __init__(self, args):
|
||||||
|
self.root = args.root.rstrip("/")
|
||||||
|
self.lib = self.root + "/Audiobooks"
|
||||||
|
self.hold = args.hold
|
||||||
|
self.intake = args.intake
|
||||||
|
self.park = args.park
|
||||||
|
self.items = {} # relPath -> item dict (healthy only)
|
||||||
|
self.stale = [] # isMissing=1 rows
|
||||||
|
self.moved = {} # src -> (dst, kind)
|
||||||
|
self.claims = {} # dst -> src (collision guard)
|
||||||
|
self.loser_files = set()
|
||||||
|
self.loser_items = set()
|
||||||
|
self.exc = defaultdict(list)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- inputs
|
||||||
|
def load_items(self, path):
|
||||||
|
for line in open(path):
|
||||||
|
r = json.loads(line)
|
||||||
|
rel = r[0]
|
||||||
|
item = {"rel": rel, "title": r[2] or "", "authors": r[3] or "",
|
||||||
|
"series": r[4] or "", "seq": r[5] or "", "year": r[6] or "",
|
||||||
|
"narr": parse_narrators(r[7]), "cover": r[8] or "",
|
||||||
|
"ebook": r[9]}
|
||||||
|
if r[1]:
|
||||||
|
self.stale.append(rel)
|
||||||
|
continue
|
||||||
|
self.items[rel] = item
|
||||||
|
|
||||||
|
def load_files(self, path):
|
||||||
|
files = []
|
||||||
|
for line in open(path):
|
||||||
|
sz, ext, p = line.rstrip("\n").split("\t", 2)
|
||||||
|
if p.startswith(self.lib + "/"):
|
||||||
|
files.append((int(sz), ext.lstrip(".").lower(), p))
|
||||||
|
return files
|
||||||
|
|
||||||
|
def owner(self, d):
|
||||||
|
"""The healthy DB item whose folder is d or an ancestor of d."""
|
||||||
|
rel = os.path.relpath(d, self.lib)
|
||||||
|
while rel and rel != ".":
|
||||||
|
if rel in self.items:
|
||||||
|
return rel
|
||||||
|
rel = os.path.dirname(rel)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- dedupe
|
||||||
|
def identity(self, item_rel):
|
||||||
|
if item_rel is None:
|
||||||
|
return None
|
||||||
|
it = self.items[item_rel]
|
||||||
|
return (first_author(it["authors"]).lower(),
|
||||||
|
core_title(it["title"], first_author(it["authors"])).lower(),
|
||||||
|
(it["year"] or "").strip())
|
||||||
|
|
||||||
|
def root_score(self, item_rel):
|
||||||
|
it = self.items[item_rel]
|
||||||
|
top = item_rel.split("/")[0]
|
||||||
|
s = 0
|
||||||
|
if top.lower() == first_author(it["authors"]).lower():
|
||||||
|
s += 2
|
||||||
|
if not top.startswith("_") and "various" not in top.lower():
|
||||||
|
s += 1
|
||||||
|
return (s, -len(item_rel))
|
||||||
|
|
||||||
|
def dedupe(self, files, dups_path):
|
||||||
|
by_hash = defaultdict(list)
|
||||||
|
if dups_path:
|
||||||
|
for line in open(dups_path):
|
||||||
|
h, sz, p = line.rstrip("\n").split("\t", 2)
|
||||||
|
if p.startswith(self.lib + "/"):
|
||||||
|
by_hash[h].append(p)
|
||||||
|
owner_of = {}
|
||||||
|
dirs = {os.path.dirname(p) for _, _, p in files}
|
||||||
|
for d in dirs:
|
||||||
|
owner_of[d] = self.owner(d)
|
||||||
|
files_by_owner = defaultdict(list)
|
||||||
|
for _, _, p in files:
|
||||||
|
files_by_owner[owner_of[os.path.dirname(p)]].append(p)
|
||||||
|
|
||||||
|
for h, paths in by_hash.items():
|
||||||
|
owners = {owner_of.get(os.path.dirname(p)) for p in paths}
|
||||||
|
owners = {o for o in owners if o is not None}
|
||||||
|
per_item = defaultdict(list)
|
||||||
|
for p in paths:
|
||||||
|
per_item[owner_of.get(os.path.dirname(p))].append(p)
|
||||||
|
if owners:
|
||||||
|
ids = {o: self.identity(o) for o in owners}
|
||||||
|
# group items by identity; loose (None) files follow their own rule
|
||||||
|
same_as_loser = []
|
||||||
|
keepers = []
|
||||||
|
byid = defaultdict(list)
|
||||||
|
for o in owners:
|
||||||
|
byid[ids[o]].append(o)
|
||||||
|
for ident, os_ in byid.items():
|
||||||
|
os_.sort(key=self.root_score, reverse=True)
|
||||||
|
keepers.append(os_[0])
|
||||||
|
same_as_loser.extend(os_[1:])
|
||||||
|
keepers = set(keepers)
|
||||||
|
loser_items = set(same_as_loser)
|
||||||
|
else:
|
||||||
|
loser_items = set()
|
||||||
|
keepers = set()
|
||||||
|
for o, ps in per_item.items():
|
||||||
|
if o is None:
|
||||||
|
continue # loose files decided in placement
|
||||||
|
if o in loser_items:
|
||||||
|
self.loser_files.update(ps)
|
||||||
|
continue
|
||||||
|
if len(ps) > 1 and o in keepers:
|
||||||
|
# redundant copies inside the surviving folder
|
||||||
|
keeper = min(ps, key=lambda p: (
|
||||||
|
bool(KEEP_HINT.search(os.path.splitext(p)[0])),
|
||||||
|
len(p), p))
|
||||||
|
self.loser_files.update(p for p in ps if p != keeper)
|
||||||
|
for o in loser_items:
|
||||||
|
self.loser_items.add(o)
|
||||||
|
# collapsed items: every media file lost
|
||||||
|
for o, ps in files_by_owner.items():
|
||||||
|
if o is None:
|
||||||
|
continue
|
||||||
|
media = [p for p in ps if os.path.splitext(p)[1].lstrip(".").lower()
|
||||||
|
in MEDIA]
|
||||||
|
if media and all(p in self.loser_files for p in media):
|
||||||
|
self.loser_items.add(o)
|
||||||
|
self.exc["collapsed items (all files were byte-duplicates)"].append(o)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------- placement
|
||||||
|
def target(self, item):
|
||||||
|
author = first_author(item["authors"])
|
||||||
|
series = clean(item["series"]) if item["series"] else ""
|
||||||
|
title = core_title(item["title"], author)
|
||||||
|
name = bookname(item["seq"], item["year"], title, item["narr"])
|
||||||
|
parts = [self.lib, clean(author)]
|
||||||
|
if series:
|
||||||
|
parts.append(clean(series))
|
||||||
|
parts.append(name)
|
||||||
|
return os.path.join(*parts)
|
||||||
|
|
||||||
|
def claim(self, dst, src, why):
|
||||||
|
"""One destination, one source; collisions rename with a marker."""
|
||||||
|
if dst not in self.claims:
|
||||||
|
self.claims[dst] = src
|
||||||
|
return dst
|
||||||
|
stem, ext = os.path.splitext(dst)
|
||||||
|
n = 2
|
||||||
|
while f"{stem} ({n}){ext}" in self.claims:
|
||||||
|
n += 1
|
||||||
|
dst2 = f"{stem} ({n}){ext}"
|
||||||
|
self.claims[dst2] = src
|
||||||
|
self.exc[why].append(f"{src} -> renamed to {dst2}")
|
||||||
|
return dst2
|
||||||
|
|
||||||
|
def place(self, files):
|
||||||
|
audio_by_item = defaultdict(int)
|
||||||
|
for _, ext, p in files:
|
||||||
|
o = self.owner(os.path.dirname(p))
|
||||||
|
if o and ext in AUDIO:
|
||||||
|
audio_by_item[o] += 1
|
||||||
|
cover_dirs = {}
|
||||||
|
# Claim one BOOK FOLDER per item: two items for the same title (the
|
||||||
|
# _unsorted era filed some books 3 times) must not merge their tracks
|
||||||
|
# into one item — the loser gets a variant folder instead.
|
||||||
|
for rel in sorted(self.items, key=lambda r: self.root_score(r),
|
||||||
|
reverse=True):
|
||||||
|
it = self.items[rel]
|
||||||
|
t = self.target(it)
|
||||||
|
if t in self.claims:
|
||||||
|
n = 2
|
||||||
|
while f"{t} (variant {n})" in self.claims:
|
||||||
|
n += 1
|
||||||
|
t = f"{t} (variant {n})"
|
||||||
|
self.exc["same title filed by two DB items"].append(f"{rel} -> {t}")
|
||||||
|
self.claims[t] = rel
|
||||||
|
cover_dirs[rel] = t
|
||||||
|
if (it["cover"].startswith("/metadata") and audio_by_item[rel]
|
||||||
|
and rel not in self.loser_items):
|
||||||
|
src = META_HOST + it["cover"]
|
||||||
|
self.covers.append((src, os.path.join(t,
|
||||||
|
"cover" + os.path.splitext(src)[1].lower())))
|
||||||
|
# Containment gate: an item folder holding media files must never sit
|
||||||
|
# above another item folder — ABS's grouping would swallow the child
|
||||||
|
# (the docs' PROBLEM example). Rename the ANCESTOR book out of the way.
|
||||||
|
for _ in range(3):
|
||||||
|
moved = False
|
||||||
|
folders = sorted(set(cover_dirs.values()))
|
||||||
|
for anc in folders:
|
||||||
|
for d in folders:
|
||||||
|
if d != anc and d.startswith(anc + os.sep):
|
||||||
|
n = 1
|
||||||
|
cand = f"{anc} (book)" if n == 1 else f"{anc} (book {n})"
|
||||||
|
while cand in self.claims:
|
||||||
|
n += 1
|
||||||
|
cand = f"{anc} (book {n})"
|
||||||
|
self.exc["book folder nested above another book"].append(
|
||||||
|
f"{anc} -> {cand}")
|
||||||
|
self.claims[cand] = self.claims.pop(anc)
|
||||||
|
for r, t in cover_dirs.items():
|
||||||
|
if t == anc:
|
||||||
|
cover_dirs[r] = cand
|
||||||
|
self.covers = [(s, d.replace(anc + os.sep, cand + os.sep, 1))
|
||||||
|
for s, d in self.covers]
|
||||||
|
moved = True
|
||||||
|
break
|
||||||
|
if moved:
|
||||||
|
break
|
||||||
|
if not moved:
|
||||||
|
break
|
||||||
|
for _, ext, p in files:
|
||||||
|
if p in self.loser_files:
|
||||||
|
d = os.path.join(self.hold, os.path.relpath(p, self.root))
|
||||||
|
self.move(p, d, "dup")
|
||||||
|
continue
|
||||||
|
d = os.path.dirname(p)
|
||||||
|
o = self.owner(d)
|
||||||
|
if o is None:
|
||||||
|
rel = os.path.relpath(p, self.lib)
|
||||||
|
if ext in MEDIA:
|
||||||
|
self.move(p, os.path.join(self.park, rel), "park")
|
||||||
|
else:
|
||||||
|
self.move(p, os.path.join(self.park, rel), "park-junk")
|
||||||
|
continue
|
||||||
|
it = self.items[o]
|
||||||
|
if ext in COMIC:
|
||||||
|
key = clean(f"{first_author(it['authors'])} - {core_title(it['title'], '')}")
|
||||||
|
dst = os.path.join(self.intake, "_comics", key, os.path.basename(p))
|
||||||
|
self.move(p, self.claim(dst, p, "intake comic collisions"), "comic")
|
||||||
|
continue
|
||||||
|
if audio_by_item[o] == 0: # ebook-only item belongs to the ebook stack
|
||||||
|
key = f"{first_author(it['authors'])} - {bookname(it['seq'], it['year'], core_title(it['title'], first_author(it['authors'])), it['narr'])}"
|
||||||
|
dst = os.path.join(self.intake, key, os.path.relpath(p, os.path.join(self.lib, o)))
|
||||||
|
self.move(p, self.claim(dst, p, "intake ebook collisions"), "ebook-only")
|
||||||
|
continue
|
||||||
|
rel_in = os.path.relpath(p, os.path.join(self.lib, o))
|
||||||
|
dst = os.path.join(cover_dirs[o], rel_in)
|
||||||
|
if dst == p:
|
||||||
|
self.kept.append(p)
|
||||||
|
self.claims.setdefault(dst, p)
|
||||||
|
continue
|
||||||
|
self.move(p, self.claim(dst, p, "target collisions"), "move")
|
||||||
|
|
||||||
|
def move(self, src, dst, kind):
|
||||||
|
self.moves.append((src, dst, kind))
|
||||||
|
|
||||||
|
def write(self, plan_path, exc_path):
|
||||||
|
dirs = {os.path.dirname(dst) for _, dst, _ in self.moves}
|
||||||
|
dirs |= {os.path.dirname(d) for _, d in self.covers}
|
||||||
|
all_dirs = set()
|
||||||
|
for d in dirs:
|
||||||
|
while d and d != self.root:
|
||||||
|
all_dirs.add(d)
|
||||||
|
d = os.path.dirname(d)
|
||||||
|
with open(plan_path, "w") as fh:
|
||||||
|
fh.write("action\tsrc\tdst\tkind\n")
|
||||||
|
for d in sorted(all_dirs):
|
||||||
|
fh.write(f"mkdir\t\t{d}\tcampaign\n")
|
||||||
|
for p in self.kept:
|
||||||
|
fh.write(f"keep\t{p}\t\tkeep\n")
|
||||||
|
for src, dst, kind in sorted(self.moves):
|
||||||
|
fh.write(f"move\t{src}\t{dst}\t{kind}\n")
|
||||||
|
if self.exc:
|
||||||
|
with open(exc_path, "w") as fh:
|
||||||
|
for why in sorted(self.exc):
|
||||||
|
fh.write(f"# {why}\n")
|
||||||
|
for line in sorted(set(self.exc[why])):
|
||||||
|
fh.write(line + "\n")
|
||||||
|
fh.write("\n")
|
||||||
|
mv = [m for m in self.moves if m[2] != "dup"]
|
||||||
|
print(f"plan: {len(all_dirs)} folders, {len(self.moves)} moves "
|
||||||
|
f"({len(mv)} relayout, {len(self.moves)-len(mv)} dup-hold), "
|
||||||
|
f"{len(self.kept)} keep, {len(self.covers)} cover-copies")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||||
|
ap.add_argument("--root", default="/mnt/entropy")
|
||||||
|
ap.add_argument("--enum", required=True)
|
||||||
|
ap.add_argument("--items", required=True)
|
||||||
|
ap.add_argument("--dups", default=None)
|
||||||
|
ap.add_argument("--hold", default="/mnt/entropy/.media-campaign/dup-hold-20261104")
|
||||||
|
ap.add_argument("--intake", default="/mnt/entropy/eBooks/_from-audiobooks")
|
||||||
|
ap.add_argument("--park", default="/mnt/entropy/NeedsProcessing/_from-audiobooks-20261104")
|
||||||
|
ap.add_argument("--plan", required=True)
|
||||||
|
ap.add_argument("--exceptions", required=True)
|
||||||
|
ap.add_argument("--covers", required=True)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
c = Campaign(args)
|
||||||
|
c.moves, c.kept, c.covers = [], [], []
|
||||||
|
c.load_items(args.items)
|
||||||
|
files = c.load_files(args.enum)
|
||||||
|
c.dedupe(files, args.dups)
|
||||||
|
c.place(files)
|
||||||
|
c.write(args.plan, args.exceptions)
|
||||||
|
with open(args.covers, "w") as fh:
|
||||||
|
for src, dst in sorted(set(c.covers)):
|
||||||
|
fh.write(f"{src}\t{dst}\n")
|
||||||
|
print(f"covers: {len(set(c.covers))} stale DB rows (delete after scan): {len(c.stale)}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+89
@@ -0,0 +1,89 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""media-ab-verifydups.py — confirm byte-duplicates among size-group survivors.
|
||||||
|
|
||||||
|
Stage 2 of the 2026-11-04 Audiobooks dedupe. Stage 1 (partial blake2b of the
|
||||||
|
first+last 4 MiB, done by the campaign's hashcand.py) over-claims: two files
|
||||||
|
of equal size and equal head+tail are CANDIDATES, not copies. This tool takes
|
||||||
|
the candidate files from --partial (rows: size, hash, path), groups them by
|
||||||
|
(size, partial-hash), full-hashes only members of groups with 2+ members, and
|
||||||
|
writes verdicts: one line per file whose full hash ties it to at least one
|
||||||
|
other file.
|
||||||
|
|
||||||
|
Output rows: full-hash <TAB> size <TAB> path (sorted by hash) — a hash that
|
||||||
|
appears once is not written at all; singletons are not duplicates.
|
||||||
|
|
||||||
|
Read-only. Never moves, never deletes. Run it after the partial pass finishes
|
||||||
|
(pgrep it first; two hashers on one output file have raced before).
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
|
||||||
|
def full_hash(path, chunk=1 << 20):
|
||||||
|
h = hashlib.blake2b(digest_size=32)
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
for block in iter(lambda: fh.read(chunk), b""):
|
||||||
|
h.update(block)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||||
|
ap.add_argument("--partial", required=True, help="partial-hashes.tsv")
|
||||||
|
ap.add_argument("--out", required=True, help="full-hash verdicts TSV")
|
||||||
|
ap.add_argument("--progress-every", type=int, default=50)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
groups = defaultdict(list)
|
||||||
|
with open(args.partial) as fh:
|
||||||
|
for line in fh:
|
||||||
|
f = line.rstrip("\n").split("\t")
|
||||||
|
if len(f) != 3 or f[0] == "ERR":
|
||||||
|
continue
|
||||||
|
groups[(int(f[0]), f[1])].append(f[2])
|
||||||
|
survivors = sorted({p for g in groups.values() if len(g) > 1 for p in g})
|
||||||
|
print(f"survivor files to full-hash: {len(survivors)}")
|
||||||
|
|
||||||
|
done = set()
|
||||||
|
if os.path.exists(args.out):
|
||||||
|
with open(args.out) as fh:
|
||||||
|
for line in fh:
|
||||||
|
done.add(line.rstrip("\n").split("\t", 2)[2])
|
||||||
|
full = {}
|
||||||
|
for line in open(args.out) if os.path.exists(args.out) else []:
|
||||||
|
h, s, p = line.rstrip("\n").split("\t", 2)
|
||||||
|
full[p] = h
|
||||||
|
|
||||||
|
todo = [p for p in survivors if p not in done]
|
||||||
|
out = open(args.out, "a")
|
||||||
|
for i, p in enumerate(todo):
|
||||||
|
try:
|
||||||
|
full[p] = full_hash(p)
|
||||||
|
except OSError as e:
|
||||||
|
print(f"SKIP {p}: {e}")
|
||||||
|
continue
|
||||||
|
if i % args.progress_every == args.progress_every - 1:
|
||||||
|
out.flush()
|
||||||
|
out.close()
|
||||||
|
|
||||||
|
# Rewrite the whole verdict file keeping only TRUE duplicates (2+ fulls).
|
||||||
|
by_hash = defaultdict(list)
|
||||||
|
for p in survivors:
|
||||||
|
if p in full:
|
||||||
|
by_hash[full[p]].append(p)
|
||||||
|
with open(args.out, "w") as out:
|
||||||
|
for h, ps in sorted(by_hash.items()):
|
||||||
|
if len(ps) > 1:
|
||||||
|
for p in sorted(ps):
|
||||||
|
out.write(f"{h}\t{os.path.getsize(p)}\t{p}\n")
|
||||||
|
dups = {h: ps for h, ps in by_hash.items() if len(ps) > 1}
|
||||||
|
n = sum(len(ps) - 1 for ps in dups.values())
|
||||||
|
waste = sum((len(ps) - 1) * os.path.getsize(ps[0]) for ps in dups.values())
|
||||||
|
print(f"true dup sets: {len(dups)} redundant files: {n} "
|
||||||
|
f"waste: {waste / 2**30:.1f} GiB")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+239
@@ -0,0 +1,239 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""media-ebook-dedupe.py — dedupe the eBooks pile against the Calibre library.
|
||||||
|
|
||||||
|
2026-11-04 campaign, ebook side. The Calibre library (/mnt/entropy/CalibreBooks,
|
||||||
|
6,289 books) is DB-owned and healthy; /mnt/entropy/eBooks is a raw pile whose
|
||||||
|
best books already live in the library, several of them twice in the pile.
|
||||||
|
Audiobooks' orphaned ebooks/comics land in the pile first (media-ab-reorg's
|
||||||
|
intake), so this campaign runs after the audiobook move.
|
||||||
|
|
||||||
|
Inputs
|
||||||
|
--meta ebook-meta.tsv: JSON lines {p, sz, title, authors, ident} for
|
||||||
|
epub (embedded OPF) and mobi/azw3 (Palm header title); the
|
||||||
|
producer is the campaign's opfcensus.py.
|
||||||
|
--enum enum TSV (size, ext, path) for every tree.
|
||||||
|
--calibre calibre-books.tsv: JSON lines [id, title, authors, series, path,
|
||||||
|
pubdate, formats] dumped from metadata.db.
|
||||||
|
--dups full blake2b verdicts (hash, size, path) — only confirmed
|
||||||
|
byte-identities remove a file. Optional: without it, identity
|
||||||
|
rules alone run and every hash-dependent pair stays a KEEP.
|
||||||
|
|
||||||
|
Outputs
|
||||||
|
--plan media-apply plan (--root /mnt/entropy): hash-duplicates of a
|
||||||
|
file already in CalibreBooks -> HOLD; byte-identical files inside
|
||||||
|
the pile keep the best format of one book once; non-book junk
|
||||||
|
(mp4/mp3/images) -> PARK.
|
||||||
|
--ingest one JSON line per surviving BOOK (a set of format files of one
|
||||||
|
identity): {paths, title, authors, source}. The ingest runner
|
||||||
|
feeds these to calibredb inside the calibre-web image.
|
||||||
|
--exceptions every identity pair the file-level evidence could not settle.
|
||||||
|
|
||||||
|
Match law: identity is (core title, first author surname). Calibre's
|
||||||
|
"Dinniman| Matt" and an OPF "Matt Dinniman" share surname "dinniman". Title
|
||||||
|
core drops the leading article and all non-alphanumerics. Identity alone NEVER
|
||||||
|
deletes against the pile — it only deletes when the same identity sits in the
|
||||||
|
Calibre library with the same format (the pile copy adds nothing), or when a
|
||||||
|
byte-hash confirms the pair. Format preference when the pile holds one book in
|
||||||
|
several copies: epub > azw3 > mobi > pdf > fb2 > lit > rtf > txt.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
BOOK_EXT = {"epub", "azw3", "mobi", "pdf", "fb2", "lit", "rtf", "txt", "cbr", "cbz"}
|
||||||
|
FORMAT_RANK = {"epub": 0, "azw3": 1, "mobi": 2, "pdf": 3, "fb2": 4,
|
||||||
|
"cbr": 5, "cbz": 5, "lit": 6, "rtf": 7, "txt": 8}
|
||||||
|
ARTICLE = re.compile(r"^(the|a|an)\s+", re.I)
|
||||||
|
|
||||||
|
|
||||||
|
def ascii_fold(s):
|
||||||
|
return "".join(c for c in unicodedata.normalize("NFKD", s or "")
|
||||||
|
if not unicodedata.combining(c))
|
||||||
|
|
||||||
|
|
||||||
|
def title_core(title):
|
||||||
|
t = ascii_fold(title).lower()
|
||||||
|
t = ARTICLE.sub("", t)
|
||||||
|
t = re.sub(r"\(.*?\)", " ", t) # (1998), (Unabridged) — noise
|
||||||
|
t = re.sub(r"[^a-z0-9 ]", " ", t)
|
||||||
|
t = re.sub(r"\b(1[5-9]|20)\d\d\b", " ", t) # years
|
||||||
|
return " ".join(t.split())[:120]
|
||||||
|
|
||||||
|
|
||||||
|
def surname(author):
|
||||||
|
a = ascii_fold(author).strip()
|
||||||
|
a = re.split(r"[;,]", a)[0] if "," in a else " ".join(a.split()[-1:])
|
||||||
|
return re.sub(r"[^a-z]", "", a.lower())
|
||||||
|
|
||||||
|
|
||||||
|
def first_author(authors):
|
||||||
|
parts = [p.strip() for p in re.split(r"[;&]", authors or "") if p.strip()]
|
||||||
|
return parts[0] if parts else ""
|
||||||
|
|
||||||
|
|
||||||
|
def stem_title(path):
|
||||||
|
s = os.path.splitext(os.path.basename(path))[0]
|
||||||
|
return ascii_fold(s)
|
||||||
|
|
||||||
|
|
||||||
|
class Campaign:
|
||||||
|
def __init__(self, args):
|
||||||
|
self.args = args
|
||||||
|
self.root = args.root.rstrip("/")
|
||||||
|
self.pile = self.root + "/eBooks"
|
||||||
|
self.lib = self.root + "/CalibreBooks"
|
||||||
|
self.hold = args.hold
|
||||||
|
self.park = args.park
|
||||||
|
self.meta = {} # path -> meta dict
|
||||||
|
self.pile_files = [] # (sz, ext, path)
|
||||||
|
self.cal_by_path = {} # file path under CalibreBooks -> calibre row
|
||||||
|
self.cal_identity = {} # (tcore, surname) -> calibre row
|
||||||
|
self.drop = {} # path -> reason
|
||||||
|
self.exc = defaultdict(list)
|
||||||
|
self.moves = []
|
||||||
|
self.dirs_pile_book = defaultdict(list) # book dir (pile) -> files
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- inputs
|
||||||
|
def load(self):
|
||||||
|
for line in open(self.args.meta):
|
||||||
|
r = json.loads(line)
|
||||||
|
self.meta[r["p"]] = r
|
||||||
|
by_hash = defaultdict(list)
|
||||||
|
if self.args.dups:
|
||||||
|
for line in open(self.args.dups):
|
||||||
|
h, sz, p = line.rstrip("\n").split("\t", 2)
|
||||||
|
by_hash[h].append(p)
|
||||||
|
self.dup_sets = [ps for ps in by_hash.values() if len(ps) > 1]
|
||||||
|
for line in open(self.args.enum):
|
||||||
|
sz, ext, p = line.rstrip("\n").split("\t", 2)
|
||||||
|
ext = ext.lstrip(".").lower()
|
||||||
|
if p.startswith(self.pile + "/") and p not in self.park:
|
||||||
|
self.pile_files.append((int(sz), ext, p))
|
||||||
|
for line in open(self.args.calibre):
|
||||||
|
row = json.loads(line)
|
||||||
|
self.cal_identity[(title_core(row[1]), surname(first_author(row[2])))] = row
|
||||||
|
|
||||||
|
def identity(self, p):
|
||||||
|
m = self.meta.get(p)
|
||||||
|
if m and m.get("title"):
|
||||||
|
t = title_core(m["title"])
|
||||||
|
a = surname(first_author(m.get("authors", "")))
|
||||||
|
if t:
|
||||||
|
return (t, a)
|
||||||
|
return (title_core(stem_title(p)), "")
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- rules
|
||||||
|
def run(self):
|
||||||
|
pile_paths = {p for _, _, p in self.pile_files}
|
||||||
|
# R1/R2: confirmed byte-duplicates
|
||||||
|
for ps in self.dup_sets:
|
||||||
|
in_lib = [p for p in ps if p.startswith(self.lib + "/")]
|
||||||
|
in_pile = [p for p in ps if p.startswith(self.pile + "/")]
|
||||||
|
if in_lib:
|
||||||
|
for p in in_pile:
|
||||||
|
self.drop[p] = "hash-duplicate of CalibreBooks file"
|
||||||
|
if len(in_pile) > 1:
|
||||||
|
keep = min(in_pile, key=lambda p: (FORMAT_RANK.get(
|
||||||
|
os.path.splitext(p)[1].lstrip(".").lower(), 9), len(p), p))
|
||||||
|
for p in in_pile:
|
||||||
|
if p != keep:
|
||||||
|
self.drop[p] = "hash-duplicate inside the pile"
|
||||||
|
# R3/R4: identity against the library (same format only deletes)
|
||||||
|
cal_files = defaultdict(set) # identity -> formats in the library
|
||||||
|
for key, row in self.cal_identity.items():
|
||||||
|
cal_files[key] = set(f for f in row[6].split("; ") if f)
|
||||||
|
for sz, ext, p in self.pile_files:
|
||||||
|
if p in self.drop or ext not in BOOK_EXT:
|
||||||
|
continue
|
||||||
|
key = self.identity(p)
|
||||||
|
if key[0] and key in self.cal_identity:
|
||||||
|
if ext in cal_files[key]:
|
||||||
|
self.drop[p] = f"same identity+format already in Calibre ({key[0][:40]})"
|
||||||
|
else:
|
||||||
|
self.exc["pile holds a format the library lacks (add-format on ingest)"].append(
|
||||||
|
f"{p}\t{key[0][:50]}\t{self.cal_identity[key][0]}")
|
||||||
|
# R5: surviving files grouped by (identity, parent-ish book dir)
|
||||||
|
books = defaultdict(list)
|
||||||
|
for sz, ext, p in self.pile_files:
|
||||||
|
if p in self.drop:
|
||||||
|
continue
|
||||||
|
if ext not in BOOK_EXT:
|
||||||
|
self.moves.append((p, os.path.join(self.park, os.path.relpath(p, self.root)),
|
||||||
|
"non-book"))
|
||||||
|
continue
|
||||||
|
m = self.meta.get(p, {})
|
||||||
|
if m.get("err") and ext in ("epub",):
|
||||||
|
self.exc["unreadable container (park for a human)"].append(p)
|
||||||
|
self.moves.append((p, os.path.join(self.park, os.path.relpath(p, self.root)),
|
||||||
|
"broken-book"))
|
||||||
|
continue
|
||||||
|
key = self.identity(p)
|
||||||
|
books[key].append(p)
|
||||||
|
for key, ps in books.items():
|
||||||
|
first = min(ps, key=lambda p: (FORMAT_RANK.get(
|
||||||
|
os.path.splitext(p)[1].lstrip(".").lower(), 9), len(p), p))
|
||||||
|
m = self.meta.get(first, {})
|
||||||
|
self.ingest = {
|
||||||
|
"key": list(key),
|
||||||
|
"paths": sorted(ps),
|
||||||
|
"title": m.get("title") or stem_title(first),
|
||||||
|
"authors": m.get("authors") or "",
|
||||||
|
"ident": m.get("ident") or "",
|
||||||
|
}
|
||||||
|
self.ingest_list.append(self.ingest)
|
||||||
|
for p, why in self.drop.items():
|
||||||
|
self.moves.append((p, os.path.join(self.hold, os.path.relpath(p, self.root)), "dup"))
|
||||||
|
|
||||||
|
def write(self):
|
||||||
|
dirs = {os.path.dirname(d) for _, d, _ in self.moves}
|
||||||
|
all_dirs = set()
|
||||||
|
for d in dirs:
|
||||||
|
while d and d != self.root:
|
||||||
|
all_dirs.add(d)
|
||||||
|
d = os.path.dirname(d)
|
||||||
|
with open(self.args.plan, "w") as fh:
|
||||||
|
fh.write("action\tsrc\tdst\tkind\n")
|
||||||
|
for d in sorted(all_dirs):
|
||||||
|
fh.write(f"mkdir\t\t{d}\tcampaign\n")
|
||||||
|
for src, dst, kind in sorted(self.moves):
|
||||||
|
fh.write(f"move\t{src}\t{dst}\t{kind}\n")
|
||||||
|
with open(self.args.ingest, "w") as fh:
|
||||||
|
for b in self.ingest_list:
|
||||||
|
fh.write(json.dumps(b, ensure_ascii=False) + "\n")
|
||||||
|
with open(self.args.exceptions, "w") as fh:
|
||||||
|
for why in sorted(self.exc):
|
||||||
|
fh.write(f"# {why}\n")
|
||||||
|
for line in sorted(set(self.exc[why])):
|
||||||
|
fh.write(line + "\n")
|
||||||
|
fh.write("\n")
|
||||||
|
ndrop = sum(1 for _, _, k in self.moves if k == "dup")
|
||||||
|
print(f"pile books to ingest: {len(self.ingest_list)} "
|
||||||
|
f"files dropped as duplicates: {ndrop} "
|
||||||
|
f"non-book/broken moved out: {sum(1 for _, _, k in self.moves if k in ('non-book', 'broken-book'))}")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||||
|
ap.add_argument("--root", default="/mnt/entropy")
|
||||||
|
ap.add_argument("--meta", required=True)
|
||||||
|
ap.add_argument("--enum", required=True)
|
||||||
|
ap.add_argument("--calibre", required=True)
|
||||||
|
ap.add_argument("--dups", default=None)
|
||||||
|
ap.add_argument("--hold", default="/mnt/entropy/.media-campaign/dup-hold-ebooks-20261104")
|
||||||
|
ap.add_argument("--park", default="/mnt/entropy/NeedsProcessing/_from-ebooks-20261104")
|
||||||
|
ap.add_argument("--plan", required=True)
|
||||||
|
ap.add_argument("--ingest", required=True)
|
||||||
|
ap.add_argument("--exceptions", required=True)
|
||||||
|
args = ap.parse_args()
|
||||||
|
c = Campaign(args)
|
||||||
|
c.ingest_list = []
|
||||||
|
c.load()
|
||||||
|
c.run()
|
||||||
|
c.write()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+223
@@ -0,0 +1,223 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Trigger and poll Jellyfin's library scan — the step that makes a media move visible.
|
||||||
|
|
||||||
|
media-refresh-jellyfin.py trigger [--force]
|
||||||
|
media-refresh-jellyfin.py status [--after EPOCH]
|
||||||
|
media-refresh-jellyfin.py wait [--after EPOCH] [--interval S] [--require-quiet N]
|
||||||
|
|
||||||
|
`status` answers {"scan": true, "taskState": "Running", "recentMedia": 17}: the
|
||||||
|
Scan Media Library row plus the rows the database has taken in recently; with
|
||||||
|
`--after` (the trigger's epoch — a file mtime is already passed by an earlier scan) it
|
||||||
|
also answers `scannedAfter` and exits non-zero
|
||||||
|
while the last finished scan predates it. `wait` calls the library settled only
|
||||||
|
on the page's law: a sighting of the task Running — or a run that finished while
|
||||||
|
it watched, or a `--after` the task already passes — and THEN `--require-quiet`
|
||||||
|
consecutive reads with no scan and no writes. Both the scan and the metadata
|
||||||
|
fetch must be quiet, and the Running sighting is what stops a watcher from
|
||||||
|
declaring a pre-scan library settled (Dankserver/Jellyfin.md).
|
||||||
|
|
||||||
|
Laws this encodes:
|
||||||
|
* Auth is `Authorization: MediaBrowser Token="…"`. A bare X-Emby-Token answers
|
||||||
|
401 (measured on franksdankcloud 12.4.0), which reads exactly like a dead
|
||||||
|
key. The key comes from a mode-600 file and never rides argv or a URL:
|
||||||
|
/proc/<pid>/cmdline is world-readable.
|
||||||
|
* `POST /Library/Refresh` (204) is the ONLY door that queues work on 12.4, and
|
||||||
|
it walks every library. `POST /Items/{id}/Refresh` answers 204 and queues
|
||||||
|
NOTHING (measured twice, 2026-11-02) — so there is no --library here.
|
||||||
|
* A SECOND accepted POST cancels and restarts the scan already running, so
|
||||||
|
progress looks frozen while you hammer it. `trigger` therefore refuses while
|
||||||
|
the Scan Media Library task is not Idle; --force says you want the restart.
|
||||||
|
* 429 means busy. Waiting is the only response that helps, and it is bounded
|
||||||
|
so a wedged server ends the run with a named failure instead of a loop.
|
||||||
|
* `RefreshStatus` on /Library/VirtualFolders stays Idle through a live scan;
|
||||||
|
never poll it. The task table is the only honest read.
|
||||||
|
|
||||||
|
Defaults are this fleet's layout: http://127.0.0.1:8096, the key at
|
||||||
|
~/.config/secrets/jellyfin, and the database found by trying, in order, $JELLYFIN_DB,
|
||||||
|
~/.local/share/jellyfin/jellyfin.db and the quadlet's config bind.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
SCAN_TASK = "Scan Media Library"
|
||||||
|
IDLE = ("Idle",) # Running, Cancelling and Waiting all mean: not settled
|
||||||
|
DB_CANDIDATES = ("~/.local/share/jellyfin/jellyfin.db",
|
||||||
|
"/home/dank/Containers/jellyfin/config/data/jellyfin.db")
|
||||||
|
|
||||||
|
|
||||||
|
def _epoch(iso: str | None) -> float | None:
|
||||||
|
"""Jellyfin's `2026-11-04T18:31:33Z` EndTimeUtc as a unix epoch, or None."""
|
||||||
|
if not iso:
|
||||||
|
return None
|
||||||
|
for fmt in ("%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"):
|
||||||
|
try:
|
||||||
|
return datetime.strptime(iso, fmt).replace(tzinfo=timezone.utc).timestamp()
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
sys.exit(f"the server sent an unparsable EndTimeUtc: {iso!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def read_key(value: str) -> str:
|
||||||
|
path = pathlib.Path(value).expanduser()
|
||||||
|
try:
|
||||||
|
token = path.read_text().strip()
|
||||||
|
except OSError as exc:
|
||||||
|
sys.exit(f"no Jellyfin API key at {path} ({exc.strerror}); "
|
||||||
|
"that file is the credential this driver authenticates with")
|
||||||
|
if not token:
|
||||||
|
sys.exit(f"Jellyfin API key file {path} is empty")
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def request(base: str, token: str, method: str, path: str,
|
||||||
|
timeout: float = 30.0) -> tuple[int, bytes]:
|
||||||
|
req = urllib.request.Request(
|
||||||
|
base.rstrip("/") + path, method=method,
|
||||||
|
headers={"Authorization": f'MediaBrowser Token="{token}"'})
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||||
|
return resp.status, resp.read()
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
return exc.code, exc.read()
|
||||||
|
|
||||||
|
|
||||||
|
def scan_task(base: str, token: str) -> dict:
|
||||||
|
"""The one task row that means something, and when its last run finished."""
|
||||||
|
code, body = request(base, token, "GET", "/ScheduledTasks")
|
||||||
|
if code != 200:
|
||||||
|
sys.exit(f"GET /ScheduledTasks -> HTTP {code}: {body[:200]!r}")
|
||||||
|
for task in json.loads(body):
|
||||||
|
if task.get("Name") == SCAN_TASK:
|
||||||
|
state = task.get("State")
|
||||||
|
return {"state": state, "running": state not in IDLE,
|
||||||
|
"lastScan": _epoch(task.get("EndTimeUtc"))}
|
||||||
|
sys.exit(f"the server does not list a {SCAN_TASK!r} task; refusing to guess")
|
||||||
|
|
||||||
|
|
||||||
|
def find_db(value: str) -> pathlib.Path:
|
||||||
|
wanted = [value] if value != "-" else []
|
||||||
|
wanted += [p for p in (os.environ.get("JELLYFIN_DB"),) if p]
|
||||||
|
wanted += list(DB_CANDIDATES)
|
||||||
|
for candidate in wanted:
|
||||||
|
path = pathlib.Path(candidate).expanduser()
|
||||||
|
if path.is_file():
|
||||||
|
return path
|
||||||
|
sys.exit("no Jellyfin database found; tried: " + ", ".join(wanted))
|
||||||
|
|
||||||
|
|
||||||
|
def recent_media(db: str, window: int) -> int:
|
||||||
|
since = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time() - window * 60))
|
||||||
|
con = sqlite3.connect(f"file:{find_db(db)}?mode=ro", uri=True)
|
||||||
|
try:
|
||||||
|
return con.execute(
|
||||||
|
"SELECT COUNT(*) FROM BaseItems WHERE DateModified >= ?", (since,)
|
||||||
|
).fetchone()[0]
|
||||||
|
finally:
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||||
|
ap.add_argument("command", choices=["trigger", "status", "wait"])
|
||||||
|
ap.add_argument("--url", default="http://127.0.0.1:8096")
|
||||||
|
ap.add_argument("--key-file", default="~/.config/secrets/jellyfin")
|
||||||
|
ap.add_argument("--db", default="-", help="jellyfin.db; - searches the known homes")
|
||||||
|
ap.add_argument("--window", type=int, default=5,
|
||||||
|
help="minutes of DB writes that still count as recent")
|
||||||
|
ap.add_argument("--retries", type=int, default=3, help="extra attempts against a 429")
|
||||||
|
ap.add_argument("--retry-sleep", type=float, default=20.0)
|
||||||
|
ap.add_argument("--force", action="store_true",
|
||||||
|
help="trigger even though a scan is already running")
|
||||||
|
ap.add_argument("--interval", type=float, default=12.0, help="wait: seconds between reads")
|
||||||
|
ap.add_argument("--require-quiet", type=int, default=3,
|
||||||
|
help="wait: consecutive quiet reads before the library counts as settled")
|
||||||
|
ap.add_argument("--after", type=float, default=None,
|
||||||
|
help="status/wait: an epoch the scan must have finished after — pass "
|
||||||
|
"the trigger's, not a file mtime an earlier scan already passes")
|
||||||
|
ap.add_argument("--start-window", type=float, default=120.0,
|
||||||
|
help="wait: seconds to allow the scan to start before reporting "
|
||||||
|
"that the 204 queued nothing")
|
||||||
|
ap.add_argument("--timeout", type=float, default=1800.0, help="wait: seconds before giving up")
|
||||||
|
args = ap.parse_args()
|
||||||
|
token = read_key(args.key_file)
|
||||||
|
|
||||||
|
def state() -> dict:
|
||||||
|
task = scan_task(args.url, token)
|
||||||
|
out = {"scan": task["running"], "taskState": task["state"],
|
||||||
|
"lastScan": task["lastScan"],
|
||||||
|
"recentMedia": recent_media(args.db, args.window)}
|
||||||
|
if args.after is not None:
|
||||||
|
out["scannedAfter"] = bool(task["lastScan"] and task["lastScan"] >= args.after)
|
||||||
|
return out
|
||||||
|
|
||||||
|
if args.command == "status":
|
||||||
|
current = state()
|
||||||
|
print(json.dumps(current))
|
||||||
|
if args.after is not None and not current["scannedAfter"]:
|
||||||
|
sys.exit(f"the last scan finished before {args.after:.0f}; "
|
||||||
|
"a refresh is still owed")
|
||||||
|
return
|
||||||
|
|
||||||
|
if args.command == "wait":
|
||||||
|
start = time.monotonic()
|
||||||
|
ended_at_entry, quiet, started, first = None, 0, False, True
|
||||||
|
while True:
|
||||||
|
current = state()
|
||||||
|
if first:
|
||||||
|
first, ended_at_entry = False, current["lastScan"]
|
||||||
|
if (current["scan"]
|
||||||
|
or (current["lastScan"] and current["lastScan"] != ended_at_entry)
|
||||||
|
or current.get("scannedAfter")):
|
||||||
|
# The settle law: a Running sighting (or a run that finished while
|
||||||
|
# watching, or a --after the task already passes) must come BEFORE
|
||||||
|
# any claim that the library is quiet.
|
||||||
|
started = True
|
||||||
|
busy = current["scan"] or current["recentMedia"]
|
||||||
|
quiet = quiet + 1 if started and not busy else 0
|
||||||
|
print(json.dumps(current | {"quiet": quiet}), flush=True)
|
||||||
|
if quiet >= args.require_quiet:
|
||||||
|
return
|
||||||
|
elapsed = time.monotonic() - start
|
||||||
|
if not started and elapsed >= args.start_window:
|
||||||
|
sys.exit(f"{SCAN_TASK} never started: it read {current['taskState']!r} "
|
||||||
|
f"for {elapsed:.0f}s and the database took no writes. Either "
|
||||||
|
"the POST queued nothing — a POST that arrives while a scan "
|
||||||
|
"is already running cancels it, and every later 204 is "
|
||||||
|
"accepted and discarded, so restart the service and trigger "
|
||||||
|
"once — or the scan finished before this watch began, in "
|
||||||
|
"which case ask with --after <journal mtime> instead.")
|
||||||
|
if elapsed >= args.timeout:
|
||||||
|
sys.exit(f"still busy after {args.timeout:.0f}s: {current}")
|
||||||
|
time.sleep(args.interval)
|
||||||
|
|
||||||
|
if scan_task(args.url, token)["running"] and not args.force:
|
||||||
|
sys.exit(f"{SCAN_TASK} is already running on {args.url}: a second "
|
||||||
|
"POST /Library/Refresh cancels and restarts it, so the index "
|
||||||
|
"looks frozen while you wait. Poll `wait` instead of triggering "
|
||||||
|
"again (--force overrides).")
|
||||||
|
|
||||||
|
for attempt in range(args.retries + 1):
|
||||||
|
code, body = request(args.url, token, "POST", "/Library/Refresh")
|
||||||
|
if code == 204:
|
||||||
|
print("refresh accepted (204); the scan walks every library")
|
||||||
|
return
|
||||||
|
if code == 429:
|
||||||
|
if attempt < args.retries:
|
||||||
|
time.sleep(args.retry_sleep)
|
||||||
|
continue
|
||||||
|
sys.exit(f"Jellyfin is busy: {args.retries + 1} attempts all answered "
|
||||||
|
"429. Nothing was queued; run `wait`, then trigger again.")
|
||||||
|
sys.exit(f"POST /Library/Refresh -> HTTP {code}: {body[:200]!r}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+130
@@ -0,0 +1,130 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Trigger and poll ONE Plex library section — the step that makes a media move visible.
|
||||||
|
#
|
||||||
|
# media-refresh-plex.sh trigger --library KEY
|
||||||
|
# media-refresh-plex.sh status --library KEY [--after EPOCH]
|
||||||
|
#
|
||||||
|
# One section only: refreshing all five costs a full walk of each and buys
|
||||||
|
# nothing for a rename that touched one tree. And a rename on the array is
|
||||||
|
# invisible until the section is rescanned — the server is bound at the SSD
|
||||||
|
# metadata view, so an array-side move is not an event its watcher ever sees.
|
||||||
|
#
|
||||||
|
# Two host realities shape this script:
|
||||||
|
# * PMS's HTTPD closes without a single byte on a request that arrives from
|
||||||
|
# another netns (the host and the 10.89.x.x container address both read as
|
||||||
|
# "empty reply"), and it serves TLS to remote clients. So the request rides
|
||||||
|
# plain http://127.0.0.1:32400 INSIDE the container, shipped in as a file —
|
||||||
|
# a three-layer ssh/podman/sh quoting chain only produces parse errors.
|
||||||
|
# * The token is `PlexOnlineToken` in the container's Preferences.xml (not
|
||||||
|
# PlexHTTPToken, which a claimed server may not carry). It is read inside
|
||||||
|
# the container and used from a curl config there: never argv (any process
|
||||||
|
# on the host reads /proc/<pid>/cmdline), never a URL (proxies log URLs).
|
||||||
|
#
|
||||||
|
# Progress needs no token and no container: `section_locations.scanned_at` in
|
||||||
|
# the library database is the same epoch the API prints as `scannedAt`
|
||||||
|
# (verified 2026-11-04), so `status` reads the database and compares it against
|
||||||
|
# --after — which makes it a census of the last RECORDED scan, never a waiter: PMS
|
||||||
|
# stamps that epoch when it accepts the request, so the door answers `settled` in the
|
||||||
|
# same second its own `trigger` ran (measured 2026-11-04; no epoch survives it).
|
||||||
|
# `trigger` answering 000 means PMS is still warming — retry, never
|
||||||
|
# restart the service over it. Proof that a rename actually landed is
|
||||||
|
# scripts/media-verify.py replaying the journal; this script only moves the
|
||||||
|
# scanner along.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
CONTAINER=plex
|
||||||
|
URL="http://127.0.0.1:32400"
|
||||||
|
PREFS='/config/Library/Application Support/Plex Media Server/Preferences.xml'
|
||||||
|
DB="/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db"
|
||||||
|
SECTION=""
|
||||||
|
AFTER=""
|
||||||
|
TRIES=6
|
||||||
|
WARM_SLEEP=10
|
||||||
|
|
||||||
|
die() { echo "$*" >&2; exit 1; }
|
||||||
|
|
||||||
|
CMD="${1:-}"; shift || true
|
||||||
|
case "$CMD" in
|
||||||
|
trigger|status) ;;
|
||||||
|
*) die "usage: $(basename "$0") trigger|status --library KEY [--after EPOCH] [--container NAME] [--db PATH]" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--library) SECTION="${2:-}"; shift 2 ;;
|
||||||
|
--after) AFTER="${2:-}"; shift 2 ;;
|
||||||
|
--tries) TRIES="${2:-}"; shift 2 ;;
|
||||||
|
--warm-sleep) WARM_SLEEP="${2:-}"; shift 2 ;;
|
||||||
|
--container) CONTAINER="${2:-}"; shift 2 ;;
|
||||||
|
--url) URL="${2:-}"; shift 2 ;;
|
||||||
|
--prefs) PREFS="${2:-}"; shift 2 ;;
|
||||||
|
--db) DB="${2:-}"; shift 2 ;;
|
||||||
|
*) die "unknown argument: $1" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[ -n "$SECTION" ] || die "--library is required (the section id from GET /library/sections; Movies is 1)"
|
||||||
|
command -v python3 >/dev/null || die "python3 is required to read the library database"
|
||||||
|
|
||||||
|
if [ "$CMD" = "status" ]; then
|
||||||
|
[ -r "$DB" ] || die "no readable Plex library database at $DB"
|
||||||
|
python3 - "$DB" "$SECTION" "$AFTER" <<'PY'
|
||||||
|
import datetime, sqlite3, sys
|
||||||
|
|
||||||
|
db, section, after = sys.argv[1], int(sys.argv[2]), sys.argv[3]
|
||||||
|
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||||
|
rows = con.execute(
|
||||||
|
"SELECT root_path, available, scanned_at FROM section_locations "
|
||||||
|
"WHERE library_section_id = ? ORDER BY id", (section,)).fetchall()
|
||||||
|
if not rows:
|
||||||
|
sys.exit(f"section {section} has no location in {db}; is the id right?")
|
||||||
|
worst = min(r[2] or 0 for r in rows)
|
||||||
|
for root, available, at in rows:
|
||||||
|
iso = (datetime.datetime.fromtimestamp(at, datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
if at else "never")
|
||||||
|
print(f" available={available} scanned_at={iso} {root}")
|
||||||
|
print(f"section={section} scanned_at={worst}")
|
||||||
|
if after:
|
||||||
|
if worst < int(after):
|
||||||
|
sys.exit(f"section {section} was last scanned before {after}; a refresh is still owed")
|
||||||
|
PY
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
command -v podman >/dev/null || die "podman is required to reach PMS inside its container"
|
||||||
|
|
||||||
|
INNER="$(mktemp -t pxscan.XXXXXX)"
|
||||||
|
trap 'rm -f "$INNER"' EXIT
|
||||||
|
cat > "$INNER" <<'EOS'
|
||||||
|
# Runs INSIDE the plex container: $1 section key, $2 base URL, $3 Preferences.xml.
|
||||||
|
key="$1"; url="$2"; prefs="$3"
|
||||||
|
[ -r "$prefs" ] || { echo "no readable $prefs inside the container" >&2; exit 3; }
|
||||||
|
T=$(sed -n 's/.*PlexOnlineToken="\([^"]*\)".*/\1/p' "$prefs" | head -1)
|
||||||
|
[ -n "$T" ] || { echo "no PlexOnlineToken in $prefs" >&2; exit 3; }
|
||||||
|
cfg=$(mktemp)
|
||||||
|
( umask 077
|
||||||
|
{ printf 'header = "X-Plex-Token: %s"\n' "$T"
|
||||||
|
printf 'silent\nshow-error\nrequest = "POST"\n'
|
||||||
|
printf 'url = "%s/library/sections/%s/refresh"\n' "$url" "$key"; } > "$cfg" )
|
||||||
|
code=$(curl -K "$cfg" -o /dev/null -w '%{http_code}' || true)
|
||||||
|
shred -u "$cfg" 2>/dev/null || rm -f "$cfg"
|
||||||
|
[ "$code" = "401" ] && { echo "section $key: HTTP 401 — the token in $prefs is not accepted" >&2; exit 4; }
|
||||||
|
echo "$code"
|
||||||
|
EOS
|
||||||
|
|
||||||
|
attempt=1
|
||||||
|
while :; do
|
||||||
|
podman cp "$INNER" "$CONTAINER:/tmp/media-refresh-plex.sh" \
|
||||||
|
|| die "could not copy the runner into container $CONTAINER"
|
||||||
|
code="$(podman exec "$CONTAINER" sh /tmp/media-refresh-plex.sh "$SECTION" "$URL" "$PREFS" || true)"
|
||||||
|
podman exec "$CONTAINER" rm -f /tmp/media-refresh-plex.sh >/dev/null 2>&1 || true
|
||||||
|
case "$code" in
|
||||||
|
204|200) echo "refresh accepted ($code) for section $SECTION"; exit 0 ;;
|
||||||
|
000)
|
||||||
|
[ "$attempt" -ge "$TRIES" ] && die "PMS never answered after $TRIES attempts (000); it may still be warming — do not restart the service over it"
|
||||||
|
echo "section $SECTION: no answer yet ($attempt/$TRIES), PMS is warming" >&2 ;;
|
||||||
|
*) die "refresh of section $SECTION failed: HTTP ${code:-curl-error}" ;;
|
||||||
|
esac
|
||||||
|
attempt=$((attempt + 1))
|
||||||
|
sleep "$WARM_SLEEP"
|
||||||
|
done
|
||||||
@@ -222,5 +222,5 @@ harness bundle and measured the day of the flip:
|
|||||||
| Endpoint answers, synthesis fails | the engine: `podman healthcheck run frank-tts-engine`, then `systemctl --user status frank-tts-engine.service` |
|
| Endpoint answers, synthesis fails | the engine: `podman healthcheck run frank-tts-engine`, then `systemctl --user status frank-tts-engine.service` |
|
||||||
| First request of the day is ~0.6 s slower than the rest — or every name returns 400 | the warm pass did not run; it carries `-`, so it cannot fail the unit. `journalctl --user -u frank-tts-engine` for the warmer's lines, then `podman exec frank-tts-engine /app/warm-voices.sh` — the registry is memory-only (law 1 of [[Frank TTS Engine]]) |
|
| First request of the day is ~0.6 s slower than the rest — or every name returns 400 | the warm pass did not run; it carries `-`, so it cannot fail the unit. `journalctl --user -u frank-tts-engine` for the warmer's lines, then `podman exec frank-tts-engine /app/warm-voices.sh` — the registry is memory-only (law 1 of [[Frank TTS Engine]]) |
|
||||||
| The unit never appears after a deploy | the generator dropped it: `frank-tts-engine.network` is missing from `~/.config/containers/systemd/` |
|
| The unit never appears after a deploy | the generator dropped it: `frank-tts-engine.network` is missing from `~/.config/containers/systemd/` |
|
||||||
| A consumer rejects `wilwheaton` by listing 29 names (`af_heart … bm_fable`) and no request ever reaches this endpoint | that consumer is synthesizing **in the browser** with Kokoro.js, whose voice pack is fixed — OpenWebUI 0.11.4 ships it as the `browser-kokoro` TTS engine; see [[OpenWebUI]] |
|
| A consumer rejects `wilwheaton` by listing 29 names (`af_heart … bm_fable`) and no request ever reaches this endpoint | that consumer is synthesizing **in the browser** with Kokoro.js, whose voice pack is fixed. In OpenWebUI the `browser-kokoro` choice is that user's own `user.settings` row (`ui.audio.tts.engine`) and the SPA re-serves it on every load, so flipping the dropdown does not stick; see [[OpenWebUI]] |
|
||||||
---
|
---
|
||||||
@@ -21,7 +21,7 @@ https://github.com/open-webui/open-webui
|
|||||||
| Item | Value |
|
| Item | Value |
|
||||||
|------|-------|
|
|------|-------|
|
||||||
| Image | `ghcr.io/open-webui/open-webui:main` (verified from the live quadlet unit) |
|
| Image | `ghcr.io/open-webui/open-webui:main` (verified from the live quadlet unit) |
|
||||||
| Deployed build | `0.11.4` — `GET http://127.0.0.1:3005/api/version` → `{"version":"0.11.4"}` (2026-10-18). `:main` is the rolling image, not a stable pin, so the running build lags upstream: releases `0.12.0` (2026-10-10) through `0.12.3` (2026-10-16) exist and are not deployed (https://github.com/open-webui/open-webui/releases). The 2026-09-21 `0.11.4` security/access-control advisory is what pinned this era of the page. Imagegen endpoint is `POST /api/v1/images/generations` (the `comfyui` engine is still upstream). |
|
| Deployed build | `0.12.5` — `GET http://127.0.0.1:3005/api/version` → `{"version":"0.12.5"}` (re-measured 2026-11-04). The repo pin `@sha256:0b399a8b…` (`dankclaw/open-webui/quadlet/open-webui.container`, bumped 2026-10-30) is what makes that build reproducible; `:main` is a rolling tag, so re-measure `/api/version` instead of quoting a build from this page. Imagegen endpoint is `POST /api/v1/images/generations` (the `comfyui` engine is still upstream). |
|
||||||
| Port | 3005 (host) → 8080 (container). Port 3000 is Gitea's HTTP, so it is not usable. |
|
| Port | 3005 (host) → 8080 (container). Port 3000 is Gitea's HTTP, so it is not usable. |
|
||||||
| Data volume | `/home/dank/Containers/open-webui/data` → `/app/backend/data` |
|
| Data volume | `/home/dank/Containers/open-webui/data` → `/app/backend/data` |
|
||||||
| Backends | The picker's content is `webui.db` state, not repo state: as recorded on 2026-10-07 it exposes **one** model, `frank-zealot`, with `frank-dragoon`/`frank-archon` configured but disabled — *unverified 2026-10-18* (the DB is the authority and is not readable from the repo). What the repo does fix is the container's own OpenAI target: `OPENAI_API_BASE_URL=http://192.168.0.143:7770` (`dankclaw/open-webui/quadlet/open-webui.container:32`, same value in the live container env) — see [The LLM backends](#the-llm-backends) |
|
| Backends | The picker's content is `webui.db` state, not repo state: as recorded on 2026-10-07 it exposes **one** model, `frank-zealot`, with `frank-dragoon`/`frank-archon` configured but disabled — *unverified 2026-10-18* (the DB is the authority and is not readable from the repo). What the repo does fix is the container's own OpenAI target: `OPENAI_API_BASE_URL=http://192.168.0.143:7770` (`dankclaw/open-webui/quadlet/open-webui.container:32`, same value in the live container env) — see [The LLM backends](#the-llm-backends) |
|
||||||
@@ -109,6 +109,39 @@ The Ollama connection is disabled (`ollama.enable=false`).
|
|||||||
> connection here only hides it from this picker — the endpoint itself keeps
|
> connection here only hides it from this picker — the endpoint itself keeps
|
||||||
> serving every other client (the harness, Hermes, API consumers).
|
> serving every other client (the harness, Hermes, API consumers).
|
||||||
|
|
||||||
|
## Default model, descriptions, and the admin API door
|
||||||
|
|
||||||
|
**The default model is `ui.default_models`** — one row of the config table, a
|
||||||
|
comma-separated **string**, not a list. Live value `frank-archon` since
|
||||||
|
2026-11-04 (per Frank). The front end resolves the picker as *the browser's own
|
||||||
|
stored pick, else `config.default_models.split(",")`*, so the value decides what
|
||||||
|
a fresh profile or a cleared-storage session lands on; after a user picks
|
||||||
|
something themselves, their pick keeps winning. Verified 2026-11-04 in a clean
|
||||||
|
browser: signed in with `localStorage` empty → the pill reads
|
||||||
|
`Selected model: frank-archon`.
|
||||||
|
|
||||||
|
Two admin-token doors, both driven against loopback `:3005` — service-to-service
|
||||||
|
traffic never rides the public vhost:
|
||||||
|
|
||||||
|
- **Config:** `POST /api/v1/configs/import` with
|
||||||
|
`{"config":{"ui.default_models":"frank-archon"}}` — the general per-key setter
|
||||||
|
(no `PATCH /configs/{key}` route exists in 0.12.5; verified against
|
||||||
|
`routers/` + `main.py`), and it returns the whole config so the write reads
|
||||||
|
back in the same call.
|
||||||
|
- **A model row:** `POST /api/v1/models/model/update` with a full `ModelForm`
|
||||||
|
(`id`, `name`, `meta`, `params`, `is_active`). **`meta` and `params` REPLACE,
|
||||||
|
they do not merge:** read the row first, change one field, send everything
|
||||||
|
back — a partial payload silently drops `meta.capabilities` and the
|
||||||
|
profile/background images.
|
||||||
|
|
||||||
|
**An API key is not accepted everywhere.** Routes guarded by `get_human_user`
|
||||||
|
(`/api/v1/auths/`, `utils/auth.py`) answer **403 `A human session is required.`**
|
||||||
|
for any token whose `typ` is not `session` — that is not a dead key. The two
|
||||||
|
mutation doors above take `get_admin_user` / `get_verified_user`, which do accept
|
||||||
|
the key. Prove a key on `GET /api/v1/models` with **no trailing slash**:
|
||||||
|
`/api/v1/models/` falls through to the SPA and answers `200` + `text/html`, which
|
||||||
|
looks like success and parses as nothing.
|
||||||
|
|
||||||
## Container introspection (DB and backend python)
|
## Container introspection (DB and backend python)
|
||||||
|
|
||||||
- Working dir for imports: `/app/backend`; package name `open_webui`
|
- Working dir for imports: `/app/backend`; package name `open_webui`
|
||||||
@@ -163,14 +196,11 @@ account under Profile → API Keys). Anonymous web visitors get the `pending`
|
|||||||
role (`ui.default_user_role=pending`), which cannot chat. Verified
|
role (`ui.default_user_role=pending`), which cannot chat. Verified
|
||||||
2026-10-07 from loopback `:3005` and over the public vhost:
|
2026-10-07 from loopback `:3005` and over the public vhost:
|
||||||
`GET /api/v1/models/list`, `GET /api/v1/models` and
|
`GET /api/v1/models/list`, `GET /api/v1/models` and
|
||||||
`POST /api/chat/completions` all return **401** with no key (also with a
|
|
||||||
bogus bearer token); only `/api/config`, `/api/version` and `/health`
|
|
||||||
`POST /api/chat/completions` all return **401** with no key (also with a bogus bearer
|
`POST /api/chat/completions` all return **401** with no key (also with a bogus bearer
|
||||||
token); only `/api/config`, `/api/version` and `/health` answer publicly, and none of
|
token); only `/api/config`, `/api/version` and `/health` answer publicly, and none of
|
||||||
them expose the model list. An external program needs an account + API key — there is
|
them expose the model list. An external program needs an account + API key — there is
|
||||||
no keyless access. Re-verified 2026-10-18 from loopback: `/health` → `{"status":true}`
|
no keyless access. Re-measured 2026-11-04 from loopback: `/api/version` → `0.12.5`,
|
||||||
`/api/version` → 0.11.4, `/api/v1/models/list`, `/api/v1/models` and
|
and `/api/config` publishes
|
||||||
`POST /api/chat/completions` → 401; `/api/config` additionally publishes
|
|
||||||
`features.enable_login_form=false`, `features.enable_signup=false` and
|
`features.enable_login_form=false`, `features.enable_signup=false` and
|
||||||
`oauth.providers.oidc="authentik"` — the SSO-only posture, visible from outside.
|
`oauth.providers.oidc="authentik"` — the SSO-only posture, visible from outside.
|
||||||
|
|
||||||
@@ -219,18 +249,26 @@ browser-driven and only touches the two exempt vhosts (`ai.` and `auth.`).
|
|||||||
into the existing local account with the same email — the pre-SSO admin
|
into the existing local account with the same email — the pre-SSO admin
|
||||||
(frank8465@protonmail.com) keeps its data. `ENABLE_OAUTH_SIGNUP=true` lets
|
(frank8465@protonmail.com) keeps its data. `ENABLE_OAUTH_SIGNUP=true` lets
|
||||||
other `openwebui_users` create accounts on first login.
|
other `openwebui_users` create accounts on first login.
|
||||||
- **First login mints role `pending`.** `ui.default_user_role=pending` (config
|
- **First login mints the role from the claim, not from
|
||||||
DB, verified 2026-10-09), so a freshly-provisioned `openwebui_users` account
|
`ui.default_user_role`.** The quadlet sets `ENABLE_OAUTH_ROLE_MANAGEMENT=true`
|
||||||
completes SSO and then cannot chat (picker empty) until its role is flipped:
|
and `OAUTH_ROLES_CLAIM` defaults to `roles`, so the `roles: ["user"]` claim
|
||||||
`UPDATE "user" SET role='user' WHERE email='<user>@franksdank.cloud';` —
|
decides the role. Measured 2026-11-04: a freshly-provisioned
|
||||||
run with host `python3` over ssh against
|
`openwebui_users` account completed SSO in a clean browser and the row
|
||||||
|
appeared as `role='user'` — chat answered and Read Aloud synthesized, with no
|
||||||
|
manual step. `ui.default_user_role=pending` (config DB) is the fallback for an
|
||||||
|
account the claim does *not* describe, and 0.12.5's `get_user_role` **refuses
|
||||||
|
the login with 403** when role management is on and no roles claim is
|
||||||
|
readable — so "SSO succeeded but I cannot chat" points at a missing
|
||||||
|
`openwebui_users` group/entitlement, not at a row waiting to be flipped. Flip
|
||||||
|
only a row that is actually stuck:
|
||||||
|
`UPDATE "user" SET role='user' WHERE email='<user>@franksdank.cloud';` — run
|
||||||
|
with host `python3` over ssh against
|
||||||
`/home/dank/Containers/open-webui/data/webui.db` (no `sqlite3` CLI on
|
`/home/dank/Containers/open-webui/data/webui.db` (no `sqlite3` CLI on
|
||||||
Dankclaw; table is `user`, singular; no `active` column). No per-user model
|
Dankclaw; table is `user`, singular; no `active` column). The row does not
|
||||||
grant is needed: the picker model `frank-zealot` carries the global
|
exist until the account signs in, so census it after a first-login sighting
|
||||||
`access_grant` row `(model, frank-zealot, user, *, read)`, so any
|
(2026-10-09: five provisioned accounts had no row). No per-user model grant is
|
||||||
`role='user'` account sees it. Flip on FIRST-LOGIN SIGHTING — the row does
|
needed: the picker models carry the global `access_grant` row
|
||||||
not exist until the account signs in (2026-10-09: five provisioned accounts
|
`(model, <id>, user, *, read)`, so any `role='user'` account sees them.
|
||||||
had no row).
|
|
||||||
- **IdP reachability:** the container is on its own bridge (10.89.3.0/24) and
|
- **IdP reachability:** the container is on its own bridge (10.89.3.0/24) and
|
||||||
cannot reach `authentik-server:9000` by name. It reaches the IdP over the
|
cannot reach `authentik-server:9000` by name. It reaches the IdP over the
|
||||||
public URL `https://auth.franksdank.cloud` — Caddy publishes `0.0.0.0:443`
|
public URL `https://auth.franksdank.cloud` — Caddy publishes `0.0.0.0:443`
|
||||||
@@ -305,28 +343,58 @@ port is the rootless quadlet `frank-tts` (this repo:
|
|||||||
400 listing the voices that exist, never a silent fallback.
|
400 listing the voices that exist, never a silent fallback.
|
||||||
|
|
||||||
**The 29-voice `af_* … bm_fable` list is NOT frank-tts.** That enum belongs
|
**The 29-voice `af_* … bm_fable` list is NOT frank-tts.** That enum belongs
|
||||||
to **Kokoro.js running in the browser**: the deployed front end (0.11.4)
|
to **Kokoro.js running in the browser**: the deployed front end ships an
|
||||||
ships an in-page Kokoro implementation (chunk carrying a frozen voice table
|
in-page Kokoro implementation (frozen voice table plus `_validate_voice`,
|
||||||
`{af_heart:{name:"Heart",…}}` and `_validate_voice`, model
|
model `onnx-community/Kokoro-82M-v1.0-ONNX`, and a personal *Kokoro.js dtype*
|
||||||
`onnx-community/Kokoro-82M-v1.0-ONNX`, plus a personal *Kokoro.js dtype*
|
setting). Its engine select takes `""` (Default) or `browser-kokoro`; with
|
||||||
setting). Its engine select takes `""` (Default) or `browser-kokoro`; when
|
`browser-kokoro` the page synthesizes locally and rejects any voice outside
|
||||||
`browser-kokoro` is chosen the page synthesizes locally and rejects any
|
its fixed pack, so `wilwheaton` fails with `Voice: "wilwheaton" not found.
|
||||||
voice outside its fixed pack — so `wilwheaton` fails with
|
Should be one of: af_heart, …` **without a single request leaving the
|
||||||
`Voice: "wilwheaton" not found. Should be one of: af_heart, …` **without a
|
browser**. Kokoro cannot clone — fixed pack only.
|
||||||
single request reaching the server**. Personal audio settings are
|
|
||||||
browser: no `user.settings` row and no container env carries an audio
|
|
||||||
override, yet the toast still appears ⇒ the choice lives in that browser's
|
|
||||||
own storage. Fix in the UI: Settings → Audio → TTS Engine → **Default**,
|
|
||||||
which hands synthesis to the admin config (frank-tts); picking `af_heart`
|
|
||||||
instead keeps Kokoro, and Kokoro cannot clone — fixed pack only.
|
|
||||||
|
|
||||||
Verified 2026-10-18 from inside the container with the exact OpenWebUI body
|
**The switch is a DB row, not browser storage.** Personal audio settings are
|
||||||
(`{"model":"frank-tts","input":…,"voice":"narrator1",
|
the `settings` JSON column of table `user`, and in 0.12.x they nest under
|
||||||
"response_format":"mp3","speed":1.0}`) → **200, `audio/mpeg`, 7,629 B**,
|
`ui`: `ui.audio.tts.engine`, `.voice`, `.defaultVoice`. In 0.11.x the same
|
||||||
while `podman logs open-webui` recorded **zero** `audio/speech` requests for
|
keys sat at the top level (`audio.tts.*`), and the 2026-10-18 probe listed
|
||||||
the failing click — that contrast is the instrument: a server-path failure
|
only top-level keys, found no audio key, and concluded the choice lived in
|
||||||
leaves a log line and a frank-tts error, a `browser-kokoro` failure leaves
|
that browser's own storage — a probe-depth bug. The row carried the override
|
||||||
neither.
|
the whole way; the 2026-10-30 pin bump to 0.12.5 moved the keys, not the bug.
|
||||||
|
The SPA reads that row at load (`GET /api/v1/users/user/settings`) and posts
|
||||||
|
the **whole** blob back whenever any preference changes
|
||||||
|
(`POST …/settings/update`, body `{...settings}`). So the dropdown can fix it —
|
||||||
|
but a second tab still holding the old copy writes `browser-kokoro` straight
|
||||||
|
back, which is the loop that makes the toast return after it was "fixed". The
|
||||||
|
user row is the authority here; the admin `config` table was never the problem
|
||||||
|
(its `audio.tts.engine` has been `openai` throughout). Repair
|
||||||
|
`settings.ui.audio.tts` with the image's stdlib `sqlite3` (`mode=ro`; quote
|
||||||
|
identifiers, `user` and `group` are reserved words — there is no `sqlite3` CLI
|
||||||
|
in the image or on the host), then reload every tab of that account.
|
||||||
|
|
||||||
|
**Repair the engine *and* the voice.** With engine `""` the front end still
|
||||||
|
sends its stored voice in the request body, so a Kokoro-era `voice`/
|
||||||
|
`defaultVoice` survives the engine fix as frank-tts's own rejection —
|
||||||
|
measured 2026-11-04: `voice: "alloy"` → **400 `unknown voice 'alloy';
|
||||||
|
available voices: dr.kern, garrickhagon, jeffhays, vaati, wilwheaton`**. Set
|
||||||
|
`ui.audio.tts` to `{"engine": "", "voice": "wilwheaton", "defaultVoice":
|
||||||
|
"wilwheaton"}` and reload. Engine `""` means "the admin config decides", and
|
||||||
|
the handler fills `voice` from `audio.tts.voice` only when the body omits it.
|
||||||
|
|
||||||
|
**Two tells settle the case in one line:**
|
||||||
|
|
||||||
|
- a session that fetched `/wasm/ort-wasm-simd-threaded.jsep.wasm` loaded
|
||||||
|
onnxruntime, so Kokoro.js is running in that browser — the server path
|
||||||
|
never asks for that file;
|
||||||
|
- a server-path failure leaves a uvicorn request line for
|
||||||
|
`/api/v1/audio/speech` **and** a frank-tts log line; a `browser-kokoro`
|
||||||
|
failure leaves neither. uvicorn logs every request, so silence is evidence.
|
||||||
|
|
||||||
|
Verified 2026-11-04: the handler's own outbound shape
|
||||||
|
(`{"model":"frank-tts","input":…,"voice":"wilwheaton"}` with
|
||||||
|
`Authorization: Bearer none`, the configured `audio.tts.openai.api_key`) →
|
||||||
|
**200 `audio/wav`** — a body without `response_format` comes back WAV and
|
||||||
|
either MIME is browser-playable, so OpenWebUI accepts it. The full route as a
|
||||||
|
real user over the vhost, `POST /api/v1/audio/speech` → **200 `audio/mpeg`**,
|
||||||
|
with frank-tts logging `speech voice=wilwheaton`.
|
||||||
|
|
||||||
## Retired instances
|
## Retired instances
|
||||||
|
|
||||||
@@ -396,6 +464,21 @@ neither.
|
|||||||
app process env; the `.webui_secret_key` file happens to equal it — use it
|
app process env; the `.webui_secret_key` file happens to equal it — use it
|
||||||
explicitly, `podman exec -e`).
|
explicitly, `podman exec -e`).
|
||||||
|
|
||||||
|
- **2026-11-04 — decoration rows drifted again; default model set to
|
||||||
|
`frank-archon`.** Two payload swaps had gone by without anyone touching the
|
||||||
|
picker: `frank-dragoon`'s row still advertised halogen `0.16.2`, and
|
||||||
|
`frank-zealot`'s still said `build 10777` / `3×262,144` / *MTP off* — live is
|
||||||
|
`0.18.20`, `build 12118`, `3 × 200,704`, MTP armed. The prose is cosmetic; the
|
||||||
|
`params.context.length` values had **swapped**, and that one bites:
|
||||||
|
`frank-zealot` advertised 262,144 while the endpoint refuses anything past its
|
||||||
|
200,704 slot, so a long zealot chat rides the endpoint's 400 instead of
|
||||||
|
compacting, and `frank-dragoon` advertised 200,704 against a 262,144 ceiling,
|
||||||
|
giving up 61k tokens of window for nothing. Both rows re-keyed through the API
|
||||||
|
door in [Default model, descriptions, and the admin API door](#default-model-descriptions-and-the-admin-api-door),
|
||||||
|
descriptions rewritten from the env files, and `ui.default_models` set to
|
||||||
|
`frank-archon`. **Law: a model swap is not finished until that endpoint's
|
||||||
|
`model` row is re-keyed as well — no deployer step touches `webui.db`.**
|
||||||
|
|
||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
- [[Dankclaw Stack]]
|
- [[Dankclaw Stack]]
|
||||||
|
|||||||
@@ -188,7 +188,7 @@ Client access paths and the public-exposure policy (auth wall required before
|
|||||||
any public vhost; Plex-style claim tokens do not exist here — Jellyfin uses its
|
any public vhost; Plex-style claim tokens do not exist here — Jellyfin uses its
|
||||||
own users/API keys) follow the fleet [[Authentik]]/[[Anubis]] chain pattern.
|
own users/API keys) follow the fleet [[Authentik]]/[[Anubis]] chain pattern.
|
||||||
|
|
||||||
## Authentik LDAP authentication (2026-09-25)
|
## Authentik LDAP authentication (live since 2026-09-25, re-verified 2026-11-04)
|
||||||
|
|
||||||
Jellyfin has no native external-auth, so it authenticates users through the
|
Jellyfin has no native external-auth, so it authenticates users through the
|
||||||
**LDAP-Auth plugin** (`jellyfin-plugin-ldapauth`) against an [[Authentik]] LDAP
|
**LDAP-Auth plugin** (`jellyfin-plugin-ldapauth`) against an [[Authentik]] LDAP
|
||||||
@@ -204,17 +204,35 @@ user to verify the password.
|
|||||||
publishing `389`/`636` on the Dankclaw host for cross-host binds.
|
publishing `389`/`636` on the Dankclaw host for cross-host binds.
|
||||||
- Service account `svc-jellyfin-ldap` (DN
|
- Service account `svc-jellyfin-ldap` (DN
|
||||||
`cn=svc-jellyfin-ldap,ou=users,dc=franksdank,dc=cloud`).
|
`cn=svc-jellyfin-ldap,ou=users,dc=franksdank,dc=cloud`).
|
||||||
- Access groups `jellyfin_users` (search filter
|
- Access group `jellyfin_users`: the plugin's search filter is
|
||||||
`(memberOf=cn=jellyfin_users,ou=groups,dc=franksdank,dc=cloud)`) and
|
`(memberOf=cn=jellyfin_users,ou=groups,dc=franksdank,dc=cloud)`, so a user
|
||||||
`jellyfin_admins` (admin filter `(cn=jellyfin_admins)`). Add a user to
|
who is not in that group is **invisible** to the plugin and the login fails
|
||||||
`jellyfin_users` to grant them Jellyfin access; `jellyfin_admins` grants
|
as "Wrong username or password". Add members with
|
||||||
the Jellyfin admin role.
|
`Group.objects.get(name="jellyfin_users").users.add(user)` in `ak shell`
|
||||||
|
(directory data — the DB owns it, not this repo).
|
||||||
|
`jellyfin_admins` is **inert as configured** — see the admin-filter law.
|
||||||
|
|
||||||
**Jellyfin side** (plugin config, set via admin → Plugins → LDAP-Auth):
|
**Jellyfin side** (plugin config, set via admin → Plugins → LDAP-Auth):
|
||||||
- Server `192.168.0.143:389` (plain LDAP over the trusted LAN), Base DN
|
- Server `192.168.0.143:389` (plain LDAP over the trusted LAN), Base DN
|
||||||
`dc=franksdank,dc=cloud`, bind user = the service-account DN, search
|
`dc=franksdank,dc=cloud`, bind user = the service-account DN, search
|
||||||
attributes `uid,cn,mail,displayName`, uid attr `uid`, username attr `cn`
|
attributes `uid,cn,mail,displayName`, uid attr `uid`, username attr `cn`
|
||||||
`CreateUsersFromLdap=true`, all folders enabled.
|
`CreateUsersFromLdap=true`, all folders enabled.
|
||||||
|
- Admin base `ou=groups,dc=franksdank,dc=cloud`, admin filter
|
||||||
|
`(cn=jellyfin_admins)`, `EnableLdapAdminFilterMemberUid=false`.
|
||||||
|
|
||||||
|
**Law — the admin filter grants nothing unless it can return the user's own
|
||||||
|
DN.** With `EnableLdapAdminFilterMemberUid=false` the plugin searches
|
||||||
|
`LdapAdminBaseDn` with `LdapAdminFilter` and sets the admin permission only when
|
||||||
|
a returned entry's DN equals the bound user's DN
|
||||||
|
(`LDAP-Auth/LDAPAuthenticationProviderPlugin.cs`). A group-OU base returns the
|
||||||
|
*group* entry, whose DN never matches, so **no LDAP user can ever become a
|
||||||
|
Jellyfin admin** — measured 2026-11-04: the search answers
|
||||||
|
`cn=jellyfin_admins,ou=groups,…` for every login. Working shape if group-granted
|
||||||
|
admins are ever wanted: base `dc=franksdank,dc=cloud` +
|
||||||
|
`(memberOf=cn=jellyfin_admins,ou=groups,dc=franksdank,dc=cloud)`. Deliberate
|
||||||
|
today: the server's own admin account stays **local**, which is the break-glass
|
||||||
|
door when authentik or the outpost is down. A *malformed* admin filter is not
|
||||||
|
inert — the plugin throws for every login, so this knob is all-or-nothing.
|
||||||
|
|
||||||
**Gotcha — the bind credential is a PASSWORD, not a token.** The LDAP bind is
|
**Gotcha — the bind credential is a PASSWORD, not a token.** The LDAP bind is
|
||||||
verified by authentik's default authentication flow (standard-password
|
verified by authentik's default authentication flow (standard-password
|
||||||
@@ -224,10 +242,23 @@ password and the bind fails with `flow error password: Invalid`. The
|
|||||||
service-account password is stored only in the Jellyfin plugin config (the
|
service-account password is stored only in the Jellyfin plugin config (the
|
||||||
`LdapBindPassword` field) — it is never committed to this repo.
|
`LdapBindPassword` field) — it is never committed to this repo.
|
||||||
|
|
||||||
**Verification (2026-09-25):** `ldapsearch` bind + the `jellyfin_users`
|
**Verification (re-taken 2026-11-04, from Dankserver):** the container reaches
|
||||||
search both succeed from Dankclaw; a real user (`alex`, in `jellyfin_users`)
|
`192.168.0.143:389`; the service-account bind plus the `jellyfin_users` search
|
||||||
logged into the web UI with their authentik password and the plugin created
|
return every member; LDAP-Auth 23.0.0.0 loads on Jellyfin 12.4; the `alex` row
|
||||||
the matching Jellyfin user.
|
still carries
|
||||||
|
`AuthenticationProviderId=Jellyfin.Plugin.LDAP_Auth.LdapAuthenticationProviderPlugin`.
|
||||||
|
|
||||||
|
**Law — "the authentik login doesn't work" has two opposite causes; read the
|
||||||
|
Users table before touching authentik.** Copy `config/data/jellyfin.db` with its
|
||||||
|
`-wal`/`-shm` and read it read-only (never sqlite the live file):
|
||||||
|
`Users.AuthenticationProviderId` names the door — `…Users.DefaultAuthentication
|
||||||
|
Provider` = local password, `…LDAP_Auth.LdapAuthenticationProviderPlugin` =
|
||||||
|
authentik. Because the LDAP search is scoped to `jellyfin_users`, a name that
|
||||||
|
exists only as a local account is rejected by the plugin and Jellyfin then
|
||||||
|
silently falls back to the local provider: same UI message, opposite fix. The
|
||||||
|
instrument for "did a bind ever reach authentik" is the **event registry**, never
|
||||||
|
the outpost journal — see [[Authentik Outpost Reference]].
|
||||||
|
|
||||||
**Gotcha — the login endpoint changed in Jellyfin 12.1.** The legacy
|
**Gotcha — the login endpoint changed in Jellyfin 12.1.** The legacy
|
||||||
`POST /Users/Auth/Password` route (and its `LegacyEmbyRouteRewriteMiddleware`)
|
`POST /Users/Auth/Password` route (and its `LegacyEmbyRouteRewriteMiddleware`)
|
||||||
was removed; probing it now returns a silent 401 with no plugin log, which
|
was removed; probing it now returns a silent 401 with no plugin log, which
|
||||||
@@ -309,33 +340,40 @@ scripts/jellyfin-providers.sh verify # plugin + key LENGTHS, never key bytes
|
|||||||
|
|
||||||
## Triggering a scan
|
## Triggering a scan
|
||||||
|
|
||||||
The scan key lives in `~/.config/secrets/jellyfin` (mode 600; the same value is
|
Run the driver from the deploy checkout: `scripts/media-refresh-jellyfin.py
|
||||||
`JELLYFIN_API_KEY` in the ignored `dankserver/.env`). It rides a mode-600 curl
|
trigger`, then `… wait`. It calls the library settled only on the rule below — the
|
||||||
config, never a header argument, in the `MediaBrowser` form: 12.4 authenticates
|
task seen `Running` (or a run that finished while it watched) first, then
|
||||||
only `Authorization: MediaBrowser Token="…"` and refuses a bare `X-Emby-Token`
|
`--require-quiet` reads with no scan and no writes — and refuses by name when the
|
||||||
for every key, valid or not. Before believing a 401, prove the key against the
|
scan never started, because a watcher that saw no `Running` would otherwise report
|
||||||
server's own table — `select count(*) from ApiKeys where AccessToken = ?` on
|
the pre-scan library as the fresh one. `… status --after <the trigger's epoch>` is the
|
||||||
`jellyfin.db` opened `mode=ro&immutable=1`. The backslashes below matter:
|
"has the scan reached my moves" answer, read from the task's `EndTimeUtc` — and that
|
||||||
|
timestamp only advances when a scan *finishes*, so the epoch must be one your trigger
|
||||||
|
could not have followed: a journal mtime is already passed by whatever scan ran before
|
||||||
|
you asked, and bare `status` cannot tell that walk from yours, while `wait` refuses such
|
||||||
|
an epoch by name. The Plex door is worse still — [[Plex]].
|
||||||
|
|
||||||
```bash
|
It carries every law below. Hand-written, the door is a `curl -K` on a mode-600
|
||||||
umask 077
|
config: the scan key lives in `~/.config/secrets/jellyfin` (mode 600; the same
|
||||||
{ printf 'header = "Authorization: MediaBrowser Token=\\"%s\\""\n' \
|
value is `JELLYFIN_API_KEY` in the ignored `dankserver/.env`) and rides as
|
||||||
"$(cat ~/.config/secrets/jellyfin)"
|
`Authorization: MediaBrowser Token="…"` — 12.4 refuses a bare `X-Emby-Token` for
|
||||||
echo 'url = "http://127.0.0.1:8096/Library/Refresh"'
|
every key, valid or not — never as a header argument, because
|
||||||
echo 'request = "POST"'; } >/tmp/jf.hdr
|
`/proc/<pid>/cmdline` is world-readable. `recentMedia` counts
|
||||||
curl -s -K /tmp/jf.hdr -o /dev/null -w "jellyfin scan: %{http_code}\n"
|
`BaseItems.DateModified` rows in `jellyfin.db`, which on this host sits in the
|
||||||
shred -u /tmp/jf.hdr
|
quadlet's config bind at `/home/dank/Containers/jellyfin/config/data/`, not the
|
||||||
```
|
stock `~/.local/share/jellyfin`.
|
||||||
|
|
||||||
`204` is acceptance; anything else queued no scan, so re-issue the identical
|
`204` is acceptance; anything else queued no scan, so re-issue the identical
|
||||||
request instead of diagnosing. A `401` from a key that answers `200` on a
|
request instead of diagnosing. A `401` from a key that answers `200` on a
|
||||||
hand-typed request is this block's quoting, not the server: bash `printf`
|
hand-typed request is this block's quoting, not the server: bash `printf`
|
||||||
collapses `\"` to `"`, curl ends the quoted value at that quote, and Jellyfin
|
collapses `\"` to `"`, curl ends the quoted value at that quote, and Jellyfin
|
||||||
receives `Authorization: MediaBrowser Token=` with nothing after it (`\\"` answers
|
receives `Authorization: MediaBrowser Token=` with nothing after it (`\\"` answers
|
||||||
204, `\"` answers 401 — both run on this host).
|
204, `\"` answers 401 — both run on this host). Before believing a 401, prove the
|
||||||
Reading the progress with the same config trick takes **one** URL: a config `url`
|
key against the server's own table — `select count(*) from ApiKeys where
|
||||||
line *plus* a command-line URL makes curl perform both transfers, and the second
|
AccessToken = ?` on `jellyfin.db` opened plain `mode=ro` (`immutable=1` only as
|
||||||
dumps the whole `ScheduledTasks` body onto your stdout.
|
the fallback for a database whose `-shm` cannot be read; it skips the write-ahead
|
||||||
|
log, see [[Media Relayout]]). Reading progress with the same config trick takes
|
||||||
|
**one** URL: a config `url` line *plus* a command-line URL makes curl perform both
|
||||||
|
transfers, and the second dumps the whole `ScheduledTasks` body onto your stdout.
|
||||||
|
|
||||||
One accepted refresh is the whole procedure. A second one while a scan runs moves
|
One accepted refresh is the whole procedure. A second one while a scan runs moves
|
||||||
`Scan Media Library` to `Cancelling`, and once the runner holds `Cancelling` every
|
`Scan Media Library` to `Cancelling`, and once the runner holds `Cancelling` every
|
||||||
|
|||||||
@@ -151,6 +151,18 @@ Each of these is a file that would have been destroyed without it:
|
|||||||
- Media containers only. Grouping every extension buries the report in subtitle
|
- Media containers only. Grouping every extension buries the report in subtitle
|
||||||
and cover pairs — and a duration guard that appears to fire on 720 groups is
|
and cover pairs — and a duration guard that appears to fire on 720 groups is
|
||||||
counting subtitles, not protecting films.
|
counting subtitles, not protecting films.
|
||||||
|
- **A short file named after a film is promotion, not a movie.** Thirty-two files
|
||||||
|
in `Movies` — 31 clips and the `.nfo` filed beside one of them — carried real
|
||||||
|
titles (`The Descent (2005) - SD.mkv`) and were 8–20-second h264 video at
|
||||||
|
640×360, 1024×576 or 1280×720, 49 MiB all together —
|
||||||
|
25 of them in a `_release-group watermarks/` folder and the rest at the library
|
||||||
|
root or inside one film's folder. No duplicate screen ever catches them: nothing
|
||||||
|
shares their size, and the feature-length guard above makes them invisible to the
|
||||||
|
similar-title pass too. They were held as `.AGENTTRASH/20261104-promo-clips`
|
||||||
|
(journal `/mnt/entropy/.media-campaign/promo-hold.jsonl`, which survives) and purged
|
||||||
|
the same evening, once both databases read `REAL: 0` against that journal.
|
||||||
|
The census that finds the next batch is a duration floor, not a hash: a video under
|
||||||
|
a minute on a name that reads like a feature is advertising.
|
||||||
|
|
||||||
## The hold
|
## The hold
|
||||||
|
|
||||||
@@ -179,21 +191,57 @@ keeper is missing — either one drops the whole group rather than half-applying
|
|||||||
era, not this pass — read the mtimes before blaming the prune.
|
era, not this pass — read the mtimes before blaming the prune.
|
||||||
- The 2026-11-01 pass held 226 files (~1.1 TiB) in
|
- The 2026-11-01 pass held 226 files (~1.1 TiB) in
|
||||||
`/mnt/entropy/.AGENTTRASH/20261101-duplicate-prune`, journals
|
`/mnt/entropy/.AGENTTRASH/20261101-duplicate-prune`, journals
|
||||||
`dup-prune-exact.jsonl` and `dup-prune-similar.jsonl`. Purge pending Frank's word.
|
`dup-prune-exact.jsonl` and `dup-prune-similar.jsonl`.
|
||||||
- The 2026-11-02 TV pass holds 220 files (54 GiB) in
|
- The 2026-11-02 TV pass held 220 files (54 GiB) in
|
||||||
`/mnt/entropy/.AGENTTRASH/20261102-tv-duplicates` — plan `tv-dupes-plan.tsv`,
|
`/mnt/entropy/.AGENTTRASH/20261102-tv-duplicates` — plan `tv-dupes-plan.tsv`,
|
||||||
journal `tv-dupes.jsonl`, 220 moves / 16 hold folders / 7 husks swept — and the
|
journal `tv-dupes.jsonl`, 220 moves / 16 hold folders / 7 husks swept — and the
|
||||||
same day's TVShows detritus pass holds 109 files (552 KiB) in
|
same day's TVShows detritus pass staged 109 files (552 KiB) in
|
||||||
`/mnt/entropy/.AGENTTRASH/20261102-tvshows-detritus` with plan
|
`/mnt/entropy/.AGENTTRASH/20261102-tvshows-detritus` with plan
|
||||||
`tvshows-detritus-plan.tsv` and journal `tvshows-detritus.jsonl`. Both purges
|
`tvshows-detritus-plan.tsv` and journal `tvshows-detritus.jsonl`. All four
|
||||||
pending Frank's word, beside the 2026-11-01 hold above. All four journals and
|
journals and plans live at the `entropy` root, not `/tmp`.
|
||||||
plans live at the `entropy` root, not `/tmp`.
|
- **Purged 2026-11-04, on Frank's word, after the gate below:** the duplicate
|
||||||
- **Re-verify each survivor by size, not by the recorded path, before purging.**
|
prune, the TV duplicate pass, the Ghibli duplicates (6 files — both `Nausicaä`
|
||||||
Detection artifacts and plans key on paths, and a folder renamed by an
|
copies among them) and the 2026-11-04 promo-clip hold left the array trash,
|
||||||
interactive session ([[Entropy Volume]], *Write doors*) makes its keeper look
|
~1.2 TB freed (the array went 218 GB → 1.4 TB available). The journals survive as
|
||||||
deleted while it sits untouched under the new name. The 2026-11-01 audit hit
|
the only record of where a purged file came from; rollback after a purge is
|
||||||
three such groups: every keeper was present, same bytes, different folder. A
|
impossible, which is why the gate runs first.
|
||||||
purge decided from the plan's paths alone would have reported survivors gone.
|
|
||||||
|
### The purge gate: a carrier for every held file
|
||||||
|
|
||||||
|
`REAL: 0` in both databases says only that no row points at a moved path; it says
|
||||||
|
nothing about whether the film survived. So a purge waits on a stronger proof:
|
||||||
|
**every held file has a same-work carrier in the live library at >= its size,
|
||||||
|
re-derived at purge time**, never copied from the plan. The 2026-11-04 purge re-found
|
||||||
|
all 452 held files this way; the first pass of the gate could not place 142 of them,
|
||||||
|
and every one of those was a matcher gap, not a lost film. What the re-derivation has
|
||||||
|
to absorb:
|
||||||
|
|
||||||
|
- **The layout moves under the plan.** The relayout renames folders to
|
||||||
|
`Title (YYYY)` and `[tvdbid-N]/Season N` and renames the keeper files too, so the
|
||||||
|
paths *and* the sizes a plan recorded no longer describe the library. Keys are work
|
||||||
|
identity — normalized title+year for a film, `(S,E)` for a TV pack — never a path.
|
||||||
|
- **Fold accents, then compare.** The library holds `Nausicaä`; a plan, a
|
||||||
|
`find -ipath "*Nausicaa*"`, and an ASCII substring search all miss it. NFKD-fold and
|
||||||
|
drop combining marks, or a real keeper reads absent for two rounds and the file
|
||||||
|
beside it starts to look like an orphan.
|
||||||
|
- **The carrier is often another container, and usually larger.** A held `.mp4` can be
|
||||||
|
kept as the sibling `.mkv`; keep-larger means the keeper is frequently far bigger (a
|
||||||
|
377 MB held episode matched a 501 MB keeper; a 1.6 GiB film stands beside a 34 GiB
|
||||||
|
BDRemux of itself in another folder). "Same folder, same bytes" reports survivors as
|
||||||
|
lost — it is the wrong test.
|
||||||
|
- **An exact twin can sit in a sibling folder**, so "no same-size file in the folder"
|
||||||
|
needs an array-wide name search before it means anything.
|
||||||
|
|
||||||
|
### Running the purge
|
||||||
|
|
||||||
|
`scripts/purge-agenttrash.sh` is the only door — never `rm`. It acts on exactly one
|
||||||
|
object: a root's own `.AGENTTRASH`. A partial purge is therefore **staged, not
|
||||||
|
filtered**: move the approved buckets into a scratch root
|
||||||
|
(`.PURGE-<date>/.AGENTTRASH/`) and point the purger at that root, and park anything
|
||||||
|
not approved at the array root as `.HOLD-<date>-*`, out of its reach. Dry-run first —
|
||||||
|
the plan must print exactly one `plan empty` row, and it is the same row `--execute`
|
||||||
|
acts on — and send `--log` into `.media-campaign/`, because `/tmp` is not a record.
|
||||||
|
Then read `df`: a purge is the only media act that moves the free-space number.
|
||||||
|
|
||||||
## After the pass
|
## After the pass
|
||||||
|
|
||||||
|
|||||||
@@ -99,12 +99,17 @@ interrupted apply still replays the batch and resumes with nothing lost. Batch 1
|
|||||||
is the old behaviour; the only reason to name it is debugging, because on the
|
is the old behaviour; the only reason to name it is debugging, because on the
|
||||||
spinning array it spends most of a large apply's wall time in fsyncs.
|
spinning array it spends most of a large apply's wall time in fsyncs.
|
||||||
|
|
||||||
After it lands, both servers need a scan or the change is invisible. [[Plex]] and
|
After it lands, both servers need a scan or the change is invisible. The two
|
||||||
[[Jellyfin]] own the exact requests; two laws hold at this step: `204` is
|
drivers are the doors — `scripts/media-refresh-jellyfin.py trigger` then `… wait`,
|
||||||
acceptance and anything else queued nothing — re-issue the identical request
|
and only after it settles, `scripts/media-refresh-plex.sh trigger --library <key>`
|
||||||
rather than diagnosing — and ONE accepted refresh is the whole procedure, because
|
with `… status --library <key> --after $(stat -c %Y <journal>)` as the poll. Run
|
||||||
a second while a scan runs moves `Scan Media Library` to `Cancelling` and every
|
them in that order and never together: Jellyfin's only door walks every library,
|
||||||
later `204` is accepted and then discarded.
|
every Plex section forks its own scanner walk, and the array is the bottleneck
|
||||||
|
both servers share. [[Plex]] and [[Jellyfin]] own the laws behind the drivers; two
|
||||||
|
of them decide this step: `204` is acceptance and anything else queued nothing —
|
||||||
|
re-issue the identical request rather than diagnosing — and ONE accepted refresh is
|
||||||
|
the whole procedure, because a second while a scan runs moves `Scan Media Library`
|
||||||
|
to `Cancelling` and every later `204` is accepted and then discarded.
|
||||||
|
|
||||||
### Proving the move landed
|
### Proving the move landed
|
||||||
|
|
||||||
|
|||||||
@@ -106,43 +106,57 @@ root runs as the `dank` user, and the deployer adopts the data root with
|
|||||||
## Triggering a scan
|
## Triggering a scan
|
||||||
|
|
||||||
A rename on disk is invisible until the section is rescanned, and a moved file is
|
A rename on disk is invisible until the section is rescanned, and a moved file is
|
||||||
a `Couldn't find the file to stream` 404 until that scan passes it. Run this from
|
a `Couldn't find the file to stream` 404 until that scan passes it. Run the
|
||||||
the host shell through `podman exec`, never from a neighbour container: PMS's
|
driver from the host shell; the section keys are `1` Movies, `3` TV Shows,
|
||||||
HTTPD closes without a single byte on any request that arrives from another
|
`5` Audiobooks, `6` Music, `11` VODs & Live Events:
|
||||||
container's netns (10.89.x.x reads as "empty reply", not as a 401). The token
|
|
||||||
rides stdin, never argv, where any process on the host could read it out of
|
|
||||||
`/proc/<pid>/cmdline`; it lives in `Preferences.xml` inside the RW data bind.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
umask 077
|
scripts/media-refresh-plex.sh trigger --library 1
|
||||||
podman exec plex sh -c "sed -n 's/.*PlexOnlineToken=\"\\([^\"]*\\)\".*/\\1/p' \
|
python3 scripts/media-verify.py --db-kind plex \
|
||||||
'/config/Library/Application Support/Plex Media Server/Preferences.xml'" >/tmp/px.tok
|
--journal /mnt/entropy/.media-campaign/<journal>.jsonl # the real proof
|
||||||
cat >/tmp/pxscan.sh <<'EOS'
|
|
||||||
read -r T
|
|
||||||
cfg=$(mktemp)
|
|
||||||
mk() { { printf 'header = "X-Plex-Token: %s"\nheader = "Accept: application/json"\n' "$T"
|
|
||||||
printf 'url = "%s"\n' "$1"; } > "$cfg"; }
|
|
||||||
mk "http://127.0.0.1:32400/library/sections"
|
|
||||||
for k in $(curl -s -K "$cfg" | grep -o '"key":"[0-9]*"' | cut -d\" -f4); do
|
|
||||||
mk "http://127.0.0.1:32400/library/sections/$k/refresh"
|
|
||||||
printf 'request = "POST"\n' >> "$cfg"
|
|
||||||
echo "plex section $k: $(curl -s -o /dev/null -w '%{http_code}' -K "$cfg")"
|
|
||||||
done
|
|
||||||
shred -u "$cfg"
|
|
||||||
EOS
|
|
||||||
podman cp /tmp/pxscan.sh plex:/tmp/pxscan.sh
|
|
||||||
podman exec -i plex sh /tmp/pxscan.sh </tmp/px.tok
|
|
||||||
shred -u /tmp/px.tok
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Every section answers `204` — or `200` with a Directory body when
|
Exactly ONE section: a refresh costs a full walk of that tree, and refreshing all
|
||||||
`Accept: application/json` is sent. A refresh fired while PMS is still warming
|
five for a rename that touched one buys nothing. `trigger` answers
|
||||||
answers `000` instead (the empty reply of gotcha 4, arriving mid-request): retry
|
`refresh accepted (204)` — `200` when a body is requested — and `000` means PMS is
|
||||||
until it answers and never restart the service over it. Inside the quoted
|
still warming, which the driver retries out loud; a service restart is never the
|
||||||
`printf` format a backslash-newline is LITERAL, not a continuation, so `mk`
|
fix.
|
||||||
writes two `printf`s: one glued line and one stray backslash is a config file
|
|
||||||
curl rejects, and the failure prints as `option -K: is unknown`, never as a bad
|
**`status` is a census, not a waiter: no epoch makes it watch a scan.** It reads
|
||||||
config line.
|
`section_locations.scanned_at` — the epoch of the last scan PMS has *recorded* for the
|
||||||
|
section, the same value `GET /library/sections` prints as `scannedAt` (verified
|
||||||
|
2026-11-04) — and compares it against `--after`. Measured twice on 2026-11-04: a chain
|
||||||
|
fell through its wait loop in the same second its own trigger was accepted, and a direct
|
||||||
|
read 30 minutes later still answered `settled` off that one stamp
|
||||||
|
(`2026-11-04T19:46:59Z`, for a request accepted at `19:46:58Z`) while the walk ran. A
|
||||||
|
journal's mtime fails, and so does the trigger's own epoch — the request satisfies the
|
||||||
|
very door it was asked to wait for. The failure direction is a false *success*, never a
|
||||||
|
false alarm, which is what makes it worth the warning: ask `status` whether a scan got
|
||||||
|
recorded, and never let a chain depend on it.
|
||||||
|
|
||||||
|
**The proof that a scan did its work is the journal replayed against the database:**
|
||||||
|
`scripts/media-verify.py --db-kind plex --journal <journal>` names every row that still
|
||||||
|
points at a source path (`still at src`) and every entry whose file is physically still
|
||||||
|
there (`REAL`). Both at zero is what gates a purge, and it needs no settle signal at all
|
||||||
|
— a scan can clear rows but cannot invent one. Jellyfin's door has no such trap:
|
||||||
|
`scripts/media-refresh-jellyfin.py wait` watches the scan task itself and refuses by
|
||||||
|
name when no task reaches `Running`.
|
||||||
|
|
||||||
|
The driver ships its runner into the container with `podman cp` rather than
|
||||||
|
quoting a command through `podman exec sh -c`, because PMS's HTTPD closes without
|
||||||
|
a single byte on a request that arrives from another netns — the host address and
|
||||||
|
the `10.89.x.x` container address both read as "empty reply", never as a 401 —
|
||||||
|
and because a three-layer `ssh → podman exec → sh -c` quote chain dies on a parse
|
||||||
|
error long before it reaches the server. The request therefore rides plain
|
||||||
|
`http://127.0.0.1:32400` from inside the container, where insecure connections
|
||||||
|
are allowed (gotcha 5). The credential is `PlexOnlineToken` in the container's
|
||||||
|
`Preferences.xml` — NOT `PlexHTTPToken`, which a claimed server need not carry —
|
||||||
|
read inside the container into a mode-600 curl config there, so it never enters
|
||||||
|
argv (`/proc/<pid>/cmdline` is world-readable) and never a URL (proxies log
|
||||||
|
URLs). Hand-writing that config has one more trap: inside a quoted `printf`
|
||||||
|
format a backslash-newline is LITERAL, so a two-line string becomes one glued
|
||||||
|
line plus a stray backslash, and curl answers `option -K: is unknown` — never
|
||||||
|
"bad config line".
|
||||||
|
|
||||||
Proof that a rename landed is not a scan response: `scripts/media-verify.py`
|
Proof that a rename landed is not a scan response: `scripts/media-verify.py`
|
||||||
replays the run's journal against the database — the only read that
|
replays the run's journal against the database — the only read that
|
||||||
|
|||||||
@@ -148,36 +148,46 @@ Read from the live database (read-only `SELECT`s against the
|
|||||||
| Object | Value |
|
| Object | Value |
|
||||||
|---|---|
|
|---|---|
|
||||||
| LDAP provider | `Jellyfin LDAP` (pk 11), base DN `dc=franksdank,dc=cloud`, `search_mode=direct`, `bind_mode=direct` |
|
| LDAP provider | `Jellyfin LDAP` (pk 11), base DN `dc=franksdank,dc=cloud`, `search_mode=direct`, `bind_mode=direct` |
|
||||||
| Bind DN | `cn=svc-jellyfin-ldap,ou=users,dc=franksdank,dc=cloud` *(unverified 2026-10-18: base DN confirmed in the DB, the `ou=users` path was not probed — an LDAP bind needs the bind secret)* |
|
| Bind DN | `cn=svc-jellyfin-ldap,ou=users,dc=franksdank,dc=cloud` — **verified by a live bind 2026-11-04** (the bind and the `memberOf` search both succeed, so the `ou=users` path is right) |
|
||||||
| Service account | `svc-jellyfin-ldap` (user pk 8). **It is not passwordless**: its password field is a real hash, not Django's `!`-prefixed unusable marker (`starts_with(password,'!')` → `f`) — an earlier revision's "passwordless/unusable" claim is wrong, which matters because an account with a usable password can sign in through the ordinary password flow *(unverified 2026-10-18: whether anyone holds that password)* |
|
| Service account | `svc-jellyfin-ldap` (user pk 8). It has a usable password and authenticates through the default flow — that password is held by Jellyfin in the plugin config (`LdapBindPassword`) and is what the bind actually uses (live bind 2026-11-04) |
|
||||||
| Bind token | exactly one row for pk 8, **`intent=app_password`**, non-expiring — an earlier revision recorded `intent=verification`; the live value is `app_password` |
|
| Bind token | exactly one row for pk 8, **`intent=app_password`**, non-expiring — and **unused by Jellyfin**, which has no token field (see the rule below) |
|
||||||
| RBAC role | `svc-jellyfin-ldap-search`, carrying exactly one object-level permission `authentik_providers_ldap.search_full_directory` (verified via `guardian_roleobjectpermission`) |
|
| RBAC role | `svc-jellyfin-ldap-search`, carrying exactly one object-level permission `authentik_providers_ldap.search_full_directory` (verified via `guardian_roleobjectpermission`) |
|
||||||
| Groups | `jellyfin_users`, `jellyfin_admins` (both exist in `authentik_core_group`) |
|
| Groups | `jellyfin_users`, `jellyfin_admins` (both exist in `authentik_core_group`) |
|
||||||
| Application | name `Jellyfin`, **slug `jellyfin-ldap`** (an earlier revision said slug `jellyfin`, which does not exist) |
|
| Application | name `Jellyfin`, **slug `jellyfin-ldap`** (an earlier revision said slug `jellyfin`, which does not exist) |
|
||||||
|
|
||||||
Hard-won rules (the first is now contested by the live database):
|
Hard-won rules:
|
||||||
|
|
||||||
- **Token intent: contested, do not trust either value blindly.** An earlier
|
- **Token intent is moot for Jellyfin.** The earlier argument (a bind token with
|
||||||
revision asserted that a bind token with `intent=app_password` is rejected by
|
`intent=app_password` is rejected by the flow's password stage, only
|
||||||
the flow's password stage with `Invalid password` and only
|
`intent=verification` passes) never applied here: LDAP-Auth sends the service
|
||||||
`intent=verification` passes. The live token for `svc-jellyfin-ldap` is
|
account's **password** from `LdapBindPassword`, not a token. A live bind plus a
|
||||||
`app_password` (`authentik_core_token`, 2026-10-18), so either the rule is
|
`memberOf=cn=jellyfin_users` search (11 entries) succeeded from Dankserver on
|
||||||
wrong or the current Jellyfin bind path is not that token. Nothing observable
|
2026-11-04. Leave the token row alone.
|
||||||
settles it today: the LDAP outpost's retained log holds only its 5-minute
|
|
||||||
`initialized direct binder` / `Update providers` config cycles and no bind
|
|
||||||
activity *(unverified 2026-10-18: no bind was performed — reproducing it
|
|
||||||
requires the bind secret)*. Re-verify with a real bind before changing the
|
|
||||||
token's intent either way.
|
|
||||||
- **Search needs `search_full_directory`.** Without the RBAC grant the
|
- **Search needs `search_full_directory`.** Without the RBAC grant the
|
||||||
outpost returns only the bind user itself — a `(memberOf=…)` filter then
|
outpost returns only the bind user itself — a `(memberOf=…)` filter then
|
||||||
matches zero entries and the plugin logs `Found no users matching X`.
|
matches zero entries and the plugin logs `Found no users matching X`.
|
||||||
The permission is object-level on the provider; grant it via an RBAC role
|
The permission is object-level on the provider; grant it via an RBAC role
|
||||||
and retest.
|
and retest.
|
||||||
|
- **The outpost log never shows a bind.** Its retained journal is only the
|
||||||
|
5-minute `initialized direct binder` / `Update providers` config cycle; the
|
||||||
|
absence of a bind line is not evidence of anything. Instruments that do answer:
|
||||||
|
`ldapsearch` with the plugin's own DN, password and filter, and the authentik
|
||||||
|
event registry (`login` / `login_failed` with `client_ip` from the podman
|
||||||
|
gateway, since the published port SNATs).
|
||||||
- `ak shell` import path for the provider model in 2026.8:
|
- `ak shell` import path for the provider model in 2026.8:
|
||||||
`from authentik.providers.ldap.models import LDAPProvider`.
|
`from authentik.providers.ldap.models import LDAPProvider` (verified inside a
|
||||||
- (unverified 2026-10-18: importing it outside a configured Django shell fails
|
live `ak shell`, 2026-11-04 — it only fails outside a configured Django shell).
|
||||||
with `ImproperlyConfigured`, and no `ak shell` was driven against the live
|
- **`ak shell` field names, 2026.8:** `Event` is keyed by `created` and `action`
|
||||||
auth service for this check)*
|
— there is no `timestamp` and no `type`; `Outpost` has `providers` but **no
|
||||||
|
`apps`**. Introspect first: `[f.name for f in Model._meta.get_fields()]`.
|
||||||
|
- **Never pipe `ak shell` inside a heredoc.** `podman exec -i … ak shell | grep
|
||||||
|
-v '^{'` binds the heredoc to `grep` (the redirection belongs to the LAST
|
||||||
|
element of the pipeline), so the shell gets no stdin and prints nothing at all.
|
||||||
|
And `2>/dev/null` on `ak shell` eats the traceback, which looks identical to "no
|
||||||
|
rows". Clean pattern: have the script write its results to a file and read that
|
||||||
|
file back —
|
||||||
|
`podman exec -i authentik-server ak shell >/dev/null 2>/dev/null <<'PY' … open("/tmp/x.txt","w") … PY`
|
||||||
|
then `podman exec authentik-server cat /tmp/x.txt`.
|
||||||
|
|
||||||
## Self-serve user-settings flow (password change)
|
## Self-serve user-settings flow (password change)
|
||||||
|
|
||||||
|
|||||||
@@ -145,18 +145,17 @@ relying-party quirks are reference lookup: [[Authentik Outpost Reference]].
|
|||||||
The second outpost — `dankclaw/authentik/quadlet/authentik-outpost-ldap.container`
|
The second outpost — `dankclaw/authentik/quadlet/authentik-outpost-ldap.container`
|
||||||
→ `authentik-outpost-ldap.service`, one of the five units in this stack —
|
→ `authentik-outpost-ldap.service`, one of the five units in this stack —
|
||||||
serves LDAP for Jellyfin's LDAP-Auth plugin (`enabled`=generated, `active`
|
serves LDAP for Jellyfin's LDAP-Auth plugin (`enabled`=generated, `active`
|
||||||
`Restart=always`, container up, 2026-10-18). The image is pinned by its **amd64
|
`Restart=always`, container up, re-checked 2026-11-04). **The image pin has one
|
||||||
single-platform digest**
|
owner: the `Image=` line of
|
||||||
`ghcr.io/goauthentik/ldap@sha256:2e7b259eb8c37faac916946ddf48ec681ee9d3a9f724fe8521bf161973f13cf9`
|
`dankclaw/authentik/quadlet/authentik-outpost-ldap.container`** — never restate
|
||||||
(`dankclaw/authentik/quadlet/authentik-outpost-ldap.container:26`, matching the
|
the digest here, because every pin bump turns this page into a lie (the digest
|
||||||
running image in `podman ps`, 2026-10-18). The digest in this page before 2026-10-18
|
cited here went stale twice: `…fa33bc8f…`, then `…2e7b259e…`). The reason a
|
||||||
(`…fa33bc8f…`) is superseded. The reason for pinning a digest at all: podman
|
digest is pinned at all: podman 5.7.0 refuses the plain tag because it resolves
|
||||||
5.7.0 refuses the plain tag because it resolves to a *nested* manifest list
|
to a *nested* manifest list (index → per-platform manifest lists) —
|
||||||
(index → per-platform manifest lists) — `unexpectedly received a manifest list
|
`unexpectedly received a manifest list instead of a manifest for a single image`
|
||||||
instead of a manifest for a single image` — first hit on `:2026.8.0`, still the
|
— first hit on `:2026.8.0` and still the rule. Resolve once with
|
||||||
rule at the current `2026.8.3` pin. Resolve once with `podman manifest inspect
|
`podman manifest inspect <image> | jq -r '.manifests[] |
|
||||||
<image> | jq -r '.manifests[] | select(.platform.architecture=="amd64") |
|
select(.platform.architecture=="amd64") | .digest'` and pin that digest.
|
||||||
.digest'` and pin that digest.
|
|
||||||
|
|
||||||
- Ports **389** (ldap) and **636** (ldaps) publish on the LAN
|
- Ports **389** (ldap) and **636** (ldaps) publish on the LAN
|
||||||
(`PublishPort=389:3389`, `636:6636`
|
(`PublishPort=389:3389`, `636:6636`
|
||||||
@@ -393,9 +392,10 @@ downstream —
|
|||||||
- `Non-Admin Users` = the plain-user marker group (precedents: qa-e2e, and
|
- `Non-Admin Users` = the plain-user marker group (precedents: qa-e2e, and
|
||||||
every human account).
|
every human account).
|
||||||
- **The human-user template** (ridley, kassie, alex, artem, kirby, ry, nic,
|
- **The human-user template** (ridley, kassie, alex, artem, kirby, ry, nic,
|
||||||
debo, freya, mitch, eric). The pks in this page are confirmed against
|
debo, freya, mitch, eric, jason). The pks were confirmed against
|
||||||
`authentik_core_user` on 2026-10-18: ridley 13, kassie 14, alex 15,
|
`authentik_core_user` on 2026-10-18 and the table re-read 2026-11-04: ridley
|
||||||
artem 16, kirby 17, ry 18, nic 19, debo 20, freya 21, mitch 22, eric 23 —
|
13, kassie 14, alex 15, artem 16, kirby 17, ry 18, nic 19, debo 20, freya 21,
|
||||||
|
mitch 22, eric 23, **jason 24 (created 2026-11-04)** —
|
||||||
and every one carries the `<user>@franksdank.cloud` address the template
|
and every one carries the `<user>@franksdank.cloud` address the template
|
||||||
prescribes. Creation dates (2026-10-07 → 2026-10-15) were not re-checked
|
prescribes. Creation dates (2026-10-07 → 2026-10-15) were not re-checked
|
||||||
- (unverified 2026-10-18)*. Template: `username` +
|
- (unverified 2026-10-18)*. Template: `username` +
|
||||||
|
|||||||
@@ -217,7 +217,10 @@ Certificates and ACME state live in the caddy data volume (`data/caddy/`). Back
|
|||||||
- A bare compose container is **not** boot-persistent: after the 2026-08-26 reboot Caddy stayed down for hours because `restart: unless-stopped` does not survive shutdown's explicit stop. It now runs only from its quadlet unit, and the repo has no `dankclaw/compose.yaml` left to re-add it to (`ls`, 2026-10-18).
|
- A bare compose container is **not** boot-persistent: after the 2026-08-26 reboot Caddy stayed down for hours because `restart: unless-stopped` does not survive shutdown's explicit stop. It now runs only from its quadlet unit, and the repo has no `dankclaw/compose.yaml` left to re-add it to (`ls`, 2026-10-18).
|
||||||
- `admin off` disables the config file watcher: rendered changes then sit on disk while Caddy serves its stale in-memory config until the container is recreated, and nothing logs an error (2026-08-22: a broken basicauth hash survived several deploys this way). `tests/caddyfile.bats` forbids `admin off`.
|
- `admin off` disables the config file watcher: rendered changes then sit on disk while Caddy serves its stale in-memory config until the container is recreated, and nothing logs an error (2026-08-22: a broken basicauth hash survived several deploys this way). `tests/caddyfile.bats` forbids `admin off`.
|
||||||
- The basicauth template must stay a bare `REPLACE_WITH_HASH` token (no `$2a$14$` prefix) — the rendered value is the complete hash. A double prefix silently rejects every password with 401; no Caddy error is logged.
|
- The basicauth template must stay a bare `REPLACE_WITH_HASH` token (no `$2a$14$` prefix) — the rendered value is the complete hash. A double prefix silently rejects every password with 401; no Caddy error is logged.
|
||||||
- A public vhost needs three things: a Pi-hole `addn-hosts` line, a Porkbun A record, and the Caddyfile block. See [[LAN-DNS]]. The deliberate exception is an **internal-only** name: `hermes.franksdank.cloud` and `hermesadmin.franksdank.cloud` are in `addn-hosts` and the Caddyfile but have no public record, which is what keeps them LAN-only — DNS-01 issuance needs no public A record (`dankclaw/Caddyfile:173-179`).
|
- **Exposure has exactly two gates: the router's 80/443 forward to Caddy, and Caddy's vhost list.** Caddy matches on SNI/Host, so *every* name in the Caddyfile already answers a WAN client that sends that name — a public A record buys **discoverability, not access** (measured 2026-11-04). A published name needs a Pi-hole `addn-hosts` line, a Porkbun A record and the Caddyfile block ([[LAN-DNS]]); DNS-01 issuance needs no public record, so an **internal-only** name is one with no A record. Posture measured 2026-11-04: the whole authentik-guarded set is published — `www.`, `search.`, `romm.`, `files.`, `grafana.`, `prometheus.`, `loki.`, `seerr.`, `marreta.`, `notebook.` — while the names with no public record are the self-authenticating exempt ones (`pihole.`, `hermes.`, `hermesadmin.`, `prowlarr.`, `radarr.`, `lidarr.`, `readarr.`, `qbittorrent.`) and the legacy redirects (`dankclaw.`, `dankserver.`). Every A record is a copy of the WAN address, which is dynamic — republish after an IP change.
|
||||||
|
- **A missing DNS record is not a gate: the names are public anyway.** Caddy issues one certificate per name and every issuance is published in Certificate Transparency logs — `crt.sh` for `%.franksdank.cloud` lists the live *and* retired set (`sonarr.`, `comfyui.`, `chat.`, `jellyfin.`, the whole *arr stack), so "LAN-only because there is no A record" is fiction and a stranger can send the Host. A household-only service needs a **source gate in its vhost**, which is what the *arr stack and qBittorrent now carry: a LAN client proxies through, anything off-LAN gets `404`. Keep those five gated even though they have no A record.
|
||||||
|
- **The LAN gate is `remote_ip`, and it reads the true client.** The G36 DNATs the 80/443 forward straight to Caddy with no proxy hop, so `@lan remote_ip 127.0.0.1/32 ::1/128 192.168.0.0/16 10.0.0.0/8 172.16.0.0/12` sees either a LAN address or the internet address — never the router's. Verify the split without waiting for a WAN client: a LAN request must reach the upstream (a dead upstream still proves it passed the gate, because the answer is its `502`, not the gate's `404`), and an external fetcher must get the `404`.
|
||||||
|
- **A new record looks like it failed for up to the SOA's negative TTL.** The zone's negative cache is 1800 s, so a resolver that asked for the name *before* the create keeps answering NXDOMAIN (1.1.1.1 did, while Quad9 and the authoritative NS had the record seconds after the create) — check `dig @<authoritative NS>` before re-creating anything.
|
||||||
- The G36 has no NAT loopback, so LAN clients must resolve the name to the LAN IP. That is the Pi-hole role in [[LAN-DNS]]. *(unverified 2026-10-18: hairpin behaviour is only observable from a WAN-side client; nothing on the LAN or in this repo can prove it)*
|
- The G36 has no NAT loopback, so LAN clients must resolve the name to the LAN IP. That is the Pi-hole role in [[LAN-DNS]]. *(unverified 2026-10-18: hairpin behaviour is only observable from a WAN-side client; nothing on the LAN or in this repo can prove it)*
|
||||||
- Caddy serves HTTPS on 443 and HTTP on 80 (redirects). The router needs no 80/443 port forward for LAN use.
|
- Caddy serves HTTPS on 443 and HTTP on 80 (redirects). The router needs no 80/443 port forward for LAN use.
|
||||||
- `transport http {versions h2c 1.1}` on a vhost makes Caddy send an h2c
|
- `transport http {versions h2c 1.1}` on a vhost makes Caddy send an h2c
|
||||||
|
|||||||
@@ -28,11 +28,12 @@ descriptions and a `docs` link to `/docs#models`.
|
|||||||
|
|
||||||
**Sections + WIP split (2026-09-21):** both pages now group their panels
|
**Sections + WIP split (2026-09-21):** both pages now group their panels
|
||||||
under section headings rendered by shared `renderSection()`/`renderPanel()`
|
under section headings rendered by shared `renderSection()`/`renderPanel()`
|
||||||
components in `app.js`. The user page orders **AI, Web Apps, Media Library
|
components in `app.js`. The user page orders **Web Apps, Media Library
|
||||||
Apps**; the admin page orders **Media Library Apps, Observability, AI,
|
Apps, AI, Work in progress** (the three not-open-yet apps moved under their
|
||||||
Infrastructure**. The WIP badge split into **WIP: OPEN** (work in progress)
|
own heading, below AI, on 2026-11-04); the admin page orders **Media Library
|
||||||
and **WIP: CLOSED** (work finished by decision — e.g. a card that is
|
Apps, Observability, AI, Infrastructure**. The WIP badge split into **WIP:
|
||||||
deliberately unprobeable) via the `wip` value (`"open"` / `"closed"`).
|
OPEN** (work in progress) and **WIP: CLOSED** (work finished by decision — e.g.
|
||||||
|
a card that is deliberately unprobeable) via the `wip` value.
|
||||||
Tautulli (Plex statistics, `192.168.0.235:8181`) joined the admin page's
|
Tautulli (Plex statistics, `192.168.0.235:8181`) joined the admin page's
|
||||||
Media Library Apps section.
|
Media Library Apps section.
|
||||||
|
|
||||||
@@ -314,9 +315,10 @@ flowchart TD
|
|||||||
## Service classification
|
## Service classification
|
||||||
|
|
||||||
**User services** (root page), in section order:
|
**User services** (root page), in section order:
|
||||||
- **AI** — OpenWebUI, and the three LLM fleet cards (frank-dragoon, frank-archon, frank-zealot — status + docs link, no Open link).
|
|
||||||
- **Web Apps** — SearXNG, Gitea, Copyparty, Marreta.
|
- **Web Apps** — SearXNG, Gitea, Copyparty, Marreta.
|
||||||
- **Media Library Apps** — Calibre-Web, RomM, Plex, Jellyfin, Seerr, Audiobookshelf (moved up from the admin page 2026-11-03, once Authentik SSO made it a household app).
|
- **Media Library Apps** — Plex, Jellyfin, Audiobookshelf (moved up from the admin page 2026-11-03, once Authentik SSO made it a household app).
|
||||||
|
- **AI** — OpenWebUI, the three LLM fleet cards (frank-dragoon, frank-archon, frank-zealot) and `frank-tts`, the cloned-voice endpoint (moved up from the admin page 2026-11-04 so the household sees whether the voice door is up). All four are status + docs link, no Open link; the fused **Model Information** panel renders under this heading.
|
||||||
|
- **Work in progress** — Calibre-Web, RomM, Seerr. WIP badge, no Open link, no household account yet: the ask-Frank door. See the docs page's *Three doors that are not open to the household yet*.
|
||||||
|
|
||||||
**Admin services** (`/admin/`), in section order:
|
**Admin services** (`/admin/`), in section order:
|
||||||
- **Media Library Apps** — the *arr stack (Prowlarr, Sonarr, Radarr, Lidarr, Readarr), qBittorrent, and Tautulli — live on FrankBook-Pro and Dankserver, operator-only: no vhosts (Tautulli included), direct IP:port access only. See [[FrankBook-Pro]].
|
- **Media Library Apps** — the *arr stack (Prowlarr, Sonarr, Radarr, Lidarr, Readarr), qBittorrent, and Tautulli — live on FrankBook-Pro and Dankserver, operator-only: no vhosts (Tautulli included), direct IP:port access only. See [[FrankBook-Pro]].
|
||||||
@@ -432,15 +434,22 @@ agent knowledge base (`~/.pi/agent/docs/caddy/caddyfile.md`).
|
|||||||
|
|
||||||
- `category` — `user` or `admin`. Selects the page.
|
- `category` — `user` or `admin`. Selects the page.
|
||||||
- `section` — the heading the panel renders under. Must be one of the page's
|
- `section` — the heading the panel renders under. Must be one of the page's
|
||||||
`SECTION_ORDER` entries (user: AI, Web Apps, Media Library Apps; admin:
|
`SECTION_ORDER` entries (user: Web Apps, Media Library Apps, AI, Work in
|
||||||
Media Library Apps, Observability, AI, Infrastructure), or the panel is
|
progress; admin: Media Library Apps, Observability, AI, Infrastructure), or
|
||||||
silently dropped from the page.
|
the panel is silently dropped from the page.
|
||||||
- `url` — the "Open" link (the Caddy site).
|
- `url` — the "Open" link (the Caddy site).
|
||||||
- `docs` — the per-service docs page (`/docs/<svc>/`).
|
- `docs` — the per-service docs page (`/docs/<svc>/`).
|
||||||
- `probe` — the backend `host:port` the checker probes.
|
- `probe` — the backend `host:port` the checker probes.
|
||||||
- `method` — `tcp` or `http`. The probe type for this service (Q10).
|
- `method` — `tcp` or `http`. The probe type for this service (Q10).
|
||||||
- `wip` — `false`, or `"open"` / `"closed"`. When set, the badge shows
|
- `wip` — `false`, or `"open"` / `"closed"` / `"down"`. When set, the badge shows
|
||||||
**WIP: OPEN** or **WIP: CLOSED** and the checker skips the probe.
|
**WIP: OPEN** / **WIP: CLOSED** / **WIP: DOWN** and the checker skips the probe.
|
||||||
|
- `model` — present only on Model Information cards. Such a card renders in the
|
||||||
|
fused `renderModelInfo()` panel (`mi-*` styles) instead of the app grid, and its
|
||||||
|
`description` is unused. `unit` is the StarCraft identity: art
|
||||||
|
`/img/unit-<unit>.jpg`, accent `.mi-<unit>`, voice lines from `/sfx/`. A card
|
||||||
|
with no `unit` is a tool (today: `frank-tts`) — art falls back to the card's own
|
||||||
|
`art`, the accent is `.mi-tool`, and a `tagline` with no `sfx` renders as plain
|
||||||
|
text rather than a play button.
|
||||||
|
|
||||||
**A card must be probeable from the monitor's host, or it must be `wip`.** The
|
**A card must be probeable from the monitor's host, or it must be `wip`.** The
|
||||||
checker runs on Dankclaw, so its reach is: that host's own loopback, its own
|
checker runs on Dankclaw, so its reach is: that host's own loopback, its own
|
||||||
|
|||||||
@@ -525,6 +525,21 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
|
|||||||
- `--prefix-cache-dir` creation is **not recursive** — the serve script `mkdir -p`s the path or
|
- `--prefix-cache-dir` creation is **not recursive** — the serve script `mkdir -p`s the path or
|
||||||
the disk tier silently never opens. The tier does survive a payload swap when arch and
|
the disk tier silently never opens. The tier does survive a payload swap when arch and
|
||||||
tokenizer match (it recovered 900,176 blocks on the 2026-10-15 swap; `cache_n` proves a hit).
|
tokenizer match (it recovered 900,176 blocks on the 2026-10-15 swap; `cache_n` proves a hit).
|
||||||
|
- **A radiance crash reads as a clean exit unless you know where to look, and the engine never
|
||||||
|
says why.** The unit's `ExecStart` ends in `exec podman wait`, so `systemctl show -p
|
||||||
|
ExecMainStatus` reports **0** after a container death — that is `podman wait`'s own status, not
|
||||||
|
the engine's — and `Result=success` beside `NRestarts=1` is the *normal* fingerprint of a
|
||||||
|
crash. The container's real code is the bare digit `podman wait` prints to stdout
|
||||||
|
(`serve-frank-archon.sh[NNNN]: 1`). The engine's fatal path writes **nothing** to its log: no
|
||||||
|
stderr line, no message before the exit and no core. The only copy of the reason is the error
|
||||||
|
the client received — `the engine stopped on an error and is not serving (<errno>)`, where the
|
||||||
|
parenthesis is the failing call's errno text (`invalid argument` = EINVAL, 2026-11-04). So
|
||||||
|
grep the *client* logs before concluding the journal is empty: omp keeps the provider error in
|
||||||
|
`~/.omp/logs/omp.<date>.<pid>.log` (`agent turn ended with provider error`), and OpenWebUI
|
||||||
|
shows it as the chat's error toast. Seen once (2026-11-04, restart counter 1 after 3d20h up):
|
||||||
|
three concurrent triple-digit-thousand-token agent turns prefilled at once, prefill rate
|
||||||
|
collapsed, all in-flight streams failed at the same instant, then a silent teardown — kernel
|
||||||
|
logged only the KFD SVM workqueue churning, with no OOM, GPU hang, MCE or swap pressure.
|
||||||
|
|
||||||
### Dead ends — do not re-try
|
### Dead ends — do not re-try
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ const CATEGORY = window.location.pathname.indexOf('/admin') !== -1 ? 'admin' : '
|
|||||||
// same renderSection()/renderPanel() below. SECTION_ORDER fixes each page's
|
// same renderSection()/renderPanel() below. SECTION_ORDER fixes each page's
|
||||||
// section order; a section with no services renders nothing.
|
// section order; a section with no services renders nothing.
|
||||||
const SECTION_ORDER = {
|
const SECTION_ORDER = {
|
||||||
user: ['Web Apps', 'Media Library Apps', 'AI'],
|
user: ['Web Apps', 'Media Library Apps', 'AI', 'Work in progress'],
|
||||||
admin: ['Media Library Apps', 'Observability', 'AI', 'Infrastructure'],
|
admin: ['Media Library Apps', 'Observability', 'AI', 'Infrastructure'],
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -79,15 +79,17 @@ function voiceBtn(text, files) {
|
|||||||
|
|
||||||
function miCard(svc, state) {
|
function miCard(svc, state) {
|
||||||
const m = svc.model;
|
const m = svc.model;
|
||||||
|
const art = svc.art || '/img/unit-' + m.unit + '.jpg';
|
||||||
const parts = ['<div class="mi-top">' +
|
const parts = ['<div class="mi-top">' +
|
||||||
'<img class="mi-art" src="/img/unit-' + esc(m.unit) + '.jpg" alt="' + esc(m.unit) + '">' +
|
'<img class="mi-art" src="' + esc(art) + '" alt="' + esc(m.unit || svc.name) + '">' +
|
||||||
'<div class="mi-head"><div class="mi-id"><span class="badge ' + state + '">' + BADGE_LABEL[state] + '</span>' +
|
'<div class="mi-head"><div class="mi-id"><span class="badge ' + state + '">' + BADGE_LABEL[state] + '</span>' +
|
||||||
'<span class="mi-name">' + esc(svc.name) + '</span></div>' +
|
'<span class="mi-name">' + esc(svc.name) + '</span></div>' +
|
||||||
'<div class="mi-role">' + esc(m.role) + '</div>' +
|
'<div class="mi-role">' + esc(m.role) + '</div>' +
|
||||||
(m.hf ? '<a class="mi-hf" href="' + esc(m.hf) +
|
(m.hf ? '<a class="mi-hf" href="' + esc(m.hf) +
|
||||||
'" target="_blank" rel="noopener">model on huggingface ↗</a>' : '') +
|
'" target="_blank" rel="noopener">model on huggingface ↗</a>' : '') +
|
||||||
'</div></div>',
|
'</div></div>',
|
||||||
voiceBtn(m.tagline, m.sfx)
|
(m.sfx && m.sfx.length ? voiceBtn(m.tagline, m.sfx)
|
||||||
|
: m.tagline ? '<div class="mi-tag">' + esc(m.tagline) + '</div>' : '')
|
||||||
];
|
];
|
||||||
if (m.equiv) parts.push('<div class="mi-equiv">' + esc(m.equiv) + '</div>');
|
if (m.equiv) parts.push('<div class="mi-equiv">' + esc(m.equiv) + '</div>');
|
||||||
if (m.spec) parts.push('<div class="mi-spec">' + esc(m.spec) + '</div>');
|
if (m.spec) parts.push('<div class="mi-spec">' + esc(m.spec) + '</div>');
|
||||||
@@ -99,7 +101,7 @@ function miCard(svc, state) {
|
|||||||
if (m.warn) parts.push('<details class="mi-warn"><summary>⚠ privacy note</summary>' +
|
if (m.warn) parts.push('<details class="mi-warn"><summary>⚠ privacy note</summary>' +
|
||||||
'<div class="warn-body">' + esc(m.warn) + '</div></details>');
|
'<div class="warn-body">' + esc(m.warn) + '</div></details>');
|
||||||
const card = document.createElement('article');
|
const card = document.createElement('article');
|
||||||
card.className = 'mi-card mi-' + esc(m.unit);
|
card.className = 'mi-card mi-' + esc(m.unit || 'tool');
|
||||||
card.innerHTML = parts.join('');
|
card.innerHTML = parts.join('');
|
||||||
return card;
|
return card;
|
||||||
}
|
}
|
||||||
|
|||||||
+62
-10
@@ -33,7 +33,7 @@ There is no signup button on purpose.</p>
|
|||||||
<p><strong>2. Pick what you want to do:</strong></p>
|
<p><strong>2. Pick what you want to do:</strong></p>
|
||||||
<table>
|
<table>
|
||||||
<tr><th>I want to…</th><th>Open</th><th>Needs its own login?</th></tr>
|
<tr><th>I want to…</th><th>Open</th><th>Needs its own login?</th></tr>
|
||||||
<tr><td>watch a film or a show</td><td><a href="https://jelly.franksdank.cloud">Jellyfin</a></td><td>yes — ask Frank</td></tr>
|
<tr><td>watch a film or a show</td><td><a href="https://jelly.franksdank.cloud">Jellyfin</a></td><td>no — your franksdank.cloud login</td></tr>
|
||||||
<tr><td>watch it on a TV that already has Plex</td><td><a href="https://plex.franksdank.cloud/web">Plex</a></td><td>yes — a Plex account, Frank invites it</td></tr>
|
<tr><td>watch it on a TV that already has Plex</td><td><a href="https://plex.franksdank.cloud/web">Plex</a></td><td>yes — a Plex account, Frank invites it</td></tr>
|
||||||
<tr><td>listen to an audiobook</td><td><a href="https://audiobooks.franksdank.cloud/audiobookshelf">Audiobookshelf</a></td><td>no — the family password</td></tr>
|
<tr><td>listen to an audiobook</td><td><a href="https://audiobooks.franksdank.cloud/audiobookshelf">Audiobookshelf</a></td><td>no — the family password</td></tr>
|
||||||
<tr><td>ask a computer a question, privately</td><td><a href="https://ai.franksdank.cloud">AI chat</a></td><td>no</td></tr>
|
<tr><td>ask a computer a question, privately</td><td><a href="https://ai.franksdank.cloud">AI chat</a></td><td>no</td></tr>
|
||||||
@@ -77,15 +77,17 @@ There is no signup button on purpose.</p>
|
|||||||
<li><strong>Authentik</strong> is the front desk. It asks who you are, once per device
|
<li><strong>Authentik</strong> is the front desk. It asks who you are, once per device
|
||||||
per few days, and after that you drift from app to app without being stopped again.</li>
|
per few days, and after that you drift from app to app without being stopped again.</li>
|
||||||
</ul>
|
</ul>
|
||||||
<p>One account opens: the homepage, search, Marreta, the AI chat, Files, Gitea.</p>
|
<p>One account opens: the homepage, search, Marreta, the AI chat, Files, Gitea, and Jellyfin
|
||||||
|
(there it is typed as username + password, not the automatic sign-in).</p>
|
||||||
<figure>
|
<figure>
|
||||||
<img src="/img/onboarding/door-authentik.webp" alt="The sign-in screen: an identification field, then a password field">
|
<img src="/img/onboarding/door-authentik.webp" alt="The sign-in screen: an identification field, then a password field">
|
||||||
<figcaption>The front desk. Your username first, then your password. Nothing here signs
|
<figcaption>The front desk. Your username first, then your password. Nothing here signs
|
||||||
you up for anything; if you don't have an account yet, ask Frank.</figcaption>
|
you up for anything; if you don't have an account yet, ask Frank.</figcaption>
|
||||||
</figure>
|
</figure>
|
||||||
<p class="key-warn">Three doors keep their own accounts and will not take the family
|
<p class="key-warn">Two doors keep their own accounts and will not take the family
|
||||||
password: <strong>Jellyfin</strong>, <strong>Plex</strong> (a Plex account, its own e-mail
|
password: <strong>Plex</strong> (a Plex account, its own e-mail
|
||||||
login) and the e-book door. Frank creates those by hand.</p>
|
login) and the e-book door. Frank creates those by hand. <strong>Jellyfin does take the
|
||||||
|
family login</strong> — your franksdank.cloud username and password.</p>
|
||||||
<p class="foot-note">Want your own password? It lives at
|
<p class="foot-note">Want your own password? It lives at
|
||||||
<a href="https://auth.franksdank.cloud/if/flow/default-user-settings-flow/">account settings</a>
|
<a href="https://auth.franksdank.cloud/if/flow/default-user-settings-flow/">account settings</a>
|
||||||
— the same button you get at the bottom of the homepage.</p>
|
— the same button you get at the bottom of the homepage.</p>
|
||||||
@@ -128,8 +130,10 @@ account of any company's, and the posters and episode guides are all filled in.<
|
|||||||
<h3>Getting in</h3>
|
<h3>Getting in</h3>
|
||||||
<ul>
|
<ul>
|
||||||
<li>Pick your name. There is no list of names on the login screen — type yours.</li>
|
<li>Pick your name. There is no list of names on the login screen — type yours.</li>
|
||||||
<li>Type the password Frank set for you. <strong>It is not the family password</strong>;
|
<li>Type your <strong>franksdank.cloud password</strong> — the same one that opens the
|
||||||
this is the most common wrong door.</li>
|
homepage, search and the AI chat. There is no separate Jellyfin password.</li>
|
||||||
|
<li>Refused, but the same password opens the homepage? Your name hasn't been given library
|
||||||
|
access yet — ask Frank, it's a one-line change on his side.</li>
|
||||||
<li>No name yet? Ask Frank to make one. There is no self-service signup, on purpose.</li>
|
<li>No name yet? Ask Frank to make one. There is no self-service signup, on purpose.</li>
|
||||||
</ul>
|
</ul>
|
||||||
<figure>
|
<figure>
|
||||||
@@ -147,7 +151,8 @@ account of any company's, and the posters and episode guides are all filled in.<
|
|||||||
</ul>
|
</ul>
|
||||||
<h3>If it misbehaves</h3>
|
<h3>If it misbehaves</h3>
|
||||||
<ul>
|
<ul>
|
||||||
<li>Wrong password and no error you understand? That's the family password again.</li>
|
<li>It rejects a password that works everywhere else? Your name isn't on the library-access
|
||||||
|
list yet — ask Frank.</li>
|
||||||
<li>A film with a blank poster is missing artwork, not missing files — Frank can refresh it.</li>
|
<li>A film with a blank poster is missing artwork, not missing files — Frank can refresh it.</li>
|
||||||
<li>Buffering away from home usually means the pipe out of the house, not the server.</li>
|
<li>Buffering away from home usually means the pipe out of the house, not the server.</li>
|
||||||
</ul>
|
</ul>
|
||||||
@@ -167,7 +172,8 @@ Plex's own sign-in, and that is correct: Plex is closed-source and insists on a
|
|||||||
both; the artwork, watch-state and profiles are separate.</li>
|
both; the artwork, watch-state and profiles are separate.</li>
|
||||||
<li>Choose Plex when your TV already has the Plex app on it, or you want Plex's trailers
|
<li>Choose Plex when your TV already has the Plex app on it, or you want Plex's trailers
|
||||||
and music. Otherwise Jellyfin is the one the household uses.</li>
|
and music. Otherwise Jellyfin is the one the household uses.</li>
|
||||||
<li>Ask Frank for an invitation; he can also tell you which of the two has the thing you're looking for.</li>
|
<li>Need a Plex account of your own? Ask Frank to invite it; he can also tell you which of
|
||||||
|
the two has the thing you're looking for.</li>
|
||||||
</ul>
|
</ul>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
@@ -359,6 +365,51 @@ Qwen3.8-Flash-Next v2, abliterated
|
|||||||
(<a href="https://huggingface.co/prism-ml/Ternary-Bonsai-2-9B-gguf">prism-ml/Ternary-Bonsai-2-9B-gguf</a>)
|
(<a href="https://huggingface.co/prism-ml/Ternary-Bonsai-2-9B-gguf">prism-ml/Ternary-Bonsai-2-9B-gguf</a>)
|
||||||
with a small adapter fitted that removes its refusal reflex. All three accept pictures. Ask
|
with a small adapter fitted that removes its refusal reflex. All three accept pictures. Ask
|
||||||
Frank before pointing a script at one of them directly; the address is on the homepage.</p>
|
Frank before pointing a script at one of them directly; the address is on the homepage.</p>
|
||||||
|
|
||||||
|
<h3>And the voices</h3>
|
||||||
|
<p>Two speech models live in this house, and they are different animals. One of them
|
||||||
|
speaks as a <em>specific person</em>; the other is a pack of stock voices that ships inside
|
||||||
|
the tools you already use.</p>
|
||||||
|
<table>
|
||||||
|
<tr><th>Name</th><th>Runs on</th><th>Good at</th><th>Notes</th></tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>frank-tts</code></td>
|
||||||
|
<td>the Strix Halo machine's built-in graphics, sharing the card with <code>frank-dragoon</code></td>
|
||||||
|
<td>cloning: it speaks as one particular voice, chosen by name</td>
|
||||||
|
<td>five voices today — <code>dr.kern</code>, <code>garrickhagon</code>,
|
||||||
|
<code>jeffhays</code>, <code>vaati</code>, <code>wilwheaton</code>; ask for one by name and
|
||||||
|
you get it. No key, but it is a house-hardware door: it answers on the house network only</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><strong>Kokoro</strong></td>
|
||||||
|
<td>your own browser, or Frank's own terminal — no server in the loop</td>
|
||||||
|
<td>instant plain narration, quick and cheap</td>
|
||||||
|
<td>about two dozen preset voices with coded names (<code>af_heart</code>,
|
||||||
|
<code>bm_george</code>…). It cannot clone, and it will never answer to the five names
|
||||||
|
above</td>
|
||||||
|
</tr>
|
||||||
|
</table>
|
||||||
|
<ul>
|
||||||
|
<li><strong>The AI chat's speaker button is Kokoro.</strong> That is why it offers a
|
||||||
|
different list of names, and why it starts talking a moment before anything on the server
|
||||||
|
could. It speaks in your browser, on your machine.</li>
|
||||||
|
<li><strong>The podcast maker uses the clones.</strong> An episode's two hosts are
|
||||||
|
<code>wilwheaton</code> and <code>vaati</code>, rendered by <code>frank-tts</code>. That door
|
||||||
|
is Frank's for now.</li>
|
||||||
|
<li><strong>Frank's coding agent narrates through the clones too</strong> — the default
|
||||||
|
voice is <code>wilwheaton</code>.</li>
|
||||||
|
<li><strong>Want your own voice in the list?</strong> Ask Frank. It needs about twenty
|
||||||
|
seconds of you talking plainly, once; the voice then carries your name.</li>
|
||||||
|
</ul>
|
||||||
|
<p class="foot-note">The exact weights: <code>frank-tts</code> is a Qwen3 talker
|
||||||
|
(1.7 billion parameters, 4-bit) plus its 12 Hz audio codec, driven by
|
||||||
|
<code>qwentts.cpp</code>; Kokoro is a much smaller 82-million-parameter model whose voices are
|
||||||
|
baked in. A cloned voice is not a setting — it is a directory of reference audio on the
|
||||||
|
server, which is why adding one is a fifteen-minute job and not a checkbox. Programmers on the
|
||||||
|
house network: <code>POST http://192.168.0.143:7772/v1/audio/speech</code> with
|
||||||
|
<code>{"input": "…", "voice": "vaati"}</code> gets you WAV or MP3, and
|
||||||
|
<a href="http://192.168.0.143:7772/v1/audio/voices"><code>/v1/audio/voices</code></a> answers
|
||||||
|
the live list, so this page can never lie to you for long.</p>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section id="api-access">
|
<section id="api-access">
|
||||||
@@ -401,7 +452,8 @@ one you were given? Ask Frank to widen the key.</p>
|
|||||||
scripts it needs (or an extension is). Allow scripts for that site, or try another browser.
|
scripts it needs (or an extension is). Allow scripts for that site, or try another browser.
|
||||||
Still looping? Ask Frank — the guard can be told you're a person.</li>
|
Still looping? Ask Frank — the guard can be told you're a person.</li>
|
||||||
<li><strong>"Invalid password" on the front desk.</strong> Right password, wrong door: the
|
<li><strong>"Invalid password" on the front desk.</strong> Right password, wrong door: the
|
||||||
family login does not open Jellyfin, Plex or the e-book door.</li>
|
family login does not open Plex or the e-book door. Jellyfin does take it, but only for the
|
||||||
|
names Frank has given library access.</li>
|
||||||
<li><strong>The AI is slow.</strong> Each model only holds a couple of conversations at once
|
<li><strong>The AI is slow.</strong> Each model only holds a couple of conversations at once
|
||||||
(two, three or four, depending which). It is a queue, not a crash — or Frank is
|
(two, three or four, depending which). It is a queue, not a crash — or Frank is
|
||||||
mid-conversation; the big one is on the same machine as the films.</li>
|
mid-conversation; the big one is on the same machine as the films.</li>
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
<svg fill="#82612C" role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Audiobookshelf</title><path d="M12 0A12 12 0 0 0 0 12a12 12 0 0 0 12 12 12 12 0 0 0 12-12A12 12 0 0 0 12 0Zm-.023.402A11.598 11.598 0 0 1 23.575 12a11.598 11.598 0 0 1-11.598 11.598A11.598 11.598 0 0 1 .378 12 11.598 11.598 0 0 1 11.977.402Zm0 1.776a7.093 7.093 0 0 0-7.092 7.093v1.536a6.395 6.395 0 0 0-.439.33.35.35 0 0 0-.126.27v1.84a.36.36 0 0 0 .126.272c.22.182.722.564 1.504.956v.179c0 .483.31.873.694.873.384 0 .694-.392.694-.873v-4.415c0-.483-.31-.873-.694-.873-.369 0-.67.359-.694.812h-.002v-.91a6.027 6.027 0 1 1 12.054.003v.91c-.025-.454-.326-.813-.695-.813-.384 0-.694.391-.694.873v4.415c0 .483.31.873.694.873.384 0 .695-.392.695-.873v-.179a7.964 7.964 0 0 0 1.503-.956.35.35 0 0 0 .126-.272v-1.843a.342.342 0 0 0-.124-.27 5.932 5.932 0 0 0-.438-.329V9.271a7.093 7.093 0 0 0-7.092-7.093zm-3.34 5.548a.84.84 0 0 0-.84.84v9.405c0 .464.376.84.84.84h.866a.84.84 0 0 0 .84-.84V8.566a.84.84 0 0 0-.84-.84Zm2.905 0a.84.84 0 0 0-.84.84v9.405c0 .464.377.84.84.84h.867a.84.84 0 0 0 .84-.84V8.566a.84.84 0 0 0-.84-.84zm2.908 0a.84.84 0 0 0-.84.84v9.405c0 .464.376.84.84.84h.867a.84.84 0 0 0 .84-.84V8.566a.84.84 0 0 0-.84-.84zM8.112 9.983h1.915v.2H8.112Zm2.906 0h1.915v.2h-1.915Zm2.908 0h1.915v.2h-1.915zm-7.58 9.119a.633.633 0 0 0 0 1.265h11.26a.632.632 0 0 0 0-1.265z"/></svg>
|
||||||
|
After Width: | Height: | Size: 1.3 KiB |
@@ -1 +0,0 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 96 96"><polygon points="48,6 86,27 86,69 48,90 10,69 10,27" fill="#1c1300" stroke="#3fc1c9" stroke-width="3"/><text x="48" y="60" font-family="monospace" font-size="30" font-weight="bold" fill="#3fc1c9" text-anchor="middle">AB</text></svg>
|
|
||||||
|
Before Width: | Height: | Size: 293 B |
@@ -57,8 +57,8 @@
|
|||||||
{
|
{
|
||||||
"name": "Calibre-Web",
|
"name": "Calibre-Web",
|
||||||
"category": "user",
|
"category": "user",
|
||||||
"section": "Media Library Apps",
|
"section": "Work in progress",
|
||||||
"description": "E-book library. Work in progress: not open to family users yet.",
|
"description": "E-book library. Not open to family users yet — ask Frank.",
|
||||||
"url": "",
|
"url": "",
|
||||||
"docs": "",
|
"docs": "",
|
||||||
"probe": "192.168.0.235:8083",
|
"probe": "192.168.0.235:8083",
|
||||||
@@ -70,8 +70,8 @@
|
|||||||
{
|
{
|
||||||
"name": "RomM",
|
"name": "RomM",
|
||||||
"category": "user",
|
"category": "user",
|
||||||
"section": "Media Library Apps",
|
"section": "Work in progress",
|
||||||
"description": "ROM and emulator library. Work in progress: not open to family users yet.",
|
"description": "ROM and emulator library. Not open to family users yet — ask Frank.",
|
||||||
"url": "",
|
"url": "",
|
||||||
"docs": "",
|
"docs": "",
|
||||||
"probe": "192.168.0.235:8085",
|
"probe": "192.168.0.235:8085",
|
||||||
@@ -110,7 +110,7 @@
|
|||||||
"name": "Jellyfin",
|
"name": "Jellyfin",
|
||||||
"category": "user",
|
"category": "user",
|
||||||
"section": "Media Library Apps",
|
"section": "Media Library Apps",
|
||||||
"description": "Open source media server for the film and TV library: its own logins (no Authentik here), so ask Frank for an account.",
|
"description": "Open source media server for the film and TV library. Sign in with your franksdank.cloud username and password — the same login as the rest of the house; ask Frank if yours isn't in yet.",
|
||||||
"url": "http://jelly.franksdank.cloud",
|
"url": "http://jelly.franksdank.cloud",
|
||||||
"docs": "",
|
"docs": "",
|
||||||
"probe": "192.168.0.235:8096",
|
"probe": "192.168.0.235:8096",
|
||||||
@@ -122,8 +122,8 @@
|
|||||||
{
|
{
|
||||||
"name": "Seerr",
|
"name": "Seerr",
|
||||||
"category": "user",
|
"category": "user",
|
||||||
"section": "Media Library Apps",
|
"section": "Work in progress",
|
||||||
"description": "Download requests. Work in progress: not open to family users yet.",
|
"description": "Download requests. Not open to family users yet — ask Frank.",
|
||||||
"url": "",
|
"url": "",
|
||||||
"docs": "",
|
"docs": "",
|
||||||
"probe": "127.0.0.1:5055",
|
"probe": "127.0.0.1:5055",
|
||||||
@@ -136,7 +136,7 @@
|
|||||||
"name": "Marreta",
|
"name": "Marreta",
|
||||||
"category": "user",
|
"category": "user",
|
||||||
"section": "Web Apps",
|
"section": "Web Apps",
|
||||||
"description": "Paywall sledgehammer: hand it an article URL and it drags the page back through the VPN with the paywall chrome, cookie wall and trackers stripped; a headless browser renders the JS-heavy ones. Best effort — hard paywalls and login walls still win.",
|
"description": "Paywall sledgehammer: hand it an article URL and it will attempt to smash its paywall. Note: does not work on all sites.",
|
||||||
"url": "http://marreta.franksdank.cloud",
|
"url": "http://marreta.franksdank.cloud",
|
||||||
"docs": "",
|
"docs": "",
|
||||||
"probe": "127.0.0.1:8082",
|
"probe": "127.0.0.1:8082",
|
||||||
@@ -522,7 +522,7 @@
|
|||||||
"method": "http",
|
"method": "http",
|
||||||
"wip": false,
|
"wip": false,
|
||||||
"showLink": true,
|
"showLink": true,
|
||||||
"art": "/img/apps/mono-ab.svg"
|
"art": "/img/apps/audiobookshelf.svg"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "Postgres",
|
"name": "Postgres",
|
||||||
@@ -552,11 +552,20 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "frank-tts",
|
"name": "frank-tts",
|
||||||
"category": "admin",
|
"category": "user",
|
||||||
"section": "AI",
|
"section": "AI",
|
||||||
"description": "The fleet's cloned-voice speech endpoint: POST /v1/audio/speech with voice=<name>, audio back in that voice. Published on 7772; the engine behind it (frank-tts-engine) is unpublished on purpose. It is also the harness speech role.",
|
"model": {
|
||||||
|
"role": "The fleet's voice — it speaks, it does not chat",
|
||||||
|
"hf": "https://huggingface.co/Serveurperso/Qwen3-TTS-GGUF",
|
||||||
|
"tagline": "Your library, read aloud in a voice you know.",
|
||||||
|
"spec": "Qwen3-TTS 1.7 B base talker (Q4_K_M) with its 12 Hz tokenizer, run by qwentts.cpp on the gfx1151 iGPU. The frank-tts door answers POST /v1/audio/speech with voice=<name> and hands back audio in that voice.",
|
||||||
|
"inputs": ["text"],
|
||||||
|
"use": "Narration: Open Notebook episodes, the operator harness's spoken output, and anything that should sound like a person instead of a text-to-speech robot.",
|
||||||
|
"permission": "Ask Frank for a voice — each pack is built from that person's own recording.",
|
||||||
|
"warn": "Cloned voices. Every pack on this endpoint is a replica of a real person's voice. Audio coming out of it is not that person speaking and must never be passed off as them."
|
||||||
|
},
|
||||||
"url": "",
|
"url": "",
|
||||||
"docs": "",
|
"docs": "/docs#models",
|
||||||
"probe": "127.0.0.1:7772",
|
"probe": "127.0.0.1:7772",
|
||||||
"method": "http",
|
"method": "http",
|
||||||
"wip": false,
|
"wip": false,
|
||||||
|
|||||||
@@ -263,6 +263,7 @@ html[data-bare="1"] .top { border-image: none; }
|
|||||||
.mi-zealot { --mi: #4fb4ff; } /* Zealot: Khai's blue blades */
|
.mi-zealot { --mi: #4fb4ff; } /* Zealot: Khai's blue blades */
|
||||||
.mi-archon { --mi: #a8ecff; } /* Archon: pure psychic light */
|
.mi-archon { --mi: #a8ecff; } /* Archon: pure psychic light */
|
||||||
.mi-dragoon { --mi: #5fe0d0; } /* Dragoon: phase-crystal teal */
|
.mi-dragoon { --mi: #5fe0d0; } /* Dragoon: phase-crystal teal */
|
||||||
|
.mi-tool { --mi: #ffc86b; } /* frank-tts: amber — a tool, not a unit */
|
||||||
.mi-top { display: flex; align-items: center; gap: 10px; }
|
.mi-top { display: flex; align-items: center; gap: 10px; }
|
||||||
.mi-art {
|
.mi-art {
|
||||||
width: 56px;
|
width: 56px;
|
||||||
@@ -294,6 +295,7 @@ html[data-bare="1"] .top { border-image: none; }
|
|||||||
.mi-voice:hover { text-shadow: 0 0 8px var(--mi); }
|
.mi-voice:hover { text-shadow: 0 0 8px var(--mi); }
|
||||||
.mi-voice.mi-speaking { animation: mi-pulse 0.7s ease-in-out infinite; }
|
.mi-voice.mi-speaking { animation: mi-pulse 0.7s ease-in-out infinite; }
|
||||||
@keyframes mi-pulse { 50% { text-shadow: 0 0 12px var(--mi); } }
|
@keyframes mi-pulse { 50% { text-shadow: 0 0 12px var(--mi); } }
|
||||||
|
.mi-tag { color: var(--mi); margin: 4px 0; }
|
||||||
.mi-equiv { color: var(--head); font-style: italic; margin: 4px 0; }
|
.mi-equiv { color: var(--head); font-style: italic; margin: 4px 0; }
|
||||||
.mi-spec { color: var(--fg); margin: 4px 0; }
|
.mi-spec { color: var(--fg); margin: 4px 0; }
|
||||||
.mi-inputs { color: var(--dim); }
|
.mi-inputs { color: var(--dim); }
|
||||||
|
|||||||
@@ -981,9 +981,12 @@ DRV
|
|||||||
# there is no address the checker could dial, and a card that can never be
|
# there is no address the checker could dial, and a card that can never be
|
||||||
# green is worse than no card. The façade carries the pair's signal — and
|
# green is worse than no card. The façade carries the pair's signal — and
|
||||||
# since the 2026-10-18 deploy that signal is a real probe (`wip: false`), not
|
# since the 2026-10-18 deploy that signal is a real probe (`wip: false`), not
|
||||||
# a placeholder. It stays `showLink: false` and admin-only: an HTTP API has no
|
# a placeholder. It stays `showLink: false`: an HTTP API has no page to open,
|
||||||
# page to open, and the checker counts ANY HTTP response as up, so the card is
|
# and the checker counts ANY HTTP response as up, so the card is honest
|
||||||
# honest without a link and without a `/health` path in the probe.
|
# without a link and without a `/health` path in the probe.
|
||||||
|
# 2026-11-04: the card moved from admin to the user page's Model Information
|
||||||
|
# block — the household guide already documents the voices (/docs#models), so
|
||||||
|
# the family reading about them should also see whether the door is up.
|
||||||
python3 - "$REPO_ROOT" <<'PY'
|
python3 - "$REPO_ROOT" <<'PY'
|
||||||
import json, os, sys
|
import json, os, sys
|
||||||
root = sys.argv[1]
|
root = sys.argv[1]
|
||||||
@@ -996,8 +999,10 @@ assert fts["wip"] is False, \
|
|||||||
"the endpoint is deployed; a wip card would hide its real state: " + repr(fts.get("wip"))
|
"the endpoint is deployed; a wip card would hide its real state: " + repr(fts.get("wip"))
|
||||||
assert fts["showLink"] is False and fts["url"] == "", \
|
assert fts["showLink"] is False and fts["url"] == "", \
|
||||||
"an API endpoint has no UI to link to — it is a status card, not a tile"
|
"an API endpoint has no UI to link to — it is a status card, not a tile"
|
||||||
assert fts["category"] == "admin", \
|
assert fts["category"] == "user" and fts["section"] == "AI", \
|
||||||
"consumers dial this; it is not an app the user homepage should advertise"
|
"the voice endpoint belongs to the user page's Model Information block: " + repr(fts["category"])
|
||||||
|
assert isinstance(fts.get("model"), dict) and fts["model"].get("role"), \
|
||||||
|
"the card renders as a Model Information card, which needs a model object"
|
||||||
assert os.path.exists(f"{root}/shared/homepage{fts['art']}"), f"missing art {fts['art']}"
|
assert os.path.exists(f"{root}/shared/homepage{fts['art']}"), f"missing art {fts['art']}"
|
||||||
PY
|
PY
|
||||||
}
|
}
|
||||||
|
|||||||
+22
-12
@@ -37,7 +37,7 @@ PY
|
|||||||
import json, sys
|
import json, sys
|
||||||
data = json.load(open(sys.argv[1]))
|
data = json.load(open(sys.argv[1]))
|
||||||
allowed = {
|
allowed = {
|
||||||
"user": {"AI", "Web Apps", "Media Library Apps"},
|
"user": {"AI", "Web Apps", "Media Library Apps", "Work in progress"},
|
||||||
"admin": {"Media Library Apps", "Observability", "AI", "Infrastructure"},
|
"admin": {"Media Library Apps", "Observability", "AI", "Infrastructure"},
|
||||||
}
|
}
|
||||||
for svc in data:
|
for svc in data:
|
||||||
@@ -76,23 +76,33 @@ PY
|
|||||||
|
|
||||||
@test "the Model Information component is complete: text, art, voices, warnings" {
|
@test "the Model Information component is complete: text, art, voices, warnings" {
|
||||||
# The fused model panel is the family's only description of the fleet.
|
# The fused model panel is the family's only description of the fleet.
|
||||||
# Every model card must carry: equivalence note, supervision level, the
|
# Every card must carry: role, spec, use, an inputs list, and art that
|
||||||
# unit art, at least one playable voice line, and (for uncensored models)
|
# exists. A card with a StarCraft unit identity (a chat model) must also
|
||||||
# the legal warning. The compaction recommendation must name the zealot.
|
# carry the equivalence note, the supervision level, and a playable unit
|
||||||
|
# voice line whose file ships; a unit-less tool card (frank-tts speaks, it
|
||||||
|
# does not chat) quotes no unit, so it carries the ask-Frank permission
|
||||||
|
# note instead. Uncensored models must carry the legal warning. The
|
||||||
|
# compaction recommendation must name the zealot.
|
||||||
python3 - "$REPO_ROOT/shared/homepage/services.json" "$REPO_ROOT/shared/homepage" <<'PY'
|
python3 - "$REPO_ROOT/shared/homepage/services.json" "$REPO_ROOT/shared/homepage" <<'PY'
|
||||||
import json, os, sys
|
import json, os, sys
|
||||||
services = json.load(open(sys.argv[1]))
|
services = json.load(open(sys.argv[1]))
|
||||||
root = sys.argv[2]
|
root = sys.argv[2]
|
||||||
fleet = [s for s in services if "model" in s]
|
fleet = [s for s in services if "model" in s]
|
||||||
assert {s["name"] for s in fleet} == {"frank-dragoon", "frank-archon", "frank-zealot"}, [s["name"] for s in fleet]
|
assert {s["name"] for s in fleet} == {"frank-dragoon", "frank-archon", "frank-zealot", "frank-tts"}, [s["name"] for s in fleet]
|
||||||
for s in fleet:
|
for s in fleet:
|
||||||
m = s["model"]
|
m = s["model"]
|
||||||
assert m["equiv"].strip(), f"{s['name']} has no equivalence note"
|
assert m["role"].strip() and m["spec"].strip() and m["use"].strip(), f"{s['name']} needs role, spec and use"
|
||||||
assert m["supervision"].strip(), f"{s['name']} has no supervision note"
|
assert m.get("inputs"), f"{s['name']} has no inputs list"
|
||||||
assert m["sfx"], f"{s['name']} has no voice line"
|
art = s.get("art") or "img/unit-%s.jpg" % m["unit"]
|
||||||
assert os.path.isfile(os.path.join(root, "img", "unit-" + m["unit"] + ".jpg")), f"{s['name']} art missing"
|
assert os.path.isfile(os.path.join(root, art.lstrip("/"))), f"{s['name']} art missing: {art}"
|
||||||
for f in m["sfx"] + m.get("voice", {}).get("sfx", []):
|
for f in list(m.get("sfx", [])) + m.get("voice", {}).get("sfx", []):
|
||||||
assert os.path.isfile(os.path.join(root, "sfx", f)), f"{s['name']} voice {f} missing"
|
assert os.path.isfile(os.path.join(root, "sfx", f)), f"{s['name']} voice {f} missing"
|
||||||
|
if m.get("unit"):
|
||||||
|
assert m["equiv"].strip(), f"{s['name']} has no equivalence note"
|
||||||
|
assert m["supervision"].strip(), f"{s['name']} has no supervision note"
|
||||||
|
assert m.get("sfx"), f"{s['name']} has no voice line"
|
||||||
|
else:
|
||||||
|
assert m.get("permission", "").strip(), f"{s['name']} has no permission note"
|
||||||
zealot = next(s for s in fleet if s["name"] == "frank-zealot")
|
zealot = next(s for s in fleet if s["name"] == "frank-zealot")
|
||||||
assert "compaction" in zealot["model"].get("compaction", ""), "zealot must be recommended as compaction model"
|
assert "compaction" in zealot["model"].get("compaction", ""), "zealot must be recommended as compaction model"
|
||||||
assert zealot["model"].get("compaction", "") != "", "compaction advice must be its own visible field"
|
assert zealot["model"].get("compaction", "") != "", "compaction advice must be its own visible field"
|
||||||
@@ -102,8 +112,8 @@ for s in fleet:
|
|||||||
uncensored = "ncensored" in m["spec"] or "bliterated" in m["spec"]
|
uncensored = "ncensored" in m["spec"] or "bliterated" in m["spec"]
|
||||||
if uncensored:
|
if uncensored:
|
||||||
assert "legal" in m.get("warn", "").lower(), f"{s['name']} is uncensored but has no legal warning"
|
assert "legal" in m.get("warn", "").lower(), f"{s['name']} is uncensored but has no legal warning"
|
||||||
assert [s["name"] for s in fleet] == ["frank-zealot", "frank-dragoon", "frank-archon"], \
|
assert [s["name"] for s in fleet] == ["frank-zealot", "frank-dragoon", "frank-archon", "frank-tts"], \
|
||||||
"card render order must be zealot, dragoon, archon: " + str([s["name"] for s in fleet])
|
"card render order must be zealot, dragoon, archon, then the voice tool: " + str([s["name"] for s in fleet])
|
||||||
w = {s["name"]: s["model"].get("warn", "") for s in fleet}
|
w = {s["name"]: s["model"].get("warn", "") for s in fleet}
|
||||||
assert "legal risk" in w["frank-dragoon"].lower() and "everybody" in w["frank-dragoon"].lower(), \
|
assert "legal risk" in w["frank-dragoon"].lower() and "everybody" in w["frank-dragoon"].lower(), \
|
||||||
"the abliterated endpoint must carry the common legal warning"
|
"the abliterated endpoint must carry the common legal warning"
|
||||||
|
|||||||
@@ -41,8 +41,8 @@ setup() {
|
|||||||
grep -q "repeat(auto-fill" "$BASE/style.css"
|
grep -q "repeat(auto-fill" "$BASE/style.css"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "the user page keeps the AI section at the bottom" {
|
@test "the user page puts the Work in progress section last, below AI" {
|
||||||
grep -q "user: \['Web Apps', 'Media Library Apps', 'AI'\]" "$BASE/app.js"
|
grep -q "user: \['Web Apps', 'Media Library Apps', 'AI', 'Work in progress'\]" "$BASE/app.js"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "WIP badges distinguish open from closed work" {
|
@test "WIP badges distinguish open from closed work" {
|
||||||
|
|||||||
Executable
+361
@@ -0,0 +1,361 @@
|
|||||||
|
#!/usr/bin/env bats
|
||||||
|
# Seam: scripts/media-refresh-jellyfin.py + scripts/media-refresh-plex.sh —
|
||||||
|
# the two library-refresh drivers every media pass ends with.
|
||||||
|
#
|
||||||
|
# The rescan is the part of a media move that actually makes it visible, and
|
||||||
|
# both servers punish the naive version of it. Jellyfin 12.4: the quoted
|
||||||
|
# MediaBrowser header is the only auth form; POST /Library/Refresh is the only
|
||||||
|
# door that queues work; a SECOND accepted POST cancels the scan already
|
||||||
|
# running, so the index looks frozen while you hammer it; 429 means wait. Plex:
|
||||||
|
# its HTTPD answers nobody outside its own netns and speaks TLS to everyone who
|
||||||
|
# is inside the LAN, so the request has to be shipped INTO the container; the
|
||||||
|
# token is PlexOnlineToken in the container's Preferences.xml; and progress is
|
||||||
|
# readable without any of that, from section_locations.scanned_at. This seam
|
||||||
|
# pins, against stubs that speak those APIs:
|
||||||
|
# 1. the Jellyfin driver authenticates as `Authorization: MediaBrowser
|
||||||
|
# Token="…"`, and an auth refusal ends the run naming the code;
|
||||||
|
# 2. a trigger while Scan Media Library is running is REFUSED with no POST at
|
||||||
|
# all (the cancel-restart trap); --force overrides;
|
||||||
|
# 3. a 429 is waited out with a bounded number of attempts, never hammered,
|
||||||
|
# and a still-busy server ends in a non-zero exit that says 429;
|
||||||
|
# 4. `status` prints {"scan": …, "recentMedia": …} from the task table plus a
|
||||||
|
# fixture DB, and `wait` returns only after N consecutive quiet reads;
|
||||||
|
# 5. the Plex driver carries the token from the container's Preferences.xml
|
||||||
|
# into a header — never argv, never a URL;
|
||||||
|
# 6. a Plex trigger that gets no answer is a warming server: retried, bounded,
|
||||||
|
# and reported as 000 rather than a service restart;
|
||||||
|
# 7. Plex `status` answers from section_locations.scanned_at and is exit-0
|
||||||
|
# only when that epoch has passed --after;
|
||||||
|
# 8. a missing credential or database is refused by NAME before any request.
|
||||||
|
|
||||||
|
load helpers
|
||||||
|
|
||||||
|
setup() {
|
||||||
|
JF="$REPO_ROOT/scripts/media-refresh-jellyfin.py"
|
||||||
|
PX="$REPO_ROOT/scripts/media-refresh-plex.sh"
|
||||||
|
TMP="$(mktemp -d)"
|
||||||
|
PORT=18947
|
||||||
|
printf 'stubkey\n' > "$TMP/jf-key"
|
||||||
|
printf '<Preferences PlexOnlineToken="stubplextoken"/>\n' > "$TMP/prefs.xml"
|
||||||
|
echo Idle > "$TMP/scan-state"
|
||||||
|
echo ok > "$TMP/refresh-mode"
|
||||||
|
echo 0 > "$TMP/polls"
|
||||||
|
mkdir -p "$TMP/bin" "$TMP/container"
|
||||||
|
# A podman that is really a local directory: the shim runs the runner the
|
||||||
|
# driver copied in, so the in-container half is exercised for real.
|
||||||
|
cat > "$TMP/bin/podman" <<'SH'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
echo "podman $*" >> "$PODMAN_LOG"
|
||||||
|
case "$1" in
|
||||||
|
cp) cp "$2" "$CONTAINER_FS/$(basename "$3" | sed 's#.*:##')" ;;
|
||||||
|
exec) shift; [ "$1" = "$CONTAINER" ] || exit 1; shift
|
||||||
|
[ "$1" = rm ] && exit 0
|
||||||
|
[ "$1" = sh ] || exit 1; shift
|
||||||
|
script="$1"; shift
|
||||||
|
sh "$CONTAINER_FS/$(basename "$script")" "$@" ;;
|
||||||
|
*) exit 1 ;;
|
||||||
|
esac
|
||||||
|
SH
|
||||||
|
chmod +x "$TMP/bin/podman"
|
||||||
|
cat > "$TMP/stub.py" <<'PY'
|
||||||
|
import json, os, sys
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
|
||||||
|
port, logf = int(sys.argv[1]), sys.argv[2]
|
||||||
|
|
||||||
|
|
||||||
|
def read(var, default=""):
|
||||||
|
# Scenarios are flipped by rewriting a file; the env var names that file.
|
||||||
|
try:
|
||||||
|
return open(os.environ.get(var, "")).read().strip()
|
||||||
|
except OSError:
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def log(line):
|
||||||
|
with open(logf, "a") as f:
|
||||||
|
f.write(line + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
class H(BaseHTTPRequestHandler):
|
||||||
|
def log_message(self, *a):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def send(self, code, body, as_json=False):
|
||||||
|
data = json.dumps(body).encode() if as_json else str(body).encode()
|
||||||
|
self.send_response(code)
|
||||||
|
self.send_header("Content-Type", "application/json" if as_json else "text/xml")
|
||||||
|
self.send_header("Content-Length", str(len(data)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(data)
|
||||||
|
|
||||||
|
def jf_authed(self):
|
||||||
|
# Jellyfin 12.4 accepts the quoted MediaBrowser form ONLY: a bare
|
||||||
|
# X-Emby-Token reads as unauthenticated here, exactly as it does live.
|
||||||
|
return self.headers.get("Authorization") == 'MediaBrowser Token="stubkey"'
|
||||||
|
|
||||||
|
def do_GET(self):
|
||||||
|
path = self.path.split("?")[0]
|
||||||
|
log("GET %s auth=%s" % (self.path, "yes" if self.headers.get("Authorization") else "no"))
|
||||||
|
if path == "/System/Info":
|
||||||
|
if self.jf_authed():
|
||||||
|
return self.send(200, {"ServerName": "stub", "Version": "12.4.0"}, True)
|
||||||
|
return self.send(401, "Unauthorized")
|
||||||
|
if path == "/ScheduledTasks":
|
||||||
|
with open(os.environ["POLL"]) as f:
|
||||||
|
n = int(f.read().strip() or 0)
|
||||||
|
with open(os.environ["POLL"], "w") as f:
|
||||||
|
f.write(str(n + 1))
|
||||||
|
state = read("SCAN", "Idle")
|
||||||
|
flip = read("FLIP", "")
|
||||||
|
if flip and n + 1 >= int(flip):
|
||||||
|
state = "Idle"
|
||||||
|
return self.send(200, [{"Name": "Audio Normalization", "State": "Running"},
|
||||||
|
{"Name": "Scan Media Library", "State": state,
|
||||||
|
"EndTimeUtc": read("ENDED", "") or None}], True)
|
||||||
|
return self.send(404, "nope")
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
path = self.path.split("?")[0]
|
||||||
|
log("POST %s auth=%s" % (self.path, "yes" if self.headers.get("Authorization") else "no"))
|
||||||
|
if path == "/Library/Refresh":
|
||||||
|
if not self.jf_authed():
|
||||||
|
return self.send(401, "Unauthorized")
|
||||||
|
if read("MODE", "ok") == "busy":
|
||||||
|
return self.send(429, "too many requests")
|
||||||
|
return self.send(204, "")
|
||||||
|
if path.endswith("/library/refresh") or "/library/sections/" in path:
|
||||||
|
if self.headers.get("X-Plex-Token") != "stubplextoken":
|
||||||
|
return self.send(401, "Unauthorized")
|
||||||
|
return self.send(204, "")
|
||||||
|
return self.send(404, "nope")
|
||||||
|
|
||||||
|
|
||||||
|
ThreadingHTTPServer(("127.0.0.1", port), H).serve_forever()
|
||||||
|
PY
|
||||||
|
export SCAN="$TMP/scan-state" MODE="$TMP/refresh-mode" POLL="$TMP/polls"
|
||||||
|
export FLIP="$TMP/flip-after"
|
||||||
|
export ENDED="$TMP/scan-ended"
|
||||||
|
export PODMAN_LOG="$TMP/podman.log" CONTAINER_FS="$TMP/container" CONTAINER=plex
|
||||||
|
: > "$PODMAN_LOG"
|
||||||
|
python3 "$TMP/stub.py" "$PORT" "$TMP/requests.log" > "$TMP/stub.out" 2>&1 &
|
||||||
|
STUB_PID=$!
|
||||||
|
for _ in $(seq 1 50); do
|
||||||
|
curl -s -o /dev/null "http://127.0.0.1:$PORT/System/Info" && break
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
teardown() {
|
||||||
|
[ -n "${STUB_PID:-}" ] && kill "$STUB_PID" 2>/dev/null || true
|
||||||
|
rm -rf "$TMP"
|
||||||
|
}
|
||||||
|
|
||||||
|
plex_db() { # plex_db <path> <scanned_at epoch>
|
||||||
|
python3 - "$1" "$2" <<'PY'
|
||||||
|
import sqlite3, sys
|
||||||
|
db = sqlite3.connect(sys.argv[1])
|
||||||
|
db.execute("CREATE TABLE section_locations (id INTEGER PRIMARY KEY, library_section_id INT,"
|
||||||
|
" root_path TEXT, available INT, scanned_at INT)")
|
||||||
|
db.execute("INSERT INTO section_locations VALUES (1, 1, '/mnt/entropy/Movies', 1, ?)",
|
||||||
|
(int(sys.argv[2]),))
|
||||||
|
db.commit()
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- jellyfin
|
||||||
|
|
||||||
|
@test "the jellyfin driver authenticates as MediaBrowser Token, never argv" {
|
||||||
|
run python3 "$JF" trigger --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
grep -q '^POST /Library/Refresh' "$TMP/requests.log"
|
||||||
|
! grep -q "stubkey" "$TMP/requests.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a server that rejects the auth form is a failure, not a success" {
|
||||||
|
# The stub 401s anything but the quoted form; the driver must report it
|
||||||
|
# rather than exit 0 — a green trigger over a 401 is a lie.
|
||||||
|
printf 'wrongkey\n' > "$TMP/bad-key"
|
||||||
|
run python3 "$JF" trigger --url "http://127.0.0.1:$PORT" --key-file "$TMP/bad-key"
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"401"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a trigger during a running scan is refused with no POST at all" {
|
||||||
|
echo Running > "$TMP/scan-state"
|
||||||
|
run python3 "$JF" trigger --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key"
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"running"* ]]
|
||||||
|
! grep -q '^POST /Library/Refresh' "$TMP/requests.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "--force triggers anyway while a scan runs" {
|
||||||
|
echo Running > "$TMP/scan-state"
|
||||||
|
run python3 "$JF" trigger --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key" --force
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
grep -q '^POST /Library/Refresh' "$TMP/requests.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a busy server is waited out with a bounded number of attempts" {
|
||||||
|
echo busy > "$TMP/refresh-mode"
|
||||||
|
run python3 "$JF" trigger --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key" \
|
||||||
|
--retries 2 --retry-sleep 0.1
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"429"* ]]
|
||||||
|
posts=$(grep -c '^POST /Library/Refresh' "$TMP/requests.log")
|
||||||
|
[ "$posts" -le 3 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "status reports the scan task and recent media writes" {
|
||||||
|
python3 - "$TMP/jellyfin.db" <<'PY'
|
||||||
|
import sqlite3, sys, time
|
||||||
|
db = sqlite3.connect(sys.argv[1])
|
||||||
|
db.execute("CREATE TABLE BaseItems (Id TEXT, DateModified TEXT)")
|
||||||
|
now = int(time.time())
|
||||||
|
for i in range(3):
|
||||||
|
db.execute("INSERT INTO BaseItems VALUES (?, datetime(?, 'unixepoch'))", (str(i), now - 30))
|
||||||
|
db.execute("INSERT INTO BaseItems VALUES ('old', datetime(?, 'unixepoch'))", (now - 86400,))
|
||||||
|
db.commit()
|
||||||
|
PY
|
||||||
|
run python3 "$JF" status --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key" \
|
||||||
|
--db "$TMP/jellyfin.db" --window 5
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
python3 -c 'import json,sys; d=json.loads(sys.argv[1]); assert d["scan"] is False, d; assert d["recentMedia"] == 3, d' "$output"
|
||||||
|
echo Running > "$TMP/scan-state"
|
||||||
|
run python3 "$JF" status --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key" \
|
||||||
|
--db "$TMP/jellyfin.db" --window 5
|
||||||
|
python3 -c 'import json,sys; assert json.loads(sys.argv[1])["scan"] is True' "$output"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "wait returns only after the scan reads quiet repeatedly" {
|
||||||
|
python3 - "$TMP/jellyfin.db" <<'PY'
|
||||||
|
import sqlite3, sys
|
||||||
|
db = sqlite3.connect(sys.argv[1])
|
||||||
|
db.execute("CREATE TABLE BaseItems (Id TEXT, DateModified TEXT)")
|
||||||
|
db.commit()
|
||||||
|
PY
|
||||||
|
echo Running > "$TMP/scan-state"
|
||||||
|
echo 4 > "$TMP/flip-after"
|
||||||
|
run python3 "$JF" wait --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key" \
|
||||||
|
--db "$TMP/jellyfin.db" --interval 0 --require-quiet 2 --timeout 30
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "${lines[-1]}" == *'"quiet": 2'* ]]
|
||||||
|
# every read before poll 4 was busy, so no earlier line may claim quiet
|
||||||
|
[ "$(printf '%s\n' "$output" | grep -c '"quiet": 0')" -ge 3 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "wait gives up out loud when the library never settles" {
|
||||||
|
python3 - "$TMP/jellyfin.db" <<'PY'
|
||||||
|
import sqlite3, sys
|
||||||
|
db = sqlite3.connect(sys.argv[1])
|
||||||
|
db.execute("CREATE TABLE BaseItems (Id TEXT, DateModified TEXT)")
|
||||||
|
db.commit()
|
||||||
|
PY
|
||||||
|
echo Running > "$TMP/scan-state"
|
||||||
|
run python3 "$JF" wait --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key" \
|
||||||
|
--db "$TMP/jellyfin.db" --interval 0 --timeout 0.2
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"still busy"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "wait refuses a scan that never started instead of calling the library settled" {
|
||||||
|
python3 - "$TMP/jellyfin.db" <<'PY'
|
||||||
|
import sqlite3, sys
|
||||||
|
db = sqlite3.connect(sys.argv[1])
|
||||||
|
db.execute("CREATE TABLE BaseItems (Id TEXT, DateModified TEXT)")
|
||||||
|
db.commit()
|
||||||
|
PY
|
||||||
|
echo Idle > "$TMP/scan-state"
|
||||||
|
run python3 "$JF" wait --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key" \
|
||||||
|
--db "$TMP/jellyfin.db" --interval 0.05 --start-window 0.4 --timeout 60
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"never started"* ]]
|
||||||
|
[[ "$output" == *"cancels it"* ]]
|
||||||
|
# a watcher that never saw the scan may not hand back a quiet count
|
||||||
|
! grep -Eq '"quiet": [1-9]' <<<"$output"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "status --after refuses to certify a scan that predates the moves" {
|
||||||
|
python3 - "$TMP/jellyfin.db" <<'PY'
|
||||||
|
import sqlite3, sys
|
||||||
|
db = sqlite3.connect(sys.argv[1])
|
||||||
|
db.execute("CREATE TABLE BaseItems (Id TEXT, DateModified TEXT)")
|
||||||
|
db.commit()
|
||||||
|
PY
|
||||||
|
owed=$(python3 -c 'import time; print(int(time.time()) - 7200)')
|
||||||
|
future=$(python3 -c 'import time; print(int(time.time()) + 7200)')
|
||||||
|
date -u -d '3 hours ago' +%Y-%m-%dT%H:%M:%SZ > "$TMP/scan-ended"
|
||||||
|
run python3 "$JF" status --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key" \
|
||||||
|
--db "$TMP/jellyfin.db" --after "$future"
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"still owed"* ]]
|
||||||
|
date -u +%Y-%m-%dT%H:%M:%SZ > "$TMP/scan-ended"
|
||||||
|
run python3 "$JF" status --url "http://127.0.0.1:$PORT" --key-file "$TMP/jf-key" \
|
||||||
|
--db "$TMP/jellyfin.db" --after "$owed"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *'"scannedAfter": true'* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a missing jellyfin credential file is refused by name" {
|
||||||
|
run python3 "$JF" trigger --url "http://127.0.0.1:$PORT" --key-file "$TMP/nope"
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"$TMP/nope"* ]]
|
||||||
|
! grep -q '^POST' "$TMP/requests.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------- plex
|
||||||
|
|
||||||
|
@test "the plex driver carries the token from Preferences.xml into a header" {
|
||||||
|
run env PATH="$TMP/bin:$PATH" bash "$PX" trigger --library 1 \
|
||||||
|
--url "http://127.0.0.1:$PORT" --prefs "$TMP/prefs.xml"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"refresh accepted (204)"* ]]
|
||||||
|
grep -q "^POST /library/sections/1/refresh" "$TMP/requests.log"
|
||||||
|
# the token reaches PMS as a header only: never in a URL, never in an argv
|
||||||
|
! grep -q "stubplextoken" "$TMP/requests.log"
|
||||||
|
! grep -q "stubplextoken" "$TMP/podman.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a plex trigger with no answer is a warming server, retried and named" {
|
||||||
|
run env PATH="$TMP/bin:$PATH" bash "$PX" trigger --library 1 \
|
||||||
|
--url "http://127.0.0.1:1" --prefs "$TMP/prefs.xml" --tries 2 --warm-sleep 0.1
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"000"* ]]
|
||||||
|
[[ "$output" == *"warming"* ]] || [[ "$output" == *"never answered"* ]]
|
||||||
|
[ "$(grep -c '^podman cp' "$TMP/podman.log")" -eq 2 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a plex trigger with no token in Preferences.xml fails before any request" {
|
||||||
|
printf '<Preferences/>\n' > "$TMP/prefs-empty.xml"
|
||||||
|
run env PATH="$TMP/bin:$PATH" bash "$PX" trigger --library 1 \
|
||||||
|
--url "http://127.0.0.1:$PORT" --prefs "$TMP/prefs-empty.xml"
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"PlexOnlineToken"* ]]
|
||||||
|
! grep -q '^POST' "$TMP/requests.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "plex status reads the scan epoch from the database" {
|
||||||
|
plex_db "$TMP/library.db" 1000
|
||||||
|
run bash "$PX" status --library 1 --db "$TMP/library.db"
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[[ "$output" == *"section=1 scanned_at=1000"* ]]
|
||||||
|
[[ "$output" == *"/mnt/entropy/Movies"* ]]
|
||||||
|
run bash "$PX" status --library 1 --db "$TMP/library.db" --after 2000
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"still owed"* ]]
|
||||||
|
run bash "$PX" status --library 1 --db "$TMP/library.db" --after 500
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a plex status with no readable database is refused by name" {
|
||||||
|
run bash "$PX" status --library 1 --db "$TMP/no-library.db"
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"$TMP/no-library.db"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "plex status names a section that has no location" {
|
||||||
|
plex_db "$TMP/library.db" 1000
|
||||||
|
run bash "$PX" status --library 77 --db "$TMP/library.db"
|
||||||
|
[ "$status" -ne 0 ]
|
||||||
|
[[ "$output" == *"section 77"* ]]
|
||||||
|
}
|
||||||
@@ -799,14 +799,13 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names),
|
|||||||
skip "franksdank/alloy image not present on this box"
|
skip "franksdank/alloy image not present on this box"
|
||||||
fi
|
fi
|
||||||
local sandbox="$BATS_TEST_TMPDIR/opt" cfg="$BATS_TEST_TMPDIR/broken.alloy"
|
local sandbox="$BATS_TEST_TMPDIR/opt" cfg="$BATS_TEST_TMPDIR/broken.alloy"
|
||||||
sed 's/GRAFANA_HOST_LABEL/dankclaw/g' shared/alloy/config.alloy > "$cfg"
|
sed 's/GRAFANA_HOST_LABEL/dankclaw/g' "$REPO_ROOT/shared/alloy/config.alloy" > "$cfg"
|
||||||
printf 'bogus_attr = "x"\n' >> "$cfg"
|
printf 'bogus_attr = "x"\n' >> "$cfg"
|
||||||
! (
|
! (
|
||||||
cd "$REPO_ROOT"
|
cd "$REPO_ROOT"
|
||||||
eval "$(sed -n '/^copy_alloy_config()/,/^}/p' scripts/deploy-remote.sh)"
|
eval "$(sed -n '/^copy_alloy_config()/,/^}/p' scripts/deploy-remote.sh)"
|
||||||
run_privileged() { "$@"; }
|
run_privileged() { "$@"; }
|
||||||
cp() { if [ "$1" = "$cfg" ]; then return 1; fi; command cp "$@"; }
|
export TARGET=dankclaw ALLOY_CONF_ROOT="$sandbox" ALLOY_CONFIG_SRC="$cfg"
|
||||||
export TARGET=dankclaw ALLOY_CONF_ROOT="$sandbox"
|
|
||||||
copy_alloy_config
|
copy_alloy_config
|
||||||
)
|
)
|
||||||
[ ! -f "$sandbox/config/config.alloy" ]
|
[ ! -f "$sandbox/config/config.alloy" ]
|
||||||
@@ -820,7 +819,7 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names),
|
|||||||
skip "franksdank/alloy image not present on this box"
|
skip "franksdank/alloy image not present on this box"
|
||||||
fi
|
fi
|
||||||
local cfg="$BATS_TEST_TMPDIR/rendered.alloy"
|
local cfg="$BATS_TEST_TMPDIR/rendered.alloy"
|
||||||
sed 's/GRAFANA_HOST_LABEL/dankclaw/g' shared/alloy/config.alloy > "$cfg"
|
sed 's/GRAFANA_HOST_LABEL/dankclaw/g' "$REPO_ROOT/shared/alloy/config.alloy" > "$cfg"
|
||||||
podman run --rm -v "$cfg:/c.alloy:ro" --entrypoint /usr/bin/alloy \
|
podman run --rm -v "$cfg:/c.alloy:ro" --entrypoint /usr/bin/alloy \
|
||||||
localhost/franksdank/alloy:latest validate /c.alloy
|
localhost/franksdank/alloy:latest validate /c.alloy
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user