Merge pull request 'feat(fleet): give the media servers an I/O share that actually exists' (#110) from feat/media-io-weight into main

dankserver/host/systemd/user.slice.d/io-accounting.conf

@@ -0,0 +1,14 @@
+# The io controller is what makes IOWeight= mean anything, and systemd does not
+# enable it below `user.slice` by default: with io accounting off, `io` never
+# appears in `user.slice/cgroup.subtree_control`, so no rootless container can
+# have an `io.weight` file and `IOWeight=` in a quadlet is silently inert.
+# Measured on Dankserver 2026-10-31: setting this property flipped
+# `user.slice/cgroup.subtree_control` from `cpu memory pids` to `cpu io memory
+# pids` immediately, with nothing restarted.
+#
+# This is half the fix; the other half is delegating io to user@1000.service
+# (user@1000.service.d/io-delegate.conf). Both must be in place before a reboot,
+# because the delegation is read when the user manager starts. Law and the
+# verification command: [[Quadlet Lifecycle]].
+[Slice]
+IOAccounting=yes

dankserver/host/systemd/user@1000.service.d/io-delegate.conf

@@ -0,0 +1,12 @@
+# The system manager gives the user manager cpu/memory/pids by default (systemd
+# 259: `systemctl show user@1000.service -p DelegateControllers` → `cpu memory
+# pids`). Without `io` in that set no `io.weight` file can exist anywhere below
+# `user.slice`, so `IOWeight=` in a rootless quadlet is silently inert — the same
+# class of trap as `RequiresMountsFor=`. Granting the controller here is what
+# makes the media servers' I/O share real.
+#
+# Inert until `user@1000.service` starts again, i.e. the next reboot (restarting
+# it would kill every rootless container on the host). Verify after the reboot
+# with the check in [[Quadlet Lifecycle]].
+[Service]
+DelegateControllers=cpu memory pids io

dankserver/jellyfin/quadlet/jellyfin.container

@@ -61,5 +61,13 @@ ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /var/lib/medi
 Restart=always
 RestartSec=5
 
+# A library scan reads the array for minutes on end. Weighted LOW (default is
+# 100) so that when the array is contended a scan loses to playback and to every
+# other container this user runs. Proportional share, not a cap: it changes
+# nothing while the disk is idle. 10 is a convention — the requirement is only
+# that it sit far below 100 — and it is inert until user@1000.service is
+# delegated the io controller ([[Quadlet Lifecycle]]).
+IOWeight=10
+
 [Install]
 WantedBy=default.target

dankserver/plex/quadlet/plex.container

@@ -26,5 +26,11 @@ ExecStartPre=/home/dank/serverAppConfigs/scripts/wait-for-mount.sh /var/lib/medi
 Restart=always
 RestartSec=5
 
+# See the jellyfin unit for the reasoning: a section scan gets a LOW weight
+# (default 100) so a walk of 50k files loses the array to playback and to every
+# other container under this user. Inert until user@1000.service holds the io
+# controller ([[Quadlet Lifecycle]]).
+IOWeight=10
+
 [Install]
 WantedBy=default.target

scripts/deploy-remote.sh

@@ -1208,14 +1208,47 @@ ensure_media_view() {
     # array below stays read-only and every sidecar Jellyfin writes lands on the
     # NVMe.
     # Raise it BEFORE either container starts — a start against a bare view
-    # directory would let a scan clear the library database. The units also carry
-    # ConditionPathIsMountPoint, so a missing view skips the start rather than
-    # damaging anything. A relayout must be followed by `media-view.sh reap`
+    # directory would let a scan clear the library database. Each unit gates on
+    # scripts/wait-for-mount.sh, which waits and then refuses the start. A
+    # relayout must be followed by `media-view.sh reap`
     # (see the Media Relayout page), which this is not.
     run_privileged bash scripts/media-view.sh install
     bash scripts/media-view.sh verify
 }
 
+ensure_user_io_delegate() {
+    # The media servers ask for an I/O share (IOWeight= in their quadlets) so a
+    # library scan cannot starve playback or the array's other reader. Two system
+    # drop-ins are what make that request real, and both are host state, so the
+    # deployer places them:
+    #   user.slice.d/io-accounting.conf        io enters user.slice's subtree
+    #   user@1000.service.d/io-delegate.conf   the user manager gets the controller
+    # Without the first, `io` never appears in user.slice/cgroup.subtree_control,
+    # so no container under it can have an io.weight file at all; without the
+    # second the user manager never asks for the controller. systemd hands
+    # user@.service cpu/memory/pids by default and drops a line it cannot honour
+    # without a word — IOWeight= on its own is inert config.
+    local src dst
+    for src in \
+        dankserver/host/systemd/user.slice.d/io-accounting.conf \
+        dankserver/host/systemd/user@1000.service.d/io-delegate.conf; do
+        dst=/etc/systemd/system/${src#dankserver/host/systemd/}
+        run_privileged mkdir -p "$(dirname "$dst")"
+        if ! run_privileged cmp -s "$src" "$dst"; then
+            run_privileged install -m 644 "$src" "$dst"
+            run_privileged systemctl daemon-reload
+        fi
+    done
+    # Accounting lands immediately; the delegation is read when user@1000.service
+    # starts, and restarting that unit would kill every rootless container on this
+    # host. So the share becomes real at the next reboot — say so out loud rather
+    # than let a quadlet line read as live config.
+    if ! systemctl show user@1000.service -p DelegateControllers | grep -qw io; then
+        echo "NOTE: io is not delegated to user@1000.service yet - IOWeight= is"
+        echo "NOTE: inert until the next reboot (see [[Quadlet Lifecycle]])"
+    fi
+}
+
 ensure_provider_keys() {
     # Jellyfin's metadata-provider keys live in the container-owned plugin
     # configuration under the data root, never in git; scripts/jellyfin-providers.sh
@@ -1237,6 +1270,7 @@ ensure_provider_keys() {
 
 deploy_jellyfin() {
     ensure_media_view
+    ensure_user_io_delegate
     ensure_provider_keys
     cp "dankserver/jellyfin/quadlet/jellyfin.container" "$quadlet_unit_dir/"
     systemctl --user daemon-reload
@@ -1326,6 +1360,7 @@ deploy_copyparty() {
 
 deploy_plex() {
     ensure_media_view
+    ensure_user_io_delegate
     cp "dankserver/plex/quadlet/plex.container" "$quadlet_unit_dir/"
     systemctl --user daemon-reload
     if ! systemctl --user is-active -q plex.service; then

serverAppConfigs-docs/Architecture/E2E Suite.md

@@ -100,3 +100,12 @@ aborts with `DEPLOY INCOMPLETE: e2e gate FAILED` when it does not pass
    hand-off list (`HANDOFFS`) instead of a single host. The guard contract is
    untouched: challenge cleared, no authentik `uidField`, status < 500, and every
    host NOT in the list still has to land on its own vhost.
+7. **The runner's resolver is part of the suite.** A 2026-10-31 deploy of one
+   Dankserver quadlet failed the gate with 21 reds spread over SSO, homepage RBAC
+   and every vhost crawl — precisely the specs that navigate to an absolute
+   `*.franksdank.cloud` name, while every spec on the relative `baseURL` stayed
+   green. The error was `net::ERR_NAME_NOT_RESOLVED`: the workstation had lost its
+   wired link, and on the fallback its resolver was no longer pinned to Pi-hole.
+   Nothing on the fleet was wrong — the same suite re-run minutes later passed
+   37/37. Before a mass red gets blamed on the guard chain, run
+   `resolvectl query grafana.franksdank.cloud` on the box that runs the gate.

serverAppConfigs-docs/Architecture/Quadlet Lifecycle.md

@@ -167,6 +167,43 @@ Gate: `tests/quadlet_image_pins.bats`. Where upstream publishes no version
 tags (copyparty-ac stops at 1.19.15), pin `:latest@sha256:` and name the
 version the config label carries in the comment beside it.
 
+## 4b. Resource control needs host permission (measured 2026-10-31, Dankserver)
+
+`IOWeight=` in a rootless quadlet does nothing on its own, and nothing says so.
+The system manager delegates `cpu memory pids` to `user@.service` and **not** `io`;
+systemd drops a resource line it cannot honour **without a word** — the generator
+still emits `IOWeight=10`, the container still starts and answers its health check,
+and the cgroup simply has no `io.weight` file. Same family as
+`RequiresMountsFor=` (inert in the user manager) and `ConditionPathIsMountPoint=`
+(fail-dead): the line is accepted, the unit is happy, the setting is dead.
+
+Two system drop-ins make it real, both placed by `ensure_user_io_delegate` in
+`scripts/deploy-remote.sh` (repo copies under `dankserver/host/systemd/`):
+
+| File | Effect |
+|---|---|
+| `user.slice.d/io-accounting.conf` → `[Slice] IOAccounting=yes` | puts `io` into `user.slice/cgroup.subtree_control`; lands immediately, no restart |
+| `user@1000.service.d/io-delegate.conf` → `[Service] DelegateControllers=cpu memory pids io` | the user manager can only request controllers it was delegated; read when it starts, so it lands on **the next reboot** (restarting it kills every rootless container on the host) |
+
+Read the chain in this order — each line is the failure mode of the one after it:
+
+```bash
+cat /sys/fs/cgroup/user.slice/cgroup.subtree_control                    # wants: io
+systemctl show user@1000.service -p DelegateControllers                 # wants: io
+cat /sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/app.slice/\<unit\>.service/libpod-payload-*/io.weight
+```
+
+`io` missing from the first → the accounting drop-in is absent. Missing from the
+second → the host has not rebooted since the delegation landed (the deployer prints
+`NOTE: io is not delegated …` for exactly this). No file at the third → that unit
+carries no weight.
+
+Two limits that survive the fix: a container's weight covers **everything** in its
+cgroup, so a library scan and a playback stream share one weight; and the comparison
+is made *inside* `user.slice`, so a rootless unit's weight cannot push it below a
+root unit (`frank-archon`). Shaping that pairing is the weight of `user.slice`
+itself — a fleet decision, not a unit line.
+
 ## 5. Rootless Quadlet log retrieval
 
 Step 0 — establish ownership: `systemctl --user status <name>.service`

serverAppConfigs-docs/Infrastructure/Pi-hole.md

@@ -77,7 +77,7 @@ Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them.
 | Port 53 `bind: permission denied` crash loop after a reboot | sysctl conflict: the value must be 53 at boot, and remember `/etc/sysctl.conf` is applied **after** all of `sysctl.d/`. Redeploy pihole — `deploy_pihole` reinstalls the drop-in, neutralizes conflicting drop-ins, and comments out a stale `/etc/sysctl.conf` line |
 | Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`), and FTL serves it. Use the FQDN, or add an `/etc/hosts` entry on the client |
 | Every `franksdank.cloud` name is NXDOMAIN for LAN clients while external names still answer | FTL holds no local records: `getent hosts <name>` on Dankclaw comes back empty → `podman exec pihole pihole restartdns reload` (the 2026-10-19 pitfall under **Records**) |
-| One client gets nothing while every other client is fine | FTL rate-limiting that client: `podman exec pihole grep Rate-limiting /var/log/pihole/FTL.log`. A `10.0.2.x` client address is a pasta container's DNS forwarder, not a machine |
+| One client gets nothing while every other client is fine | FTL rate-limiting that client: `podman exec pihole grep Rate-limiting /var/log/pihole/FTL.log`. A `10.0.2.x` client address is a pasta container's DNS forwarder, not a machine — and **every** rootless container on that host shares the one address, so a single chatty container throttles DNS for all of them (measured 2026-10-31: 8 throttle events in a day, all `10.0.2.100`) |
 
 ## See Also
 

tests/dankserver_quadlet.bats

@@ -106,3 +106,24 @@ load helpers
     # env file, not an Environment= line.
     ! grep -qE '^Environment=CUSTOM_(USER|PASSWORD)=' "$unit" || return 1
 }
+
+@test "a quadlet that asks for an I/O share ships the host permission that makes it real" {
+    # IOWeight= is silently inert unless the system manager both enables io for
+    # user.slice and delegates it to the user manager: systemd drops a resource
+    # line it cannot honour without a word and the container still starts healthy.
+    # So an in-repo weight is only honest together with both drop-ins AND with a
+    # deployer that places them. Measured 2026-10-31; verification chain in
+    # Architecture/Quadlet Lifecycle 4b.
+    if grep -lq '^IOWeight=' "$REPO_ROOT"/dankserver/*/quadlet/*.container; then
+        asked=1
+    else
+        skip "no dankserver quadlet asks for an I/O share"
+    fi
+    grep -q '^IOAccounting=yes$' \
+        "$REPO_ROOT/dankserver/host/systemd/user.slice.d/io-accounting.conf" || return 1
+    grep -q '^DelegateControllers=.* io' \
+        "$REPO_ROOT/dankserver/host/systemd/user@1000.service.d/io-delegate.conf" || return 1
+    for f in user.slice.d/io-accounting.conf 'user@1000.service.d/io-delegate.conf'; do
+        grep -q "dankserver/host/systemd/$f" "$REPO_ROOT/scripts/deploy-remote.sh" || return 1
+    done
+}