Files
serverAppConfigs/scripts/deploy-remote.sh
T
Frank 69f87c8483 feat(llm): rename inference services dropping 'default' from names
frank-default-large -> frank-large (Dankclaw :7770), frank-default-small ->
frank-small (Dankserver :7770), frank-default-sightsound -> frank-npu
(Dankclaw :7771). Units, env files, serve scripts, deploy-plan mappings,
deploy-remote dispatch (stop/disable old units + relink + enable new), and the
test seams move together. Retired frank-default-npu.* artifacts moved to
scripts/.AGENTTRASH/ (tracked-in-history; ignored on disk).

Served aliases become frank-large / frank-small via --alias; the NPU endpoint
serves the whole FLM registry, so its rename changes no served tags.
2026-09-03 16:15:43 -07:00

1104 lines
46 KiB
Bash
Executable File

#!/usr/bin/env bash
# deploy-remote.sh — apply an explicit service plan on one target host.
#
# This script runs on the target checkout. It deliberately does not infer a
# plan, pull Git, or restart an unrelated service. deploy.sh computes the plan
# from the pushed revision and passes the selected service names here.
# Per-service revisions live in /var/lib/serverAppConfigs/deploy-state (or
# DEPLOY_STATE_DIR), outside the checkout, so an unselected change remains
# pending until its service is explicitly deployed.
set -euo pipefail
TARGET="${1:-}"
REVISION="${2:-}"
shift 2 || true
FROM_REVISION=""
PENDING=0
SERVICES=()
usage() {
cat <<'EOF'
Usage: deploy-remote.sh <dankclaw|dankserver|frankbook-pro> <git-revision> [--pending] [service]...
The service list is explicit. An empty list is a successful no-op.
EOF
}
while [[ "$#" -gt 0 ]]; do
case "$1" in
--from-revision)
[[ "$#" -ge 2 ]] || { echo "--from-revision requires a revision" >&2; exit 2; }
FROM_REVISION="$2"
shift 2
;;
--pending)
PENDING=1
shift
;;
--help|-h)
usage
exit 0
;;
*)
SERVICES+=("$1")
shift
;;
esac
done
REMOTE_DIR="${REMOTE_DIR:-/home/dank/serverAppConfigs}"
STATE_DIR="${DEPLOY_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/serverAppConfigs/deploy-state}"
run_privileged() {
if [[ -n "${FRANK_SUDO_PW:-}" ]]; then
printf '%s\n' "$FRANK_SUDO_PW" | sudo -S -- "$@"
else
sudo -- "$@"
fi
}
if [[ "$TARGET" != dankclaw && "$TARGET" != dankserver && "$TARGET" != frankbook-pro ]]; then
usage >&2
exit 2
fi
if [[ -z "$REVISION" ]]; then
usage >&2
exit 2
fi
if [[ "$REVISION" == *[!0-9a-fA-F]* ]]; then
echo "invalid Git revision: $REVISION" >&2
exit 2
fi
if ! git -C "$REMOTE_DIR" cat-file -e "$REVISION^{commit}" 2>/dev/null; then
echo "revision is not a commit in the target checkout: $REVISION" >&2
exit 2
fi
cd "$REMOTE_DIR"
# Keep the state directory outside Git. The file is written only after a
# handler completes successfully; an interrupted handler remains pending.
mkdir -p "$STATE_DIR"
service_is_allowed() {
local service="$1"
case "$TARGET:$service" in
dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\
dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
dankclaw:comfyui|dankclaw:hermes|\
dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|\
dankclaw:authentik-postgres|dankclaw:authentik-server|\
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|\
dankclaw:frank-large|dankclaw:frank-npu|\
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
dankserver:frank-small|\
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
frankbook-pro:readarr)
return 0
;;
*)
return 1
;;
esac
}
mark_status() {
local service="$1" status="$2" tmp
tmp="$(mktemp "$STATE_DIR/.${service}.XXXXXX")"
printf 'revision=%s\nstatus=%s\n' "$REVISION" "$status" > "$tmp"
mv -f -- "$tmp" "$STATE_DIR/$service"
}
mark_deployed() {
mark_status "$1" deployed
}
state_revision() {
local service="$1" line value
if [[ -r "$STATE_DIR/$service" ]]; then
while IFS= read -r line; do
case "$line" in
revision=*) value="${line#revision=}" ;;
esac
done < "$STATE_DIR/$service"
printf '%s' "${value:-}"
fi
}
state_status() {
local service="$1" line value=pending
if [[ -r "$STATE_DIR/$service" ]]; then
while IFS= read -r line; do
case "$line" in
status=*) value="${line#status=}" ;;
esac
done < "$STATE_DIR/$service"
fi
printf '%s' "$value"
}
register_changed_services() {
local base="$1" service recorded status
[[ -n "$base" ]] || return 0
while IFS= read -r service; do
[[ -n "$service" ]] || continue
recorded="$(state_revision "$service")"
status="$(state_status "$service")"
if [[ "$status" != deployed || "$recorded" != "$REVISION" ]]; then
mark_status "$service" pending
fi
done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
--base "$base" --head "$REVISION")
}
add_changed_services() {
local base="$1"
local service
[[ -n "$base" ]] || return 0
while IFS= read -r service; do
[[ -n "$service" ]] || continue
SERVICES+=("$service")
done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
--base "$base" --head "$REVISION")
}
add_pending_services() {
local service recorded status changed
for state_file in "$STATE_DIR"/*; do
[[ -f "$state_file" ]] || continue
service="${state_file##*/}"
recorded="$(state_revision "$service")"
status="$(state_status "$service")"
if [[ "$status" != deployed ]]; then
# A pending service was intentionally skipped or its handler
# failed. Retry it even when the checkout has not advanced.
SERVICES+=("$service")
continue
fi
[[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
if [[ "$TARGET" == dankclaw && "$service" == frank-small ]]; then continue; fi
if [[ "$TARGET" == dankserver && "$service" == frank-large ]]; then continue; fi
if [[ "$TARGET" == dankserver && "$service" == frank-npu ]]; then continue; fi
if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
--repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
echo "WARN: unable to compare state for $service; retaining it as pending" >&2
SERVICES+=("$service")
continue
fi
if grep -Fxq "$service" <<< "$changed"; then
SERVICES+=("$service")
fi
done
}
copy_searxng_config() {
run_privileged mkdir -p /opt/searxng/config
run_privileged cp shared/searxng/settings.yml /opt/searxng/config/settings.yml
run_privileged cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml
}
copy_prometheus_config() {
run_privileged mkdir -p /opt/prometheus/config
run_privileged cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml
}
copy_loki_config() {
run_privileged mkdir -p /opt/loki/config
run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
}
copy_alloy_config() {
run_privileged mkdir -p /opt/alloy/config
run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
}
render_caddyfile() {
local hash
hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
if [[ -z "$hash" ]]; then
echo "WARN: CADDY_BASICAUTH_HASH is not set; refusing to overwrite rendered Caddyfile" >&2
return 1
fi
run_privileged mkdir -p /home/dank/Containers/caddy
sed -e "s|REPLACE_WITH_HASH|$hash|g" dankclaw/Caddyfile > /tmp/serverAppConfigs.Caddyfile
run_privileged cp /tmp/serverAppConfigs.Caddyfile /home/dank/Containers/caddy/Caddyfile
}
deploy_homepage() {
if [[ "$TARGET" != dankclaw ]]; then
echo "homepage is owned by dankclaw; refusing to deploy it on $TARGET" >&2
return 2
fi
run_privileged mkdir -p /srv/homepage
run_privileged cp shared/homepage/index.html shared/homepage/admin.html shared/homepage/app.js \
shared/homepage/style.css shared/homepage/services.json /srv/homepage/
if [[ -d shared/homepage/fonts ]]; then
run_privileged cp -r shared/homepage/fonts /srv/homepage/
fi
local deployed_at
deployed_at="$(git log -1 --format=%cI)"
printf 'window.DEPLOYED_AT = "%s";\n' "$deployed_at" > /tmp/deployed.js
run_privileged cp /tmp/deployed.js /srv/homepage/deployed.js
run_privileged chmod -R a+rX /srv/homepage
run_privileged cp scripts/homepage-status.py /usr/local/bin/homepage-status.py
run_privileged cp scripts/homepage-status.service scripts/homepage-status.timer /etc/systemd/system/
run_privileged systemctl daemon-reload
run_privileged systemctl enable homepage-status.timer
run_privileged systemctl restart homepage-status.timer
run_privileged python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json
}
stage_legacy_gitea_caddy_state() {
local legacy_state="$STATE_DIR/gitea-caddy"
[[ -e "$legacy_state" ]] || return 0
local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
mkdir -p "$legacy_trash"
mv -- "$legacy_state" "$legacy_trash/gitea-caddy.state"
}
retire_legacy_gitea_caddy() {
# The old split proxy owns ports 80/443 on hosts upgraded in place. Stop
# and disable its generated unit before the canonical Caddy is started.
if systemctl --user is-active --quiet gitea-caddy.service 2>/dev/null; then
systemctl --user stop gitea-caddy.service
fi
if systemctl --user is-enabled --quiet gitea-caddy.service 2>/dev/null; then
# A Quadlet-generated unit may report "generated" without having an
# enablement symlink. Disable is therefore best-effort; moving the
# source out of the generator directory is the durable action.
systemctl --user disable gitea-caddy.service 2>/dev/null || true
fi
local unit_dir="$HOME/.config/containers/systemd"
local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
# Podman generators scan the unit directory recursively on this host, so
# leaving the retired file in systemd/.AGENTTRASH would regenerate it.
# Stage it outside the unit tree in the dated home trash instead.
for unit_file in "$unit_dir/gitea-caddy.container" \
"$unit_dir/.AGENTTRASH/gitea-caddy.container"; do
if [[ -e "$unit_file" ]]; then
mkdir -p "$legacy_trash"
mv -- "$unit_file" "$legacy_trash/"
fi
done
stage_legacy_gitea_caddy_state
systemctl --user daemon-reload
if podman container exists gitea-caddy; then
podman rm --force gitea-caddy
fi
}
deploy_caddy() {
if [[ "$TARGET" != dankclaw ]]; then
echo "caddy is owned by dankclaw; refusing to deploy it on $TARGET" >&2
return 2
fi
render_caddyfile
deploy_homepage
local cfhash oldhash image
cfhash="$(sha256sum dankclaw/caddy/Containerfile | cut -d' ' -f1)"
image=franksdank/caddy-porkbun
oldhash="$(podman image inspect "$image:latest" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
if [[ "$oldhash" != "$cfhash" ]]; then
podman build --no-cache --label "cfhash=$cfhash" -t "$image:latest" \
-f dankclaw/caddy/Containerfile .
fi
# Retire the old split proxy before claiming ports 80/443. This is
# idempotent and leaves the old unit recoverable in the target trash.
retire_legacy_gitea_caddy
# Caddy's env: Porkbun DNS-01 credentials from the checkout's .env. The
# live file holds secrets, so it exists only on the host; the quadlet
# reads it via EnvironmentFile.
mkdir -p /home/dank/Containers/caddy
if [[ ! -f /home/dank/Containers/caddy/env ]]; then
awk -F= '/^PORKBUN_API_KEY=|^PORKBUN_API_SECRET_KEY=/ { print }' \
dankclaw/.env > /home/dank/Containers/caddy/env
fi
local var
for var in PORKBUN_API_KEY PORKBUN_API_SECRET_KEY; do
grep -q "^${var}=." /home/dank/Containers/caddy/env \
|| { echo "caddy env missing ${var}; set it in dankclaw/.env" >&2; return 1; }
done
# Retire the bare compose container: all state (Caddyfile, certs, config)
# lives in host volumes, so removing the container loses nothing.
podman stop caddy >/dev/null 2>&1 || true
podman rm caddy >/dev/null 2>&1 || true
# install_quadlet runs daemon-reload, which re-runs podman's quadlet
# generator: it creates caddy.service AND the default.target.wants symlink
# from the unit's [Install] section (is-enabled reports "generated" —
# that is the healthy state for a quadlet; an explicit systemctl enable
# fails on generated units). Boot persistence comes from the generator,
# same as pihole/gitea/searxng/open-webui.
install_quadlet caddy dankclaw/caddy/quadlet caddy
systemctl --user restart caddy.service
if podman container exists caddy; then
local reloaded=0
for _ in 1 2 3 4 5; do
if podman exec caddy caddy reload --config /etc/caddy/Caddyfile \
--adapter caddyfile >/dev/null 2>&1; then
reloaded=1
echo "Caddy reloaded."
break
fi
sleep 3
done
if [[ "$reloaded" -ne 1 ]]; then
echo "Caddy reload failed; restarting only caddy."
podman restart caddy
fi
fi
}
deploy_compose_service() {
local service="$1"
if [[ "$TARGET" == dankserver ]]; then
# Dankserver's compose file is the source-of-truth inventory, but its
# production containers are individually systemd-managed/bare. Do not
# use compose up here: it can create a second container with a wrong
# name and can interfere with a running workload.
echo "WARN: $service is recorded in dankserver/compose.yaml; production container management is host-specific and no restart was attempted."
return 0
fi
case "$service" in
comfyui|hermes|hermes-dashboard|hermes-webui|prometheus|grafana|loki|alloy|node-exporter)
case "$service" in
comfyui|loki|alloy)
build_compose_image "$service"
;;
prometheus)
copy_prometheus_config
;;
loki)
copy_loki_config
;;
alloy)
copy_alloy_config
;;
esac
podman-compose -f dankclaw/compose.yaml up -d --no-deps "$service"
;;
*)
echo "unknown compose service: $service" >&2
return 2
;;
esac
}
build_compose_image() {
local service="$1" image cfhash oldhash containerfile
case "$service" in
comfyui)
image=comfyui-rocm:gfx1151
containerfile=dankclaw/Containerfile.comfyui
;;
loki|alloy)
image="franksdank/$service:latest"
containerfile="dankclaw/$service/Containerfile"
;;
*)
return 0
;;
esac
cfhash="$(sha256sum "$containerfile" | cut -d' ' -f1)"
oldhash="$(podman image inspect "$image" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
if [[ "$oldhash" != "$cfhash" ]]; then
podman build --no-cache --label "cfhash=$cfhash" -t "$image" -f "$containerfile" .
fi
}
quadlet_unit_dir="$HOME/.config/containers/systemd"
# dankserver's user manager had no systemd container generator dir before the
# quadlet migration; every handler that installs a unit must find it present.
mkdir -p "$quadlet_unit_dir"
install_quadlet() {
local service="$1"
local source_dir="$2"
local unit="$3"
mkdir -p "$quadlet_unit_dir"
cp "$source_dir/$unit.container" "$quadlet_unit_dir/"
if [[ -f "$source_dir/$unit.network" ]]; then
cp "$source_dir/$unit.network" "$quadlet_unit_dir/"
fi
systemctl --user daemon-reload
}
deploy_searxng() {
copy_searxng_config
install_quadlet searxng dankclaw/searxng/quadlet searxng
install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
install_quadlet gluetun dankclaw/searxng/quadlet gluetun
# The network is shared by all three units; install it even when only the
# SearXNG service was selected. These copies are user-owned Quadlet state.
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user start searxng-network.service
systemctl --user restart gluetun.service searxng-valkey.service searxng.service
}
deploy_gluetun() {
install_quadlet gluetun dankclaw/searxng/quadlet gluetun
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart gluetun.service searxng.service
}
deploy_searxng_valkey() {
install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart searxng-valkey.service searxng.service
}
deploy_open_webui() {
install_quadlet open-webui dankclaw/open-webui/quadlet open-webui
mkdir -p /home/dank/Containers/open-webui/data
systemctl --user daemon-reload
systemctl --user restart open-webui.service
}
deploy_pihole() {
# Rootless port 53 needs net.ipv4.ip_unprivileged_port_start <= 53, and it
# must hold at boot. The repo drop-in is the single source of truth.
# systemd-sysctl applies /etc/sysctl.d/*.conf (lexicographic), then
# /run/sysctl.d, /usr/lib/sysctl.d, and finally /etc/sysctl.conf — later
# files override earlier ones. On 2026-08-26 a later-sorted drop-in (=80)
# AND a stale line in /etc/sysctl.conf (=80) both reset the value after a
# reboot and took LAN DNS down. Neutralize every conflicting source:
# stage sysctl.d files that set a value above 53 in the home trash, and
# comment out an active line in /etc/sysctl.conf (idempotent).
run_privileged install -m 0644 dankclaw/pihole/sysctl/99-dnsmasq.conf \
/etc/sysctl.d/99-dnsmasq.conf
local sysctl_file sysctl_val sysctl_trash
while IFS= read -r sysctl_file; do
[[ "$sysctl_file" == */99-dnsmasq.conf ]] && continue
sysctl_val="$(sed -nE 's/^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=[[:space:]]*([0-9]+).*/\1/p' "$sysctl_file" | head -1)"
if [[ "$sysctl_val" =~ ^[0-9]+$ ]] && (( sysctl_val > 53 )); then
sysctl_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-sysctl-unprivileged-port"
mkdir -p "$sysctl_trash"
run_privileged mv "$sysctl_file" "$sysctl_trash/"
fi
done < <(grep -lE '^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=' /etc/sysctl.d/*.conf 2>/dev/null || true)
if [[ -f /etc/sysctl.conf ]] && grep -qE '^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=' /etc/sysctl.conf; then
run_privileged sed -i -E 's/^[[:space:]]*(net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=)/# \1 managed by dankclaw\/pihole\/sysctl\/99-dnsmasq.conf/' /etc/sysctl.conf
fi
run_privileged sysctl --system >/dev/null
mkdir -p /home/dank/Containers/pihole/etc-pihole /home/dank/Containers/pihole/etc-dockers
if [[ ! -f /home/dank/Containers/pihole/env ]]; then
cp dankclaw/pihole/env.example /home/dank/Containers/pihole/env
echo "WARN: created Pi-hole env from the example; set WEBPASSWORD before using its UI" >&2
fi
cp dankclaw/pihole/addn-hosts /home/dank/Containers/pihole/addn-hosts
mkdir -p "$quadlet_unit_dir"
cp dankclaw/pihole/quadlet/pihole.container "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart pihole.service
}
deploy_gitea_unit() {
local service="$1"
local unit="$2"
mkdir -p "$quadlet_unit_dir"
cp "dankclaw/gitea/quadlet/$unit.container" "$quadlet_unit_dir/"
[[ -f dankclaw/gitea/quadlet/gitea.network ]] && cp dankclaw/gitea/quadlet/gitea.network "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart "$service.service"
}
relink_unit() {
local service="$1"
run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service"
}
# Authentik runs as three boot-persistent Quadlet units (postgres -> server ->
# worker). The trio deploys as one unit: unit files are installed, stale
# compose-era containers holding the claimed names are retired, then the chain
# starts in dependency order with health gates between stages. Re-running the
# handler on an already-active stack is a deliberate no-op (no DB bounce).
wait_container_healthy() {
local name="$1" limit="$2" status="unknown"
for ((i = 1; i <= limit; i += 5)); do
status="$(podman inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null || echo missing)"
[[ "$status" == "healthy" ]] && {
echo "$name healthy after approximately ${i}s"
return 0
}
sleep 5
done
echo "FAIL: $name did not become healthy in ${limit}s (status: $status)" >&2
return 1
}
# --- dankserver media/library stack ----------------------------------------------
# These services migrated from compose-era bare containers to boot-persistent
# Quadlet units installed from this repo (podman 5.x v1 generator honors the
# WantedBy=default.target inside each .container file). The first deploy of a
# service retires the old container and migrates the data root before starting
# the unit; re-running the handler on an already-active unit restarts only the
# selected unit.
migrate_data_root() { # $1=old /opt path, $2=new data root, $3=chown|keep (default chown)
local from="$1" to="$2" mode="${3:-chown}" entry sub base
[[ -d "$from" ]] || return 0
# Renaming a directory needs WRITE permission on the directory being
# renamed itself. Compose-era data roots contain directories owned by
# the containers' mapped host uids (mariadb/redis land in the subuid
# range as 100998), which the deploy user cannot rename. Privileged mv
# preserves numeric ownership — the rootless mapping contract.
mkdir -p "$to"
shopt -s dotglob
for entry in "$from"/*; do
[[ -e "$entry" || -L "$entry" ]] || continue
base="${entry##*/}"
if [[ -e "$to/$base" ]]; then
# Collides with pre-seeded state (env file) or a rescued
# anonymous-volume stub: merge one level deep, never overwriting
# an existing entry.
if [[ -d "$entry" && -d "$to/$base" ]]; then
for sub in "$entry"/*; do
[[ -e "$sub" || -L "$sub" ]] || continue
run_privileged mv -n -- "$sub" "$to/$base/"
done
run_privileged rmdir -- "$entry" 2>/dev/null || \
echo "WARN: $entry not empty after merge; left in place" >&2
else
echo "WARN: collision on $to/$base kept existing; $entry left in place" >&2
fi
continue
fi
run_privileged mv -- "$entry" "$to/"
done
shopt -u dotglob
run_privileged rmdir -- "$from" 2>/dev/null || \
echo "WARN: $from not empty after migration; left in place" >&2
if [[ "$mode" == chown ]]; then
chown -R "$(id -u):$(id -g)" "$to"
fi
}
deploy_romm_stack() {
local service="$1" unit vol tmp
case "$service" in
mariadb) service=romm-mariadb ;;
redis) service=romm-redis ;;
esac
for unit in romm-mariadb romm-redis romm; do
cp "dankserver/romm/quadlet/$unit.container" "$quadlet_unit_dir/"
done
systemctl --user daemon-reload
if ! systemctl --user is-active --quiet romm-mariadb.service; then
# First cutover: the env file must exist before the first start, or
# the units would boot with empty secrets. Fail before stopping anything.
[[ -f /home/dank/Containers/romm/env ]] || {
echo "FAIL: /home/dank/Containers/romm/env does not exist; seed it before the first romm deploy" >&2
return 1
}
# Capture anonymous volume names while the containers are still inspectable,
# then stop and rm the compose-era trio.
local vols
vols="$(podman inspect -f '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}' romm 2>/dev/null || true)"
podman stop -t 60 romm-mariadb romm-redis romm || true
podman rm romm romm-redis romm-mariadb || true
for vol in $vols; do
tmp="$(mktemp -d)"
podman volume export --output "$tmp/v.tar" "$vol"
# --skip-old-files keeps live bind-mounted content (config/, library/,
# resources/, mariadb/, redis/) and rescues only unshadowed entries like assets/.
tar --skip-old-files -C /home/dank/Containers/romm -xf "$tmp/v.tar"
rm -rf -- "$tmp"
done
migrate_data_root /opt/romm /home/dank/Containers/romm keep
systemctl --user start romm-mariadb.service
wait_container_healthy romm-mariadb 120 || return 1
systemctl --user start romm-redis.service
wait_container_healthy romm-redis 60 || return 1
if [[ "$service" == "romm" ]]; then
systemctl --user start romm.service
fi
return 0
fi
systemctl --user restart "$service.service"
}
deploy_jellyfin() {
cp "dankserver/jellyfin/quadlet/jellyfin.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q jellyfin.service; then
podman stop -t 30 jellyfin || true
podman rm jellyfin || true
migrate_data_root /opt/jellyfin /home/dank/Containers/jellyfin
fi
systemctl --user restart jellyfin.service
}
deploy_calibre_web() {
cp "dankserver/calibre-web/quadlet/calibre-web.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q calibre-web.service; then
# The compose-era container ran under a podman-generate-systemd unit; stop
# that unit first so its Restart=always cannot resurrect the old container
# and collide with the new ContainerName.
systemctl --user stop container-calibre-web.service 2>/dev/null || true
podman stop calibre-web || true
podman rm calibre-web || true
fi
systemctl --user restart calibre-web.service
}
deploy_copyparty() {
# The admin account is created from ${COPAY_ACCOUNT}:${COPAY_PASSWORD} expanded
# at ExecStart time; a missing env file must fail the unit, not boot copyparty
# with a garbage admin password.
[[ -f /home/dank/Containers/copyparty/env ]] || {
echo "FAIL: /home/dank/Containers/copyparty/env does not exist; seed it before the first copyparty deploy" >&2
return 1
}
cp "dankserver/copyparty/quadlet/copyparty.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q copyparty.service; then
podman stop -t 15 copyparty || true
podman rm copyparty || true
migrate_data_root /opt/copyparty /home/dank/Containers/copyparty
fi
systemctl --user restart copyparty.service
}
deploy_plex() {
cp "dankserver/plex/quadlet/plex.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q plex.service; then
# First cutover: retire the native package unit BEFORE starting the
# container — both cannot own 32400. Mask (not disable): on these hosts
# `systemctl disable` deletes the unit file itself; mask keeps a
# one-command rollback while nothing is signed off yet.
run_privileged systemctl stop plexmediaserver.service
run_privileged systemctl mask plexmediaserver.service
# Rootless identity model: container root maps to the dank user
# (uid 1000) — real uid 997 is not mappable rootless. Adopt the data
# root into the fleet convention (jellyfin et al. run as 1000:1000).
# Rollback chowns it back to 997:984 before restarting the native unit.
run_privileged chown -R 1000:1000 /var/lib/plexmediaserver
fi
systemctl --user restart plex.service
# Readiness gate: /identity answers MediaContainer XML once PMS is up.
# Reality check from the 2026-09 cutover: the FIRST container boot of a
# years-old library ran ~10 min (PMS pre-binds 32400 during boot and
# empty-replies until the HTTP layer is ready), so the gate allows 600s.
# Fail the deploy (pending state preserved) rather than mark it deployed blind.
for ((i = 1; i <= 120; i++)); do
curl -fsS --max-time 3 http://127.0.0.1:32400/identity >/dev/null && return 0
sleep 5
done
echo "FAIL: plex did not answer /identity within 600s" >&2
return 1
}
deploy_tautulli() {
cp "dankserver/tautulli/quadlet/tautulli.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q tautulli.service; then
run_privileged snap stop tautulli || true
run_privileged snap disable tautulli || true
mkdir -p /home/dank/Containers/tautulli/config
if [[ ! -f /home/dank/Containers/tautulli/config/config.ini ]]; then
printf '[General]\nPMS_Web_Host = plex\nPMS_Web_Port = 32400\nPMS_Secure = 0\n' \
> /home/dank/Containers/tautulli/config/config.ini
fi
fi
systemctl --user restart tautulli.service
}
deploy_authentik() {
local unit
mkdir -p "$quadlet_unit_dir"
for unit in authentik-postgres authentik-server authentik-worker; do
cp "dankclaw/authentik/quadlet/$unit.container" "$quadlet_unit_dir/"
done
systemctl --user daemon-reload
# Retire compose-era containers only while the units are not yet active;
# unit-managed containers self-heal via Restart=always.
if ! systemctl --user is-active --quiet authentik-postgres.service; then
podman stop authentik-postgres authentik-server authentik-worker 2>/dev/null || true
podman rm authentik-postgres authentik-server authentik-worker 2>/dev/null || true
fi
# No `systemctl --user enable` here: this host's quadlet generator
# (podman 5.x v1) generates units into /run/user/1000/systemd/generator/,
# and generated units cannot be enabled — enable fails with "transient or
# generated". Boot persistence comes from [Install] WantedBy=default.target
# inside each .container file, which the generator honors at user-manager
# start (linger must stay on for the deploy user).
systemctl --user start authentik-postgres.service
wait_container_healthy authentik-postgres 120 || return 1
systemctl --user start authentik-server.service
wait_container_healthy authentik-server 300 || return 1
systemctl --user start authentik-worker.service
}
deploy_authentik_outpost() {
# The outpost is the worker-role binary of the same server image; it dials
# back to core over websocket and serves /outpost.goauthentik.io/* for
# Caddy's forward_auth. Image already in storage — zero pulls. Restarting
# only this unit cycles auth for guarded vhosts for a few seconds; the
# rest of the stack is untouched.
cp dankclaw/authentik/quadlet/authentik-outpost.container "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart authentik-outpost.service
sleep 5
systemctl --user is-active --quiet authentik-outpost.service || \
echo "WARN: authentik-outpost did not come up active" >&2
}
deploy_authentik_outpost_ldap() {
# Dedicated LDAP outpost for Jellyfin's LDAP-Auth plugin. The type-specific
# ghcr.io/goauthentik/ldap image serves 3389/6636 container-side, published
# as host 389/636 for cross-host binds from dankserver. The outpost token
# lives in the host .env, never in this repo. Restarting only this unit
# cycles LDAP auth for a few seconds; the rest of the stack is untouched.
cp dankclaw/authentik/quadlet/authentik-outpost-ldap.container "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart authentik-outpost-ldap.service
sleep 5
systemctl --user is-active --quiet authentik-outpost-ldap.service || \
echo "WARN: authentik-outpost-ldap did not come up active" >&2
}
deploy_hermes() {
# The hermes trio (gateway, dashboard, webui) shares one image and the
# /data volume and deploys as one unit: install the units, retire the
# compose-era containers holding the claimed names while the units are not
# yet active, then restart each unit. An explicit hermes deploy is a
# deliberate action, so restarting an already-running trio is intended.
# Boot persistence comes from [Install] in each unit file (generated units
# cannot be enabled by this host's v1 quadlet generator).
local unit
mkdir -p "$quadlet_unit_dir"
# The CDP browser daemon is a companion service to the trio: install its
# unit and start it if inactive, but never restart it on deploy — a
# restart would kill in-flight browser sessions. Consumers (gateway,
# webui) reconnect with backoff.
cp "dankclaw/hermes/quadlet/chrome-devtools-protocol.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active --quiet chrome-devtools-protocol.service; then
systemctl --user start chrome-devtools-protocol.service
fi
# The units run the image under a localhost/-aliased tag: this host's
# quadlet path can fail to resolve registry-prefixed refs from local
# storage at create time. Tag first, then (re)create via deploy.
podman tag docker.io/nousresearch/hermes-agent:latest localhost/hermes-agent:latest 2>/dev/null || true
for unit in hermes hermes-dashboard hermes-webui; do
cp "dankclaw/hermes/quadlet/$unit.container" "$quadlet_unit_dir/"
done
# The generator must see the freshly-copied unit files before the
# restarts below; without this reload the first deploy after a unit edit
# restarts with the stale generated ExecStart (statfs ENOENT on renamed
# Volume= sources).
systemctl --user daemon-reload
if ! systemctl --user is-active --quiet hermes.service; then
podman stop hermes hermes-dashboard hermes-webui 2>/dev/null || true
podman rm hermes hermes-dashboard hermes-webui 2>/dev/null || true
fi
systemctl --user restart hermes.service
systemctl --user restart hermes-dashboard.service
systemctl --user restart hermes-webui.service
}
deploy_anubis() {
# Anubis is the bot-challenge proxy in front of the guarded vhosts.
# The image lives on ghcr.io; pull once, then the unit runs it from
# local storage (Pull=never keeps boots off the registry).
if ! podman image exists ghcr.io/techarohq/anubis:v1.27.0; then
podman pull ghcr.io/techarohq/anubis:v1.27.0
fi
# The unit runs the localhost/ alias (see the unit file comment): this
# host's podman quadlet path cannot resolve registry-prefixed refs from
# local storage at create time. Tag first, then (re)create via deploy.
podman tag ghcr.io/techarohq/anubis:v1.27.0 localhost/techarohq/anubis:v1.27.0 2>/dev/null || true
cp "dankclaw/anubis/quadlet/anubis.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
# No enable: generated units cannot be enabled on this host (v1 quadlet
# generator); boot persistence comes from [Install] in the unit file.
# restart, not start: policy.json is parsed at process start only (no
# reload signal exists), so every redeploy must cycle the process.
systemctl --user restart anubis.service
wait_container_healthy anubis 60 || return 1
}
# Rename cutover (2026-09-03): frank-default-large -> frank-large,
# frank-default-small -> frank-small, frank-default-sightsound -> frank-npu.
# The legacy units are stopped and disabled before the renamed unit takes the
# port so no legacy process is left holding it; stop/disable of an already
# unloaded legacy unit fail harmlessly and are tolerated.
deploy_llm_large() {
run_privileged systemctl stop frank-default-large || true
run_privileged systemctl disable frank-default-large || true
relink_unit frank-large
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-large
run_privileged systemctl restart frank-large
wait_for_health frank-large http://127.0.0.1:7770/health 1200
}
deploy_llm_small() {
run_privileged systemctl stop frank-default-small || true
run_privileged systemctl disable frank-default-small || true
relink_unit frank-small
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-small
run_privileged systemctl restart frank-small
wait_for_health frank-small http://127.0.0.1:7770/health 1200
}
# frank-npu is a live serving endpoint (FLM on the XDNA2 NPU), not a
# relink-only unit: it is enabled at boot and restarted on deploy. The health
# gate polls /v1/version, FLM's cheapest always-on route (there is no /health
# route in the FLM server).
deploy_llm_npu() {
run_privileged systemctl stop frank-default-sightsound || true
run_privileged systemctl disable frank-default-sightsound || true
run_privileged systemctl stop frank-default-npu || true
run_privileged systemctl disable frank-default-npu || true
relink_unit frank-npu
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-npu
run_privileged systemctl restart frank-npu
wait_for_health frank-npu http://127.0.0.1:7771/v1/version 300
}
wait_for_health() {
local service="$1" url="$2" limit="$3"
local i
for ((i=1; i<=limit; i++)); do
if curl -fsS "$url" >/dev/null 2>&1; then
echo "$service healthy after approximately $((i * 2))s"
return 0
fi
sleep 2
done
echo "FAIL: $service did not become healthy in $((limit * 2))s" >&2
return 1
}
# ---------------------------------------------------------------------------
# FrankBook-Pro (macOS): quadlet units live INSIDE the podman machine VM
# (/home/core/.config/containers/systemd), not in the macOS user session.
# This handler pipes unit files and seed logic into the VM over
# `podman machine ssh`; it never touches systemctl on the Mac host and never
# uses run_privileged (the Mac account is not passwordless sudo). Unit files
# are re-piped on every call (idempotent, cheap) before the selected unit is
# restarted; first boot also seeds the qBittorrent config once.
FRANKBOOK_SVCS=(gluetun qbittorrent prowlarr sonarr radarr lidarr readarr)
deploy_frankbook_stack() {
local service="$1"
export PATH="/usr/local/bin:/opt/homebrew/bin:$PATH"
if ! podman machine inspect --format '{{.State}}' 2>/dev/null | grep -qx running; then
echo "FAIL: podman machine is not running on frankbook-pro" >&2
return 1
fi
local vm_dir="/home/core/.config/containers/systemd" svc
podman machine ssh "mkdir -p '$vm_dir' /home/core/Containers/gluetun /home/core/Containers/qbittorrent"
for svc in "${FRANKBOOK_SVCS[@]}"; do
podman machine ssh "cat > '$vm_dir/$svc.container'" \
< "frankbook-pro/$svc/quadlet/$svc.container"
done
local tmp
tmp="$(mktemp /tmp/frankbook-vm.XXXXXX)"
cat > "$tmp" <<'VM_SEED'
set -euo pipefail
# qBittorrent first-boot seed: conf + categories + machine-local PBKDF2
# password. qB >= 4.2 ignores plaintext passwords, so the conf carries the
# @ByteArray(salt:hash) form (PBKDF2-HMAC-SHA512, 100k iterations, 64 bytes).
if [ ! -e /home/core/Containers/qbittorrent/qbit-seeded ]; then
mkdir -p /Volumes/COMPLETE/qbittorrent-data/qBittorrent
QBIT_PW="$(openssl rand -base64 18 | tr -d '/+=' | head -c 20)"
QBIT_HASH="$(python3 - "$QBIT_PW" <<'PYH'
import hashlib, base64, os, sys
pw = sys.argv[1].encode()
salt = os.urandom(16)
h = hashlib.pbkdf2_hmac("sha512", pw, salt, 100000, dklen=64)
print("@ByteArray(%s:%s)" % (base64.b64encode(salt).decode(), base64.b64encode(h).decode()))
PYH
)"
umask 077
printf 'QBIT_WEBUI_PASSWORD=%s\n' "$QBIT_PW" > /home/core/Containers/qbittorrent/env
umask 022
cat > /Volumes/COMPLETE/qbittorrent-data/qBittorrent/qBittorrent.conf <<QCONF
[BitTorrent]
Session\\DefaultSavePath=/Volumes/COMPLETE
Session\\Port=6881
Session\\QueueingSystemEnabled=true
Session\\TempPath=/Volumes/INCOMPLETE/incomplete
Session\\TempPathEnabled=true
[LegalNotice]
Accepted=true
[Meta]
MigrationVersion=8
[Network]
Cookies\\Enabled=true
[Preferences]
WebUI\\Address=*
WebUI\\ServerDomains=*
WebUI\\HostHeaderValidation=false
WebUI\\Username=admin
WebUI\\Password_PBKDF2=$QBIT_HASH
QCONF
cat > /Volumes/COMPLETE/qbittorrent-data/qBittorrent/categories.json <<'QCAT'
{
"Finished Torrents": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
"lidarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
"radarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
"tv-sonarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"}
}
QCAT
touch /home/core/Containers/qbittorrent/qbit-seeded
echo " seeded qBittorrent config (WebUI password: /home/core/Containers/qbittorrent/env inside the VM)"
fi
VM_SEED
podman machine ssh bash -s < "$tmp"
rm -f "$tmp"
# Credential gate: without a WireGuard env the tunnel stays parked, and
# qBittorrent (which downloads through it) waits with it. The arrs run
# without the tunnel, so only these two services are gated.
case "$service" in
gluetun|qbittorrent)
if ! podman machine ssh '[ -s /home/core/Containers/gluetun/env ]' >/dev/null 2>&1; then
echo "SKIP: $service waits for gluetun credentials."
echo " Write the WireGuard env into /home/core/Containers/gluetun/env inside the VM"
echo " (podman machine ssh), then re-run: ./scripts/deploy.sh frankbook-pro --service gluetun --service qbittorrent"
return 0
fi
;;
esac
tmp="$(mktemp /tmp/frankbook-vm.XXXXXX)"
cat > "$tmp" <<'VM_START'
set -euo pipefail
SVC="${1:?service required}"
systemctl --user daemon-reload
unit="$(systemctl --user list-units --all --no-legend "${SVC}.service" "container-${SVC}.service" 2>/dev/null | sed 's/^● //' | head -n1 | awk '{print $1}')"
[ -n "$unit" ] || { echo "FAIL: no generated systemd unit for $SVC" >&2; exit 1; }
systemctl --user restart "$unit"
sleep 3
state="$(systemctl --user is-active "$unit" || true)"
if [ "$state" != active ] && [ "$state" != activating ]; then
echo "FAIL: $unit is $state" >&2
systemctl --user status "$unit" --no-pager -l || true
exit 1
fi
VM_START
podman machine ssh bash -s "$service" < "$tmp"
rm -f "$tmp"
local port path
case "$service" in
qbittorrent) port=8080; path=/ ;;
prowlarr) port=9696; path=/ping ;;
sonarr) port=8989; path=/ping ;;
radarr) port=7878; path=/ping ;;
lidarr) port=8686; path=/ping ;;
readarr) port=8787; path=/ping ;;
*) return 0 ;; # gluetun publishes only qBittorrent's ports
esac
printf ' waiting for http://127.0.0.1:%s%s on the Mac...\n' "$port" "$path"
local ok=0 i
for i in $(seq 1 60); do
if curl -fsS -m 4 "http://127.0.0.1:${port}${path}" >/dev/null 2>&1; then
ok=1
break
fi
sleep 5
done
[[ "$ok" -eq 1 ]] || { echo "FAIL: $service did not answer on 127.0.0.1:$port" >&2; return 1; }
echo " $service answers on 127.0.0.1:$port"
}
deploy_seerr() {
install_quadlet seerr dankclaw/seerr/quadlet seerr
systemctl --user restart seerr.service
}
deploy_service() {
local service="$1"
if [[ "$TARGET" == "frankbook-pro" ]]; then
# All frankbook-pro services route through the Mac handler: the name
# case below addresses the Linux hosts, and service names overlap.
deploy_frankbook_stack "$service"
return
fi
case "$service" in
caddy) deploy_caddy ;;
homepage) deploy_homepage ;;
searxng) deploy_searxng ;;
gluetun) deploy_gluetun ;;
searxng-valkey) deploy_searxng_valkey ;;
open-webui) deploy_open_webui ;;
pihole) deploy_pihole ;;
gitea-postgres) deploy_gitea_unit gitea-postgres gitea-postgres ;;
gitea-server) deploy_gitea_unit gitea-server gitea-server ;;
authentik-postgres|authentik-server|authentik-worker)
deploy_authentik
;;
authentik-outpost)
deploy_authentik_outpost
;;
authentik-outpost-ldap)
deploy_authentik_outpost_ldap
;;
anubis) deploy_anubis ;;
seerr) deploy_seerr ;;
frank-large) deploy_llm_large ;;
frank-small) deploy_llm_small ;;
frank-npu) deploy_llm_npu ;;
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
jellyfin) deploy_jellyfin ;;
calibre-web) deploy_calibre_web ;;
copyparty) deploy_copyparty ;;
plex) deploy_plex ;;
tautulli) deploy_tautulli ;;
*) deploy_compose_service "$service" ;;
esac
}
if [[ -n "$FROM_REVISION" ]]; then
# Register every changed service before applying the requested scope. This
# preserves an explicit --service selection as pending work for the next
# ordinary deployment.
register_changed_services "$FROM_REVISION"
fi
if [[ "$PENDING" -eq 1 ]]; then
add_changed_services "$FROM_REVISION"
add_pending_services
fi
# Remove duplicate service names while preserving caller/planner order.
UNIQUE_SERVICES=()
declare -A SEEN_SERVICES=()
for service in "${SERVICES[@]}"; do
[[ -n "$service" ]] || continue
if [[ -z "${SEEN_SERVICES[$service]:-}" ]]; then
UNIQUE_SERVICES+=("$service")
SEEN_SERVICES[$service]=1
fi
done
SERVICES=("${UNIQUE_SERVICES[@]}")
if [[ "${#SERVICES[@]}" -eq 0 ]]; then
echo "No services selected; nothing to deploy."
exit 0
fi
for service in "${SERVICES[@]}"; do
[[ "$service" =~ ^[a-z0-9][a-z0-9-]*$ ]] || { echo "invalid service: $service" >&2; exit 2; }
service_is_allowed "$service" || { echo "service $service is not deployed on $TARGET" >&2; exit 2; }
echo "=== Deploying $service at $REVISION ==="
mark_status "$service" pending
deploy_service "$service"
mark_deployed "$service"
echo " STATE[$service] = $REVISION (status=deployed)"
done