Frank
4fca739bf8
The radiance MXFP4 stack cannot run TP=2 on this host. RCCL's collective
kernels carry a hidden hostcall-buffer parameter, and ROCr refuses to
dispatch them on a device without PCIe atomics: the AQL dispatch fails with
hipErrorIllegalState ("the operation cannot be performed in the present
state"), which is what every two-rank collective hit. The second R9700 sits
behind the chipset bridges 05:00.0/06:09.0, which advertise
`AtomicOpsCap: Routing-` — a read-only capability, so no software can enable
atomic routing on that path and the driver's
pci_enable_atomic_ops_to_root() fails for that card. The first card, on CPU
root port 00:01.1, has atomics enabled.
Ruled out: kernel version (7.0.0-28/-30/-31 identical, after a full upgrade),
iommu=pt, P2P hardware (45.3 GB/s between the cards), the shipped librccl (a
from-source rebuild reproduces the failure at the same call path), vLLM and
pynccl (a bare two-rank torch.distributed allreduce fails the same way), all
NCCL transport knobs, and single-rank tests (NCCL short-circuits at
world_size=1, so one-card success proves nothing).
Deployment: three rootless Quadlets — frank-medium-a (:7771, card 1),
frank-medium-b (:7772, card 2), both TP=1 at 114688 context (262144 and
131072 do not fit one card's KV) with vLLM-profiled KV — and
frank-medium-proxy (:7770), a round-robin that preserves the fleet contract
of one endpoint and one served model name. frank-medium.container stays as
the TP=2 target for a host whose second card is atomic-capable; the deploy
handler does not install it and trashes any stale copy.
Measured through the proxy (vllm bench serve, random dataset, --ignore-eos):
conc 1 42.8 tok/s, conc 4 147.6 tok/s, conc 8 230.8 tok/s, 32k prefill at
~2.8k tok/s per card, TPOT p50 23.6-28.8 ms.
Also: healthcheck reads PORT so one probe serves all three units, the hermes
example and the harness models.yml drop to the real 114688 context, and the
runbook records the full evidence chain.
1255 lines
54 KiB
Bash
Executable File
1255 lines
54 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# deploy-remote.sh — apply an explicit service plan on one target host.
|
|
#
|
|
# This script runs on the target checkout. It deliberately does not infer a
|
|
# plan, pull Git, or restart an unrelated service. deploy.sh computes the plan
|
|
# from the pushed revision and passes the selected service names here.
|
|
# Per-service revisions live in /var/lib/serverAppConfigs/deploy-state (or
|
|
# DEPLOY_STATE_DIR), outside the checkout, so an unselected change remains
|
|
# pending until its service is explicitly deployed.
|
|
set -euo pipefail
|
|
|
|
TARGET="${1:-}"
|
|
REVISION="${2:-}"
|
|
shift 2 || true
|
|
FROM_REVISION=""
|
|
PENDING=0
|
|
SERVICES=()
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: deploy-remote.sh <dankclaw|dankserver|frankbook-pro> <git-revision> [--pending] [service]...
|
|
|
|
The service list is explicit. An empty list is a successful no-op.
|
|
EOF
|
|
}
|
|
|
|
while [[ "$#" -gt 0 ]]; do
|
|
case "$1" in
|
|
--from-revision)
|
|
[[ "$#" -ge 2 ]] || { echo "--from-revision requires a revision" >&2; exit 2; }
|
|
FROM_REVISION="$2"
|
|
shift 2
|
|
;;
|
|
--pending)
|
|
PENDING=1
|
|
shift
|
|
;;
|
|
--help|-h)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
SERVICES+=("$1")
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
REMOTE_DIR="${REMOTE_DIR:-/home/dank/serverAppConfigs}"
|
|
STATE_DIR="${DEPLOY_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/serverAppConfigs/deploy-state}"
|
|
|
|
run_privileged() {
|
|
if [[ -n "${FRANK_SUDO_PW:-}" ]]; then
|
|
printf '%s\n' "$FRANK_SUDO_PW" | sudo -S -- "$@"
|
|
else
|
|
sudo -- "$@"
|
|
fi
|
|
}
|
|
|
|
if [[ "$TARGET" != dankclaw && "$TARGET" != dankserver && "$TARGET" != frankbook-pro ]]; then
|
|
usage >&2
|
|
exit 2
|
|
fi
|
|
if [[ -z "$REVISION" ]]; then
|
|
usage >&2
|
|
exit 2
|
|
fi
|
|
if [[ "$REVISION" == *[!0-9a-fA-F]* ]]; then
|
|
echo "invalid Git revision: $REVISION" >&2
|
|
exit 2
|
|
fi
|
|
if ! git -C "$REMOTE_DIR" cat-file -e "$REVISION^{commit}" 2>/dev/null; then
|
|
echo "revision is not a commit in the target checkout: $REVISION" >&2
|
|
exit 2
|
|
fi
|
|
cd "$REMOTE_DIR"
|
|
|
|
# Keep the state directory outside Git. The file is written only after a
|
|
# handler completes successfully; an interrupted handler remains pending.
|
|
mkdir -p "$STATE_DIR"
|
|
|
|
service_is_allowed() {
|
|
local service="$1"
|
|
case "$TARGET:$service" in
|
|
dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\
|
|
dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
|
|
dankclaw:comfyui|dankclaw:hermes|\
|
|
dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
|
|
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|\
|
|
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
|
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
|
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
|
|
dankclaw:frank-large|\
|
|
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
|
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
|
|
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
|
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
|
|
dankserver:frank-medium|\
|
|
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
|
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
|
frankbook-pro:readarr)
|
|
return 0
|
|
;;
|
|
*)
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
mark_status() {
|
|
local service="$1" status="$2" tmp
|
|
tmp="$(mktemp "$STATE_DIR/.${service}.XXXXXX")"
|
|
printf 'revision=%s\nstatus=%s\n' "$REVISION" "$status" > "$tmp"
|
|
mv -f -- "$tmp" "$STATE_DIR/$service"
|
|
}
|
|
|
|
mark_deployed() {
|
|
mark_status "$1" deployed
|
|
}
|
|
|
|
state_revision() {
|
|
local service="$1" line value
|
|
if [[ -r "$STATE_DIR/$service" ]]; then
|
|
while IFS= read -r line; do
|
|
case "$line" in
|
|
revision=*) value="${line#revision=}" ;;
|
|
esac
|
|
done < "$STATE_DIR/$service"
|
|
printf '%s' "${value:-}"
|
|
fi
|
|
}
|
|
|
|
state_status() {
|
|
local service="$1" line value=pending
|
|
if [[ -r "$STATE_DIR/$service" ]]; then
|
|
while IFS= read -r line; do
|
|
case "$line" in
|
|
status=*) value="${line#status=}" ;;
|
|
esac
|
|
done < "$STATE_DIR/$service"
|
|
fi
|
|
printf '%s' "$value"
|
|
}
|
|
|
|
register_changed_services() {
|
|
local base="$1" service recorded status
|
|
[[ -n "$base" ]] || return 0
|
|
while IFS= read -r service; do
|
|
[[ -n "$service" ]] || continue
|
|
recorded="$(state_revision "$service")"
|
|
status="$(state_status "$service")"
|
|
if [[ "$status" != deployed || "$recorded" != "$REVISION" ]]; then
|
|
mark_status "$service" pending
|
|
fi
|
|
done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
|
|
--base "$base" --head "$REVISION")
|
|
}
|
|
|
|
add_changed_services() {
|
|
local base="$1"
|
|
local service
|
|
[[ -n "$base" ]] || return 0
|
|
while IFS= read -r service; do
|
|
[[ -n "$service" ]] || continue
|
|
SERVICES+=("$service")
|
|
done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
|
|
--base "$base" --head "$REVISION")
|
|
}
|
|
|
|
add_pending_services() {
|
|
local service recorded status changed
|
|
for state_file in "$STATE_DIR"/*; do
|
|
[[ -f "$state_file" ]] || continue
|
|
service="${state_file##*/}"
|
|
recorded="$(state_revision "$service")"
|
|
status="$(state_status "$service")"
|
|
if [[ "$status" != deployed ]]; then
|
|
# A pending service was intentionally skipped or its handler
|
|
# failed. Retry it even when the checkout has not advanced.
|
|
SERVICES+=("$service")
|
|
continue
|
|
fi
|
|
[[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
|
|
if [[ "$TARGET" == dankserver && "$service" == frank-large ]]; then continue; fi
|
|
if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
|
|
--repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
|
|
echo "WARN: unable to compare state for $service; retaining it as pending" >&2
|
|
SERVICES+=("$service")
|
|
continue
|
|
fi
|
|
if grep -Fxq "$service" <<< "$changed"; then
|
|
SERVICES+=("$service")
|
|
fi
|
|
done
|
|
}
|
|
|
|
copy_searxng_config() {
|
|
run_privileged mkdir -p /opt/searxng/config
|
|
run_privileged cp shared/searxng/settings.yml /opt/searxng/config/settings.yml
|
|
run_privileged cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml
|
|
}
|
|
|
|
copy_prometheus_config() {
|
|
run_privileged mkdir -p /opt/prometheus/config
|
|
run_privileged cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml
|
|
}
|
|
|
|
copy_loki_config() {
|
|
run_privileged mkdir -p /opt/loki/config
|
|
run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
|
|
}
|
|
|
|
copy_alloy_config() {
|
|
run_privileged mkdir -p /opt/alloy/config
|
|
run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
|
|
}
|
|
|
|
render_caddyfile() {
|
|
local hash
|
|
hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
|
|
if [[ -z "$hash" ]]; then
|
|
echo "WARN: CADDY_BASICAUTH_HASH is not set; refusing to overwrite rendered Caddyfile" >&2
|
|
return 1
|
|
fi
|
|
run_privileged mkdir -p /home/dank/Containers/caddy
|
|
sed -e "s|REPLACE_WITH_HASH|$hash|g" dankclaw/Caddyfile > /tmp/serverAppConfigs.Caddyfile
|
|
run_privileged cp /tmp/serverAppConfigs.Caddyfile /home/dank/Containers/caddy/Caddyfile
|
|
}
|
|
|
|
deploy_homepage() {
|
|
if [[ "$TARGET" != dankclaw ]]; then
|
|
echo "homepage is owned by dankclaw; refusing to deploy it on $TARGET" >&2
|
|
return 2
|
|
fi
|
|
run_privileged mkdir -p /srv/homepage
|
|
run_privileged cp shared/homepage/index.html shared/homepage/admin.html shared/homepage/app.js \
|
|
shared/homepage/style.css shared/homepage/services.json /srv/homepage/
|
|
if [[ -d shared/homepage/fonts ]]; then
|
|
run_privileged cp -r shared/homepage/fonts /srv/homepage/
|
|
fi
|
|
local deployed_at
|
|
deployed_at="$(git log -1 --format=%cI)"
|
|
printf 'window.DEPLOYED_AT = "%s";\n' "$deployed_at" > /tmp/deployed.js
|
|
run_privileged cp /tmp/deployed.js /srv/homepage/deployed.js
|
|
run_privileged chmod -R a+rX /srv/homepage
|
|
run_privileged cp scripts/homepage-status.py /usr/local/bin/homepage-status.py
|
|
run_privileged cp scripts/homepage-status.service scripts/homepage-status.timer /etc/systemd/system/
|
|
run_privileged systemctl daemon-reload
|
|
run_privileged systemctl enable homepage-status.timer
|
|
run_privileged systemctl restart homepage-status.timer
|
|
run_privileged python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json
|
|
}
|
|
|
|
stage_legacy_gitea_caddy_state() {
|
|
local legacy_state="$STATE_DIR/gitea-caddy"
|
|
[[ -e "$legacy_state" ]] || return 0
|
|
local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
|
|
mkdir -p "$legacy_trash"
|
|
mv -- "$legacy_state" "$legacy_trash/gitea-caddy.state"
|
|
}
|
|
|
|
retire_legacy_gitea_caddy() {
|
|
# The old split proxy owns ports 80/443 on hosts upgraded in place. Stop
|
|
# and disable its generated unit before the canonical Caddy is started.
|
|
if systemctl --user is-active --quiet gitea-caddy.service 2>/dev/null; then
|
|
systemctl --user stop gitea-caddy.service
|
|
fi
|
|
if systemctl --user is-enabled --quiet gitea-caddy.service 2>/dev/null; then
|
|
# A Quadlet-generated unit may report "generated" without having an
|
|
# enablement symlink. Disable is therefore best-effort; moving the
|
|
# source out of the generator directory is the durable action.
|
|
systemctl --user disable gitea-caddy.service 2>/dev/null || true
|
|
fi
|
|
local unit_dir="$HOME/.config/containers/systemd"
|
|
local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
|
|
# Podman generators scan the unit directory recursively on this host, so
|
|
# leaving the retired file in systemd/.AGENTTRASH would regenerate it.
|
|
# Stage it outside the unit tree in the dated home trash instead.
|
|
for unit_file in "$unit_dir/gitea-caddy.container" \
|
|
"$unit_dir/.AGENTTRASH/gitea-caddy.container"; do
|
|
if [[ -e "$unit_file" ]]; then
|
|
mkdir -p "$legacy_trash"
|
|
mv -- "$unit_file" "$legacy_trash/"
|
|
fi
|
|
done
|
|
stage_legacy_gitea_caddy_state
|
|
systemctl --user daemon-reload
|
|
if podman container exists gitea-caddy; then
|
|
podman rm --force gitea-caddy
|
|
fi
|
|
}
|
|
|
|
deploy_caddy() {
|
|
if [[ "$TARGET" != dankclaw ]]; then
|
|
echo "caddy is owned by dankclaw; refusing to deploy it on $TARGET" >&2
|
|
return 2
|
|
fi
|
|
render_caddyfile
|
|
deploy_homepage
|
|
local cfhash oldhash image
|
|
cfhash="$(sha256sum dankclaw/caddy/Containerfile | cut -d' ' -f1)"
|
|
image=franksdank/caddy-porkbun
|
|
oldhash="$(podman image inspect "$image:latest" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
|
|
if [[ "$oldhash" != "$cfhash" ]]; then
|
|
podman build --no-cache --label "cfhash=$cfhash" -t "$image:latest" \
|
|
-f dankclaw/caddy/Containerfile .
|
|
fi
|
|
# Retire the old split proxy before claiming ports 80/443. This is
|
|
# idempotent and leaves the old unit recoverable in the target trash.
|
|
retire_legacy_gitea_caddy
|
|
# Caddy's env: Porkbun DNS-01 credentials from the checkout's .env. The
|
|
# live file holds secrets, so it exists only on the host; the quadlet
|
|
# reads it via EnvironmentFile.
|
|
mkdir -p /home/dank/Containers/caddy
|
|
if [[ ! -f /home/dank/Containers/caddy/env ]]; then
|
|
awk -F= '/^PORKBUN_API_KEY=|^PORKBUN_API_SECRET_KEY=/ { print }' \
|
|
dankclaw/.env > /home/dank/Containers/caddy/env
|
|
fi
|
|
local var
|
|
for var in PORKBUN_API_KEY PORKBUN_API_SECRET_KEY; do
|
|
grep -q "^${var}=." /home/dank/Containers/caddy/env \
|
|
|| { echo "caddy env missing ${var}; set it in dankclaw/.env" >&2; return 1; }
|
|
done
|
|
# Retire the bare compose container: all state (Caddyfile, certs, config)
|
|
# lives in host volumes, so removing the container loses nothing.
|
|
podman stop caddy >/dev/null 2>&1 || true
|
|
podman rm caddy >/dev/null 2>&1 || true
|
|
# install_quadlet runs daemon-reload, which re-runs podman's quadlet
|
|
# generator: it creates caddy.service AND the default.target.wants symlink
|
|
# from the unit's [Install] section (is-enabled reports "generated" —
|
|
# that is the healthy state for a quadlet; an explicit systemctl enable
|
|
# fails on generated units). Boot persistence comes from the generator,
|
|
# same as pihole/gitea/searxng/open-webui.
|
|
install_quadlet caddy dankclaw/caddy/quadlet caddy
|
|
systemctl --user restart caddy.service
|
|
if podman container exists caddy; then
|
|
local reloaded=0
|
|
for _ in 1 2 3 4 5; do
|
|
if podman exec caddy caddy reload --config /etc/caddy/Caddyfile \
|
|
--adapter caddyfile >/dev/null 2>&1; then
|
|
reloaded=1
|
|
echo "Caddy reloaded."
|
|
break
|
|
fi
|
|
sleep 3
|
|
done
|
|
if [[ "$reloaded" -ne 1 ]]; then
|
|
echo "Caddy reload failed; restarting only caddy."
|
|
podman restart caddy
|
|
fi
|
|
fi
|
|
}
|
|
|
|
deploy_compose_service() {
|
|
local service="$1"
|
|
if [[ "$TARGET" == dankserver ]]; then
|
|
# Dankserver's compose file is the source-of-truth inventory, but its
|
|
# production containers are individually systemd-managed/bare. Do not
|
|
# use compose up here: it can create a second container with a wrong
|
|
# name and can interfere with a running workload.
|
|
echo "WARN: $service is recorded in dankserver/compose.yaml; production container management is host-specific and no restart was attempted."
|
|
return 0
|
|
fi
|
|
case "$service" in
|
|
comfyui|hermes|hermes-dashboard|hermes-webui)
|
|
case "$service" in
|
|
comfyui)
|
|
build_compose_image "$service"
|
|
;;
|
|
esac
|
|
podman-compose -f dankclaw/compose.yaml up -d --no-deps "$service"
|
|
;;
|
|
*)
|
|
echo "unknown compose service: $service" >&2
|
|
return 2
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# grafana is the one observability service that holds a credential: the
|
|
# Authentik OAuth2 client secret. It is seeded into a per-service env file (the
|
|
# fleet convention: caddy, pihole, copyparty, romm, seerr) instead of being
|
|
# imported from the repo .env wholesale. The repo-root .env that the compose
|
|
# service used to pick up is a gluetun env file — it also carries HOME=/root
|
|
# and the WireGuard keys, and an EnvironmentFile of that file sets HOME for the
|
|
# service, which makes podman look for its config under /root and fail.
|
|
seed_grafana_env() {
|
|
local dest=/home/dank/Containers/grafana/env
|
|
mkdir -p /home/dank/Containers/grafana
|
|
if [[ ! -f "$dest" ]]; then
|
|
awk -F= '/^GRAFANA_OAUTH_CLIENT_SECRET=/ { print }' dankclaw/.env > "$dest"
|
|
chmod 600 "$dest"
|
|
fi
|
|
grep -q '^GRAFANA_OAUTH_CLIENT_SECRET=.' "$dest" \
|
|
|| { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; }
|
|
}
|
|
|
|
# The five observability containers left compose on 2026-09-11 (they were
|
|
# started by hand with podman-compose and never came back after a reboot).
|
|
# They are ordinary rootless Quadlet units now: same images, same mounts, same
|
|
# arguments, same container names, so nothing that addresses them by name —
|
|
# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
|
|
# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
|
|
# compose services published on 0.0.0.0 and none of them authenticates.
|
|
deploy_observability() {
|
|
local service="$1"
|
|
case "$service" in
|
|
prometheus)
|
|
copy_prometheus_config
|
|
;;
|
|
loki)
|
|
copy_loki_config
|
|
build_compose_image loki
|
|
;;
|
|
alloy)
|
|
copy_alloy_config
|
|
build_compose_image alloy
|
|
;;
|
|
grafana)
|
|
seed_grafana_env || return 1
|
|
;;
|
|
node-exporter)
|
|
;;
|
|
*)
|
|
echo "unknown observability service: $service" >&2
|
|
return 2
|
|
;;
|
|
esac
|
|
install_quadlet observability dankclaw/observability/quadlet "$service"
|
|
systemctl --user restart "$service.service"
|
|
case "$service" in
|
|
prometheus) wait_for_health prometheus http://127.0.0.1:9090/-/healthy 60 ;;
|
|
grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
|
|
loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
|
|
alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
|
|
node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
|
|
esac
|
|
}
|
|
|
|
build_compose_image() {
|
|
local service="$1" image cfhash oldhash containerfile
|
|
case "$service" in
|
|
comfyui)
|
|
image=comfyui-rocm:gfx1151
|
|
containerfile=dankclaw/Containerfile.comfyui
|
|
;;
|
|
loki|alloy)
|
|
image="franksdank/$service:latest"
|
|
containerfile="dankclaw/$service/Containerfile"
|
|
;;
|
|
*)
|
|
return 0
|
|
;;
|
|
esac
|
|
cfhash="$(sha256sum "$containerfile" | cut -d' ' -f1)"
|
|
oldhash="$(podman image inspect "$image" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
|
|
if [[ "$oldhash" != "$cfhash" ]]; then
|
|
# --format docker is required, not cosmetic: podman defaults to the OCI
|
|
# image format, which has no HEALTHCHECK field at all, so a build in
|
|
# the default format silently drops the loki image's healthcheck and
|
|
# the container then reports no health instead of failing loudly.
|
|
podman build --format docker --no-cache --label "cfhash=$cfhash" \
|
|
-t "$image" -f "$containerfile" .
|
|
fi
|
|
}
|
|
|
|
quadlet_unit_dir="$HOME/.config/containers/systemd"
|
|
# dankserver's user manager had no systemd container generator dir before the
|
|
# quadlet migration; every handler that installs a unit must find it present.
|
|
mkdir -p "$quadlet_unit_dir"
|
|
install_quadlet() {
|
|
local service="$1"
|
|
local source_dir="$2"
|
|
local unit="$3"
|
|
mkdir -p "$quadlet_unit_dir"
|
|
cp "$source_dir/$unit.container" "$quadlet_unit_dir/"
|
|
if [[ -f "$source_dir/$unit.network" ]]; then
|
|
cp "$source_dir/$unit.network" "$quadlet_unit_dir/"
|
|
fi
|
|
systemctl --user daemon-reload
|
|
}
|
|
|
|
deploy_searxng() {
|
|
copy_searxng_config
|
|
install_quadlet searxng dankclaw/searxng/quadlet searxng
|
|
install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
|
|
install_quadlet gluetun dankclaw/searxng/quadlet gluetun
|
|
# The network is shared by all three units; install it even when only the
|
|
# SearXNG service was selected. These copies are user-owned Quadlet state.
|
|
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user start searxng-network.service
|
|
systemctl --user restart gluetun.service searxng-valkey.service searxng.service
|
|
}
|
|
|
|
deploy_gluetun() {
|
|
install_quadlet gluetun dankclaw/searxng/quadlet gluetun
|
|
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart gluetun.service searxng.service
|
|
}
|
|
|
|
deploy_searxng_valkey() {
|
|
install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
|
|
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart searxng-valkey.service searxng.service
|
|
}
|
|
|
|
# Marreta's secrets are seeded into a per-service env file under the deploy
|
|
# user's home (rootless-readable), following the grafana convention rather
|
|
# than the /opt/searxng one, so the rootless unit can read it without any
|
|
# ownership gymnastics. Only the three MARRETA_* values are copied; the
|
|
# quadlet maps them to the app's APP_KEY / ADMIN_EMAIL / ADMIN_PASSWORD.
|
|
seed_marreta_env() {
|
|
local dest=/home/dank/Containers/marreta/env
|
|
mkdir -p /home/dank/Containers/marreta
|
|
if [[ ! -f "$dest" ]]; then
|
|
awk -F= '/^MARRETA_APP_KEY=/ || /^MARRETA_ADMIN_EMAIL=/ || /^MARRETA_ADMIN_PASSWORD=/ { print }' \
|
|
dankclaw/.env > "$dest"
|
|
chmod 600 "$dest"
|
|
fi
|
|
local var
|
|
for var in MARRETA_APP_KEY MARRETA_ADMIN_EMAIL MARRETA_ADMIN_PASSWORD; do
|
|
grep -q "^${var}=." "$dest" \
|
|
|| { echo "marreta env missing ${var}; set it in dankclaw/.env" >&2; return 1; }
|
|
done
|
|
}
|
|
|
|
deploy_marreta() {
|
|
seed_marreta_env
|
|
install_quadlet marreta dankclaw/marreta/quadlet marreta
|
|
install_quadlet marreta-browser dankclaw/marreta/quadlet marreta-browser
|
|
systemctl --user daemon-reload
|
|
# Both containers share gluetun's netns; the tunnel must be up first.
|
|
systemctl --user start gluetun.service
|
|
# Browser before app: marreta drives the CDP endpoint on loopback.
|
|
systemctl --user restart marreta-browser.service
|
|
systemctl --user restart marreta.service
|
|
}
|
|
|
|
deploy_open_webui() {
|
|
install_quadlet open-webui dankclaw/open-webui/quadlet open-webui
|
|
mkdir -p /home/dank/Containers/open-webui/data
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart open-webui.service
|
|
}
|
|
|
|
deploy_pihole() {
|
|
# Rootless port 53 needs net.ipv4.ip_unprivileged_port_start <= 53, and it
|
|
# must hold at boot. The repo drop-in is the single source of truth.
|
|
# systemd-sysctl applies /etc/sysctl.d/*.conf (lexicographic), then
|
|
# /run/sysctl.d, /usr/lib/sysctl.d, and finally /etc/sysctl.conf — later
|
|
# files override earlier ones. On 2026-08-26 a later-sorted drop-in (=80)
|
|
# AND a stale line in /etc/sysctl.conf (=80) both reset the value after a
|
|
# reboot and took LAN DNS down. Neutralize every conflicting source:
|
|
# stage sysctl.d files that set a value above 53 in the home trash, and
|
|
# comment out an active line in /etc/sysctl.conf (idempotent).
|
|
run_privileged install -m 0644 dankclaw/pihole/sysctl/99-dnsmasq.conf \
|
|
/etc/sysctl.d/99-dnsmasq.conf
|
|
local sysctl_file sysctl_val sysctl_trash
|
|
while IFS= read -r sysctl_file; do
|
|
[[ "$sysctl_file" == */99-dnsmasq.conf ]] && continue
|
|
sysctl_val="$(sed -nE 's/^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=[[:space:]]*([0-9]+).*/\1/p' "$sysctl_file" | head -1)"
|
|
if [[ "$sysctl_val" =~ ^[0-9]+$ ]] && (( sysctl_val > 53 )); then
|
|
sysctl_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-sysctl-unprivileged-port"
|
|
mkdir -p "$sysctl_trash"
|
|
run_privileged mv "$sysctl_file" "$sysctl_trash/"
|
|
fi
|
|
done < <(grep -lE '^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=' /etc/sysctl.d/*.conf 2>/dev/null || true)
|
|
if [[ -f /etc/sysctl.conf ]] && grep -qE '^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=' /etc/sysctl.conf; then
|
|
run_privileged sed -i -E 's/^[[:space:]]*(net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=)/# \1 managed by dankclaw\/pihole\/sysctl\/99-dnsmasq.conf/' /etc/sysctl.conf
|
|
fi
|
|
run_privileged sysctl --system >/dev/null
|
|
mkdir -p /home/dank/Containers/pihole/etc-pihole /home/dank/Containers/pihole/etc-dockers
|
|
if [[ ! -f /home/dank/Containers/pihole/env ]]; then
|
|
cp dankclaw/pihole/env.example /home/dank/Containers/pihole/env
|
|
echo "WARN: created Pi-hole env from the example; set WEBPASSWORD before using its UI" >&2
|
|
fi
|
|
cp dankclaw/pihole/addn-hosts /home/dank/Containers/pihole/addn-hosts
|
|
mkdir -p "$quadlet_unit_dir"
|
|
cp dankclaw/pihole/quadlet/pihole.container "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart pihole.service
|
|
}
|
|
|
|
deploy_gitea_unit() {
|
|
local service="$1"
|
|
local unit="$2"
|
|
mkdir -p "$quadlet_unit_dir"
|
|
cp "dankclaw/gitea/quadlet/$unit.container" "$quadlet_unit_dir/"
|
|
[[ -f dankclaw/gitea/quadlet/gitea.network ]] && cp dankclaw/gitea/quadlet/gitea.network "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart "$service.service"
|
|
}
|
|
|
|
relink_unit() {
|
|
local service="$1"
|
|
run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service"
|
|
}
|
|
|
|
# Authentik runs as three boot-persistent Quadlet units (postgres -> server ->
|
|
# worker). The trio deploys as one unit: unit files are installed, stale
|
|
# compose-era containers holding the claimed names are retired, then the chain
|
|
# starts in dependency order with health gates between stages. Re-running the
|
|
# handler on an already-active stack is a deliberate no-op (no DB bounce).
|
|
wait_container_healthy() {
|
|
local name="$1" limit="$2" status="unknown"
|
|
for ((i = 1; i <= limit; i += 5)); do
|
|
status="$(podman inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null || echo missing)"
|
|
[[ "$status" == "healthy" ]] && {
|
|
echo "$name healthy after approximately ${i}s"
|
|
return 0
|
|
}
|
|
sleep 5
|
|
done
|
|
echo "FAIL: $name did not become healthy in ${limit}s (status: $status)" >&2
|
|
return 1
|
|
}
|
|
|
|
# --- dankserver media/library stack ----------------------------------------------
|
|
# These services migrated from compose-era bare containers to boot-persistent
|
|
# Quadlet units installed from this repo (podman 5.x v1 generator honors the
|
|
# WantedBy=default.target inside each .container file). The first deploy of a
|
|
# service retires the old container and migrates the data root before starting
|
|
# the unit; re-running the handler on an already-active unit restarts only the
|
|
# selected unit.
|
|
|
|
migrate_data_root() { # $1=old /opt path, $2=new data root, $3=chown|keep (default chown)
|
|
local from="$1" to="$2" mode="${3:-chown}" entry sub base
|
|
[[ -d "$from" ]] || return 0
|
|
# Renaming a directory needs WRITE permission on the directory being
|
|
# renamed itself. Compose-era data roots contain directories owned by
|
|
# the containers' mapped host uids (mariadb/redis land in the subuid
|
|
# range as 100998), which the deploy user cannot rename. Privileged mv
|
|
# preserves numeric ownership — the rootless mapping contract.
|
|
mkdir -p "$to"
|
|
shopt -s dotglob
|
|
for entry in "$from"/*; do
|
|
[[ -e "$entry" || -L "$entry" ]] || continue
|
|
base="${entry##*/}"
|
|
if [[ -e "$to/$base" ]]; then
|
|
# Collides with pre-seeded state (env file) or a rescued
|
|
# anonymous-volume stub: merge one level deep, never overwriting
|
|
# an existing entry.
|
|
if [[ -d "$entry" && -d "$to/$base" ]]; then
|
|
for sub in "$entry"/*; do
|
|
[[ -e "$sub" || -L "$sub" ]] || continue
|
|
run_privileged mv -n -- "$sub" "$to/$base/"
|
|
done
|
|
run_privileged rmdir -- "$entry" 2>/dev/null || \
|
|
echo "WARN: $entry not empty after merge; left in place" >&2
|
|
else
|
|
echo "WARN: collision on $to/$base kept existing; $entry left in place" >&2
|
|
fi
|
|
continue
|
|
fi
|
|
run_privileged mv -- "$entry" "$to/"
|
|
done
|
|
shopt -u dotglob
|
|
run_privileged rmdir -- "$from" 2>/dev/null || \
|
|
echo "WARN: $from not empty after migration; left in place" >&2
|
|
if [[ "$mode" == chown ]]; then
|
|
chown -R "$(id -u):$(id -g)" "$to"
|
|
fi
|
|
}
|
|
|
|
deploy_romm_stack() {
|
|
local service="$1" unit vol tmp
|
|
case "$service" in
|
|
mariadb) service=romm-mariadb ;;
|
|
redis) service=romm-redis ;;
|
|
esac
|
|
for unit in romm-mariadb romm-redis romm; do
|
|
cp "dankserver/romm/quadlet/$unit.container" "$quadlet_unit_dir/"
|
|
done
|
|
systemctl --user daemon-reload
|
|
if ! systemctl --user is-active --quiet romm-mariadb.service; then
|
|
# First cutover: the env file must exist before the first start, or
|
|
# the units would boot with empty secrets. Fail before stopping anything.
|
|
[[ -f /home/dank/Containers/romm/env ]] || {
|
|
echo "FAIL: /home/dank/Containers/romm/env does not exist; seed it before the first romm deploy" >&2
|
|
return 1
|
|
}
|
|
# Capture anonymous volume names while the containers are still inspectable,
|
|
# then stop and rm the compose-era trio.
|
|
local vols
|
|
vols="$(podman inspect -f '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}' romm 2>/dev/null || true)"
|
|
podman stop -t 60 romm-mariadb romm-redis romm || true
|
|
podman rm romm romm-redis romm-mariadb || true
|
|
for vol in $vols; do
|
|
tmp="$(mktemp -d)"
|
|
podman volume export --output "$tmp/v.tar" "$vol"
|
|
# --skip-old-files keeps live bind-mounted content (config/, library/,
|
|
# resources/, mariadb/, redis/) and rescues only unshadowed entries like assets/.
|
|
tar --skip-old-files -C /home/dank/Containers/romm -xf "$tmp/v.tar"
|
|
rm -rf -- "$tmp"
|
|
done
|
|
migrate_data_root /opt/romm /home/dank/Containers/romm keep
|
|
systemctl --user start romm-mariadb.service
|
|
wait_container_healthy romm-mariadb 120 || return 1
|
|
systemctl --user start romm-redis.service
|
|
wait_container_healthy romm-redis 60 || return 1
|
|
if [[ "$service" == "romm" ]]; then
|
|
systemctl --user start romm.service
|
|
fi
|
|
return 0
|
|
fi
|
|
systemctl --user restart "$service.service"
|
|
}
|
|
|
|
deploy_jellyfin() {
|
|
cp "dankserver/jellyfin/quadlet/jellyfin.container" "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
if ! systemctl --user is-active -q jellyfin.service; then
|
|
podman stop -t 30 jellyfin || true
|
|
podman rm jellyfin || true
|
|
migrate_data_root /opt/jellyfin /home/dank/Containers/jellyfin
|
|
fi
|
|
systemctl --user restart jellyfin.service
|
|
}
|
|
|
|
deploy_calibre_web() {
|
|
cp "dankserver/calibre-web/quadlet/calibre-web.container" "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
if ! systemctl --user is-active -q calibre-web.service; then
|
|
# The compose-era container ran under a podman-generate-systemd unit; stop
|
|
# that unit first so its Restart=always cannot resurrect the old container
|
|
# and collide with the new ContainerName.
|
|
systemctl --user stop container-calibre-web.service 2>/dev/null || true
|
|
podman stop calibre-web || true
|
|
podman rm calibre-web || true
|
|
fi
|
|
systemctl --user restart calibre-web.service
|
|
}
|
|
|
|
deploy_copyparty() {
|
|
# The admin account is created from ${COPAY_ACCOUNT}:${COPAY_PASSWORD} expanded
|
|
# at ExecStart time; a missing env file must fail the unit, not boot copyparty
|
|
# with a garbage admin password.
|
|
[[ -f /home/dank/Containers/copyparty/env ]] || {
|
|
echo "FAIL: /home/dank/Containers/copyparty/env does not exist; seed it before the first copyparty deploy" >&2
|
|
return 1
|
|
}
|
|
cp "dankserver/copyparty/quadlet/copyparty.container" "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
if ! systemctl --user is-active -q copyparty.service; then
|
|
podman stop -t 15 copyparty || true
|
|
podman rm copyparty || true
|
|
migrate_data_root /opt/copyparty /home/dank/Containers/copyparty
|
|
fi
|
|
systemctl --user restart copyparty.service
|
|
}
|
|
|
|
deploy_plex() {
|
|
cp "dankserver/plex/quadlet/plex.container" "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
if ! systemctl --user is-active -q plex.service; then
|
|
# First cutover: retire the native package unit BEFORE starting the
|
|
# container — both cannot own 32400. Mask (not disable): on these hosts
|
|
# `systemctl disable` deletes the unit file itself; mask keeps a
|
|
# one-command rollback while nothing is signed off yet.
|
|
run_privileged systemctl stop plexmediaserver.service
|
|
run_privileged systemctl mask plexmediaserver.service
|
|
# Rootless identity model: container root maps to the dank user
|
|
# (uid 1000) — real uid 997 is not mappable rootless. Adopt the data
|
|
# root into the fleet convention (jellyfin et al. run as 1000:1000).
|
|
# Rollback chowns it back to 997:984 before restarting the native unit.
|
|
run_privileged chown -R 1000:1000 /var/lib/plexmediaserver
|
|
fi
|
|
systemctl --user restart plex.service
|
|
# Readiness gate: /identity answers MediaContainer XML once PMS is up.
|
|
# Reality check from the 2026-09 cutover: the FIRST container boot of a
|
|
# years-old library ran ~10 min (PMS pre-binds 32400 during boot and
|
|
# empty-replies until the HTTP layer is ready), so the gate allows 600s.
|
|
# Fail the deploy (pending state preserved) rather than mark it deployed blind.
|
|
for ((i = 1; i <= 120; i++)); do
|
|
curl -fsS --max-time 3 http://127.0.0.1:32400/identity >/dev/null && return 0
|
|
sleep 5
|
|
done
|
|
echo "FAIL: plex did not answer /identity within 600s" >&2
|
|
return 1
|
|
}
|
|
|
|
deploy_tautulli() {
|
|
cp "dankserver/tautulli/quadlet/tautulli.container" "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
if ! systemctl --user is-active -q tautulli.service; then
|
|
run_privileged snap stop tautulli || true
|
|
run_privileged snap disable tautulli || true
|
|
mkdir -p /home/dank/Containers/tautulli/config
|
|
if [[ ! -f /home/dank/Containers/tautulli/config/config.ini ]]; then
|
|
printf '[General]\nPMS_Web_Host = plex\nPMS_Web_Port = 32400\nPMS_Secure = 0\n' \
|
|
> /home/dank/Containers/tautulli/config/config.ini
|
|
fi
|
|
fi
|
|
systemctl --user restart tautulli.service
|
|
}
|
|
|
|
deploy_authentik() {
|
|
local unit
|
|
mkdir -p "$quadlet_unit_dir"
|
|
for unit in authentik-postgres authentik-server authentik-worker; do
|
|
cp "dankclaw/authentik/quadlet/$unit.container" "$quadlet_unit_dir/"
|
|
done
|
|
systemctl --user daemon-reload
|
|
# Retire compose-era containers only while the units are not yet active;
|
|
# unit-managed containers self-heal via Restart=always.
|
|
if ! systemctl --user is-active --quiet authentik-postgres.service; then
|
|
podman stop authentik-postgres authentik-server authentik-worker 2>/dev/null || true
|
|
podman rm authentik-postgres authentik-server authentik-worker 2>/dev/null || true
|
|
fi
|
|
# No `systemctl --user enable` here: this host's quadlet generator
|
|
# (podman 5.x v1) generates units into /run/user/1000/systemd/generator/,
|
|
# and generated units cannot be enabled — enable fails with "transient or
|
|
# generated". Boot persistence comes from [Install] WantedBy=default.target
|
|
# inside each .container file, which the generator honors at user-manager
|
|
# start (linger must stay on for the deploy user).
|
|
systemctl --user start authentik-postgres.service
|
|
wait_container_healthy authentik-postgres 120 || return 1
|
|
systemctl --user start authentik-server.service
|
|
wait_container_healthy authentik-server 300 || return 1
|
|
systemctl --user start authentik-worker.service
|
|
}
|
|
|
|
deploy_authentik_outpost() {
|
|
# The outpost is the worker-role binary of the same server image; it dials
|
|
# back to core over websocket and serves /outpost.goauthentik.io/* for
|
|
# Caddy's forward_auth. Image already in storage — zero pulls. Restarting
|
|
# only this unit cycles auth for guarded vhosts for a few seconds; the
|
|
# rest of the stack is untouched.
|
|
cp dankclaw/authentik/quadlet/authentik-outpost.container "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart authentik-outpost.service
|
|
sleep 5
|
|
systemctl --user is-active --quiet authentik-outpost.service || \
|
|
echo "WARN: authentik-outpost did not come up active" >&2
|
|
}
|
|
|
|
deploy_authentik_outpost_ldap() {
|
|
# Dedicated LDAP outpost for Jellyfin's LDAP-Auth plugin. The type-specific
|
|
# ghcr.io/goauthentik/ldap image serves 3389/6636 container-side, published
|
|
# as host 389/636 for cross-host binds from dankserver. The outpost token
|
|
# lives in the host .env, never in this repo. Restarting only this unit
|
|
# cycles LDAP auth for a few seconds; the rest of the stack is untouched.
|
|
cp dankclaw/authentik/quadlet/authentik-outpost-ldap.container "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart authentik-outpost-ldap.service
|
|
sleep 5
|
|
systemctl --user is-active --quiet authentik-outpost-ldap.service || \
|
|
echo "WARN: authentik-outpost-ldap did not come up active" >&2
|
|
}
|
|
|
|
deploy_hermes() {
|
|
# The hermes trio (gateway, dashboard, webui) shares one image and the
|
|
# /data volume and deploys as one unit: install the units, retire the
|
|
# compose-era containers holding the claimed names while the units are not
|
|
# yet active, then restart each unit. An explicit hermes deploy is a
|
|
# deliberate action, so restarting an already-running trio is intended.
|
|
# Boot persistence comes from [Install] in each unit file (generated units
|
|
# cannot be enabled by this host's v1 quadlet generator).
|
|
local unit
|
|
mkdir -p "$quadlet_unit_dir"
|
|
# The CDP browser daemon is a companion service to the trio: install its
|
|
# unit and start it if inactive, but never restart it on deploy — a
|
|
# restart would kill in-flight browser sessions. Consumers (gateway,
|
|
# webui) reconnect with backoff.
|
|
cp "dankclaw/hermes/quadlet/chrome-devtools-protocol.container" "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
if ! systemctl --user is-active --quiet chrome-devtools-protocol.service; then
|
|
systemctl --user start chrome-devtools-protocol.service
|
|
fi
|
|
# The units run the image under a localhost/-aliased tag: this host's
|
|
# quadlet path can fail to resolve registry-prefixed refs from local
|
|
# storage at create time. Tag first, then (re)create via deploy.
|
|
podman tag docker.io/nousresearch/hermes-agent:latest localhost/hermes-agent:latest 2>/dev/null || true
|
|
for unit in hermes hermes-dashboard hermes-webui; do
|
|
cp "dankclaw/hermes/quadlet/$unit.container" "$quadlet_unit_dir/"
|
|
done
|
|
# The generator must see the freshly-copied unit files before the
|
|
# restarts below; without this reload the first deploy after a unit edit
|
|
# restarts with the stale generated ExecStart (statfs ENOENT on renamed
|
|
# Volume= sources).
|
|
systemctl --user daemon-reload
|
|
if ! systemctl --user is-active --quiet hermes.service; then
|
|
podman stop hermes hermes-dashboard hermes-webui 2>/dev/null || true
|
|
podman rm hermes hermes-dashboard hermes-webui 2>/dev/null || true
|
|
fi
|
|
systemctl --user restart hermes.service
|
|
systemctl --user restart hermes-dashboard.service
|
|
systemctl --user restart hermes-webui.service
|
|
}
|
|
|
|
deploy_anubis() {
|
|
# Anubis is the bot-challenge proxy in front of the guarded vhosts.
|
|
# The image lives on ghcr.io; pull once, then the unit runs it from
|
|
# local storage (Pull=never keeps boots off the registry).
|
|
if ! podman image exists ghcr.io/techarohq/anubis:v1.27.0; then
|
|
podman pull ghcr.io/techarohq/anubis:v1.27.0
|
|
fi
|
|
# The unit runs the localhost/ alias (see the unit file comment): this
|
|
# host's podman quadlet path cannot resolve registry-prefixed refs from
|
|
# local storage at create time. Tag first, then (re)create via deploy.
|
|
podman tag ghcr.io/techarohq/anubis:v1.27.0 localhost/techarohq/anubis:v1.27.0 2>/dev/null || true
|
|
cp "dankclaw/anubis/quadlet/anubis.container" "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
# No enable: generated units cannot be enabled on this host (v1 quadlet
|
|
# generator); boot persistence comes from [Install] in the unit file.
|
|
# restart, not start: policy.json is parsed at process start only (no
|
|
# reload signal exists), so every redeploy must cycle the process.
|
|
systemctl --user restart anubis.service
|
|
wait_container_healthy anubis 60 || return 1
|
|
}
|
|
|
|
# frank-large runs as a ROOT unit again since 2026-09-15: ik_llama.cpp
|
|
# (qwen4exp support merged upstream) replaced the halogen-flash-server
|
|
# rootless Quadlet, and the ik server is a native binary — CUDA on the RTX
|
|
# 3090 for dense/attention/KV, the Strix Halo CPU path for the MoE experts.
|
|
# There is no container image to pull and no generated unit to avoid:
|
|
# relink_unit points /etc/systemd/system/frank-large.service at the repo
|
|
# file, and boot persistence is the unit's [Install] section.
|
|
#
|
|
# The halogen Quadlet is retired first. Both managers must never know this
|
|
# unit name at once (they would race for port 7770): stop the user unit,
|
|
# poll until it is really gone, move the installed .container into the home
|
|
# trash (never rm), then reload the user manager.
|
|
LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M/Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf}"
|
|
|
|
retire_quadlet_llm_large() {
|
|
local trash="$HOME/.AGENTTRASH/20260915-ik-swap/halogen-quadlet"
|
|
systemctl --user stop frank-large.service 2>/dev/null || true
|
|
# A stopping quadlet runs podman stop (SIGTERM, then SIGKILL at the
|
|
# timeout); llama-family engines never preempt an in-flight generation.
|
|
# Poll until the user unit is really inactive before removing its file.
|
|
local i
|
|
for ((i = 0; i < 30; i++)); do
|
|
systemctl --user is-active --quiet frank-large.service 2>/dev/null || break
|
|
sleep 2
|
|
done
|
|
if [[ -e "$HOME/.config/containers/systemd/frank-large.container" || -L "$HOME/.config/containers/systemd/frank-large.container" ]]; then
|
|
mkdir -p "$trash"
|
|
mv -- "$HOME/.config/containers/systemd/frank-large.container" \
|
|
"$trash/frank-large.container"
|
|
echo "retired halogen quadlet -> $trash"
|
|
fi
|
|
systemctl --user daemon-reload
|
|
}
|
|
|
|
deploy_llm_large() {
|
|
retire_quadlet_llm_large
|
|
if [[ ! -f "$LLM_LARGE_CHECKPOINT" ]]; then
|
|
echo "ERROR: checkpoint absent at $LLM_LARGE_CHECKPOINT" >&2
|
|
echo "frank-large: download the mradermacher i1-Q4_K_M GGUF first; refusing to start a unit that cannot load." >&2
|
|
return 1
|
|
fi
|
|
relink_unit frank-large
|
|
run_privileged systemctl enable frank-large.service
|
|
run_privileged systemctl restart frank-large.service
|
|
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
|
}
|
|
|
|
# deploy_llm_medium — frank-medium runs as a rootless Quadlet since the
|
|
# 2026-09-15 runtime swap to the vllm-radiance MXFP4 stack. It began the day
|
|
# as a root llama.cpp unit (the ROCmFPX/Q8_0_ROCMFPX_AGENT deployment from
|
|
# the morning GPU swap); that root unit is retired here first — stop,
|
|
# disable, move the installed unit into the home trash. Same failure mode as
|
|
# frank-large's retirement: this host's `systemctl disable` has been
|
|
# observed to delete the unit file itself (2026-08-26 incident), so both
|
|
# outcomes are tolerated.
|
|
retire_root_llm_medium() {
|
|
local trash="$HOME/.AGENTTRASH/20260915-rocmfpx-retire/host-units"
|
|
run_privileged systemctl stop frank-medium.service 2>/dev/null || true
|
|
run_privileged systemctl disable frank-medium.service 2>/dev/null || true
|
|
if [[ -e /etc/systemd/system/frank-medium.service || -L /etc/systemd/system/frank-medium.service ]]; then
|
|
mkdir -p "$trash"
|
|
run_privileged mv -- /etc/systemd/system/frank-medium.service \
|
|
"$trash/frank-medium.service"
|
|
echo "retired root unit frank-medium.service -> $trash"
|
|
fi
|
|
run_privileged systemctl daemon-reload
|
|
}
|
|
|
|
deploy_llm_medium() {
|
|
retire_root_llm_medium
|
|
# Pull once, then run the localhost/-aliased tag: this host's quadlet
|
|
# path cannot resolve registry-prefixed refs from local storage at
|
|
# create time (same as frank-large on dankclaw). Pull=never keeps boots
|
|
# off the registry.
|
|
podman pull docker.io/stilldeadcode/vllm-radiance:0.9.3 >/dev/null 2>&1 || true
|
|
podman tag docker.io/stilldeadcode/vllm-radiance:0.9.3 localhost/radiance-vllm:0.9.3
|
|
# The compile cache must exist before the quadlet mounts resolve; the
|
|
# one-time setup (setup-mxfp4.sh / PREPARE_ONLY) normally creates it.
|
|
mkdir -p /home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq/{vllm,inductor,triton,aiter}
|
|
# Three units, not one. TP=2 across both R9700s cannot launch a collective
|
|
# on this host — RCCL's kernels need PCIe atomics and the second card's
|
|
# chipset path advertises AtomicOpsCap: Routing- — so the endpoint is two
|
|
# single-GPU replicas behind a round-robin proxy. frank-medium.container
|
|
# stays in the repo as the TP=2 target for a host whose second card sits
|
|
# on a CPU root port; it is deliberately NOT installed here.
|
|
install_quadlet frank-medium-a dankserver/frank-medium/quadlet frank-medium-a
|
|
install_quadlet frank-medium-b dankserver/frank-medium/quadlet frank-medium-b
|
|
install_quadlet frank-medium-proxy dankserver/frank-medium/quadlet frank-medium-proxy
|
|
# Drop a stale TP=2 unit from an earlier deploy of this service, or the
|
|
# generator keeps trying to start a container that can never become
|
|
# healthy. Never `rm`: it moves into the home trash.
|
|
if [[ -e "$quadlet_unit_dir/frank-medium.container" ]]; then
|
|
local trash="$HOME/.AGENTTRASH/20260916-frank-medium-tp2"
|
|
mkdir -p "$trash"
|
|
systemctl --user stop frank-medium.service 2>/dev/null || true
|
|
mv -- "$quadlet_unit_dir/frank-medium.container" "$trash/frank-medium.container"
|
|
systemctl --user daemon-reload
|
|
echo "retired the TP=2 quadlet -> $trash (see the runbook for why)"
|
|
fi
|
|
# The proxy waits on the replicas, but a restart of all three is what a
|
|
# deploy does: the replicas reload the model, the proxy reconnects.
|
|
systemctl --user restart frank-medium-a.service frank-medium-b.service frank-medium-proxy.service
|
|
wait_for_health frank-medium http://127.0.0.1:7770/health 1800
|
|
}
|
|
|
|
wait_for_health() {
|
|
local service="$1" url="$2" limit="$3"
|
|
local i
|
|
for ((i=1; i<=limit; i++)); do
|
|
if curl -fsS "$url" >/dev/null 2>&1; then
|
|
echo "$service healthy after approximately $((i * 2))s"
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "FAIL: $service did not become healthy in $((limit * 2))s" >&2
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# FrankBook-Pro (macOS): quadlet units live INSIDE the podman machine VM
|
|
# (/home/core/.config/containers/systemd), not in the macOS user session.
|
|
# This handler pipes unit files and seed logic into the VM over
|
|
# `podman machine ssh`; it never touches systemctl on the Mac host and never
|
|
# uses run_privileged (the Mac account is not passwordless sudo). Unit files
|
|
# are re-piped on every call (idempotent, cheap) before the selected unit is
|
|
# restarted; first boot also seeds the qBittorrent config once.
|
|
FRANKBOOK_SVCS=(gluetun qbittorrent prowlarr sonarr radarr lidarr readarr)
|
|
deploy_frankbook_stack() {
|
|
local service="$1"
|
|
export PATH="/usr/local/bin:/opt/homebrew/bin:$PATH"
|
|
if ! podman machine inspect --format '{{.State}}' 2>/dev/null | grep -qx running; then
|
|
echo "FAIL: podman machine is not running on frankbook-pro" >&2
|
|
return 1
|
|
fi
|
|
local vm_dir="/home/core/.config/containers/systemd" svc
|
|
podman machine ssh "mkdir -p '$vm_dir' /home/core/Containers/gluetun /home/core/Containers/qbittorrent"
|
|
for svc in "${FRANKBOOK_SVCS[@]}"; do
|
|
podman machine ssh "cat > '$vm_dir/$svc.container'" \
|
|
< "frankbook-pro/$svc/quadlet/$svc.container"
|
|
done
|
|
local tmp
|
|
tmp="$(mktemp /tmp/frankbook-vm.XXXXXX)"
|
|
cat > "$tmp" <<'VM_SEED'
|
|
set -euo pipefail
|
|
# qBittorrent first-boot seed: conf + categories + machine-local PBKDF2
|
|
# password. qB >= 4.2 ignores plaintext passwords, so the conf carries the
|
|
# @ByteArray(salt:hash) form (PBKDF2-HMAC-SHA512, 100k iterations, 64 bytes).
|
|
if [ ! -e /home/core/Containers/qbittorrent/qbit-seeded ]; then
|
|
mkdir -p /Volumes/COMPLETE/qbittorrent-data/qBittorrent
|
|
QBIT_PW="$(openssl rand -base64 18 | tr -d '/+=' | head -c 20)"
|
|
QBIT_HASH="$(python3 - "$QBIT_PW" <<'PYH'
|
|
import hashlib, base64, os, sys
|
|
pw = sys.argv[1].encode()
|
|
salt = os.urandom(16)
|
|
h = hashlib.pbkdf2_hmac("sha512", pw, salt, 100000, dklen=64)
|
|
print("@ByteArray(%s:%s)" % (base64.b64encode(salt).decode(), base64.b64encode(h).decode()))
|
|
PYH
|
|
)"
|
|
umask 077
|
|
printf 'QBIT_WEBUI_PASSWORD=%s\n' "$QBIT_PW" > /home/core/Containers/qbittorrent/env
|
|
umask 022
|
|
cat > /Volumes/COMPLETE/qbittorrent-data/qBittorrent/qBittorrent.conf <<QCONF
|
|
[BitTorrent]
|
|
Session\\DefaultSavePath=/Volumes/COMPLETE
|
|
Session\\Port=6881
|
|
Session\\QueueingSystemEnabled=true
|
|
Session\\TempPath=/Volumes/INCOMPLETE/incomplete
|
|
Session\\TempPathEnabled=true
|
|
|
|
[LegalNotice]
|
|
Accepted=true
|
|
|
|
[Meta]
|
|
MigrationVersion=8
|
|
|
|
[Network]
|
|
Cookies\\Enabled=true
|
|
|
|
[Preferences]
|
|
WebUI\\Address=*
|
|
WebUI\\ServerDomains=*
|
|
WebUI\\HostHeaderValidation=false
|
|
WebUI\\Username=admin
|
|
WebUI\\Password_PBKDF2=$QBIT_HASH
|
|
QCONF
|
|
cat > /Volumes/COMPLETE/qbittorrent-data/qBittorrent/categories.json <<'QCAT'
|
|
{
|
|
"Finished Torrents": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
|
|
"lidarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
|
|
"radarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
|
|
"tv-sonarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"}
|
|
}
|
|
QCAT
|
|
touch /home/core/Containers/qbittorrent/qbit-seeded
|
|
echo " seeded qBittorrent config (WebUI password: /home/core/Containers/qbittorrent/env inside the VM)"
|
|
fi
|
|
VM_SEED
|
|
podman machine ssh bash -s < "$tmp"
|
|
rm -f "$tmp"
|
|
# Credential gate: without a WireGuard env the tunnel stays parked, and
|
|
# qBittorrent (which downloads through it) waits with it. The arrs run
|
|
# without the tunnel, so only these two services are gated.
|
|
case "$service" in
|
|
gluetun|qbittorrent)
|
|
if ! podman machine ssh '[ -s /home/core/Containers/gluetun/env ]' >/dev/null 2>&1; then
|
|
echo "SKIP: $service waits for gluetun credentials."
|
|
echo " Write the WireGuard env into /home/core/Containers/gluetun/env inside the VM"
|
|
echo " (podman machine ssh), then re-run: ./scripts/deploy.sh frankbook-pro --service gluetun --service qbittorrent"
|
|
return 0
|
|
fi
|
|
;;
|
|
esac
|
|
tmp="$(mktemp /tmp/frankbook-vm.XXXXXX)"
|
|
cat > "$tmp" <<'VM_START'
|
|
set -euo pipefail
|
|
SVC="${1:?service required}"
|
|
systemctl --user daemon-reload
|
|
unit="$(systemctl --user list-units --all --no-legend "${SVC}.service" "container-${SVC}.service" 2>/dev/null | sed 's/^● //' | head -n1 | awk '{print $1}')"
|
|
[ -n "$unit" ] || { echo "FAIL: no generated systemd unit for $SVC" >&2; exit 1; }
|
|
systemctl --user restart "$unit"
|
|
sleep 3
|
|
state="$(systemctl --user is-active "$unit" || true)"
|
|
if [ "$state" != active ] && [ "$state" != activating ]; then
|
|
echo "FAIL: $unit is $state" >&2
|
|
systemctl --user status "$unit" --no-pager -l || true
|
|
exit 1
|
|
fi
|
|
VM_START
|
|
podman machine ssh bash -s "$service" < "$tmp"
|
|
rm -f "$tmp"
|
|
local port path
|
|
case "$service" in
|
|
qbittorrent) port=8080; path=/ ;;
|
|
prowlarr) port=9696; path=/ping ;;
|
|
sonarr) port=8989; path=/ping ;;
|
|
radarr) port=7878; path=/ping ;;
|
|
lidarr) port=8686; path=/ping ;;
|
|
readarr) port=8787; path=/ping ;;
|
|
*) return 0 ;; # gluetun publishes only qBittorrent's ports
|
|
esac
|
|
printf ' waiting for http://127.0.0.1:%s%s on the Mac...\n' "$port" "$path"
|
|
local ok=0 i
|
|
for i in $(seq 1 60); do
|
|
if curl -fsS -m 4 "http://127.0.0.1:${port}${path}" >/dev/null 2>&1; then
|
|
ok=1
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
[[ "$ok" -eq 1 ]] || { echo "FAIL: $service did not answer on 127.0.0.1:$port" >&2; return 1; }
|
|
echo " $service answers on 127.0.0.1:$port"
|
|
}
|
|
|
|
deploy_seerr() {
|
|
install_quadlet seerr dankclaw/seerr/quadlet seerr
|
|
systemctl --user restart seerr.service
|
|
}
|
|
|
|
deploy_service() {
|
|
local service="$1"
|
|
if [[ "$TARGET" == "frankbook-pro" ]]; then
|
|
# All frankbook-pro services route through the Mac handler: the name
|
|
# case below addresses the Linux hosts, and service names overlap.
|
|
deploy_frankbook_stack "$service"
|
|
return
|
|
fi
|
|
case "$service" in
|
|
caddy) deploy_caddy ;;
|
|
homepage) deploy_homepage ;;
|
|
searxng) deploy_searxng ;;
|
|
gluetun) deploy_gluetun ;;
|
|
searxng-valkey) deploy_searxng_valkey ;;
|
|
open-webui) deploy_open_webui ;;
|
|
pihole) deploy_pihole ;;
|
|
gitea-postgres) deploy_gitea_unit gitea-postgres gitea-postgres ;;
|
|
gitea-server) deploy_gitea_unit gitea-server gitea-server ;;
|
|
authentik-postgres|authentik-server|authentik-worker)
|
|
deploy_authentik
|
|
;;
|
|
authentik-outpost)
|
|
deploy_authentik_outpost
|
|
;;
|
|
authentik-outpost-ldap)
|
|
deploy_authentik_outpost_ldap
|
|
;;
|
|
anubis) deploy_anubis ;;
|
|
seerr) deploy_seerr ;;
|
|
marreta) deploy_marreta ;;
|
|
frank-large) deploy_llm_large ;;
|
|
frank-medium) deploy_llm_medium ;;
|
|
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
|
|
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
|
|
jellyfin) deploy_jellyfin ;;
|
|
calibre-web) deploy_calibre_web ;;
|
|
copyparty) deploy_copyparty ;;
|
|
plex) deploy_plex ;;
|
|
tautulli) deploy_tautulli ;;
|
|
prometheus|grafana|loki|alloy|node-exporter) deploy_observability "$service" ;;
|
|
*) deploy_compose_service "$service" ;;
|
|
esac
|
|
}
|
|
|
|
if [[ -n "$FROM_REVISION" ]]; then
|
|
# Register every changed service before applying the requested scope. This
|
|
# preserves an explicit --service selection as pending work for the next
|
|
# ordinary deployment.
|
|
register_changed_services "$FROM_REVISION"
|
|
fi
|
|
if [[ "$PENDING" -eq 1 ]]; then
|
|
add_changed_services "$FROM_REVISION"
|
|
add_pending_services
|
|
fi
|
|
|
|
# Remove duplicate service names while preserving caller/planner order.
|
|
UNIQUE_SERVICES=()
|
|
declare -A SEEN_SERVICES=()
|
|
for service in "${SERVICES[@]}"; do
|
|
[[ -n "$service" ]] || continue
|
|
if [[ -z "${SEEN_SERVICES[$service]:-}" ]]; then
|
|
UNIQUE_SERVICES+=("$service")
|
|
SEEN_SERVICES[$service]=1
|
|
fi
|
|
done
|
|
SERVICES=("${UNIQUE_SERVICES[@]}")
|
|
|
|
if [[ "${#SERVICES[@]}" -eq 0 ]]; then
|
|
echo "No services selected; nothing to deploy."
|
|
exit 0
|
|
fi
|
|
|
|
for service in "${SERVICES[@]}"; do
|
|
[[ "$service" =~ ^[a-z0-9][a-z0-9-]*$ ]] || { echo "invalid service: $service" >&2; exit 2; }
|
|
service_is_allowed "$service" || { echo "service $service is not deployed on $TARGET" >&2; exit 2; }
|
|
echo "=== Deploying $service at $REVISION ==="
|
|
mark_status "$service" pending
|
|
deploy_service "$service"
|
|
mark_deployed "$service"
|
|
echo " STATE[$service] = $REVISION (status=deployed)"
|
|
done
|