dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
7f65cf02523ff8065047f0cfb00481f904fff231
serverAppConfigs/scripts/deploy-remote.sh
T
Frankandomp 346e1d22f4 feat(frank-dragoon): put the endpoint on the app bridge so app containers can name it 
An endpoint on podman's rootless default (pasta) has no address another
container can use, measured on 2026-10-19 from inside open-notebook:

  - 192.168.0.143:7770            connection refused (pasta forwards the
                                  published port in the host netns; no netavark
                                  DNAT exists for traffic arriving from a
                                  rootless bridge)
  - host.containers.internal:7770 200, but the alias resolves to 169.254.1.2 —
                                  link-local, which open-notebook's URL guard
                                  refuses to store
  - frank-dragoon-halogen:8731    name unknown: different netns
  - --network pasta --network dankclaw_default
                                  "cannot set multiple networks without bridge
                                  network mode" — pasta cannot be added to

So reaching the NPU endpoints from an app container means moving the container
onto dankclaw_default (FRANK_DRAGOON_NETWORK; empty keeps today's shape). This
is not an exposure change: the engine's unauthenticated token port binds the
container's own loopback (/proc/net/tcp: 127.0.0.1:8730 against 0.0.0.0:8731),
so only the API port becomes name-resolvable — the same trust the published
7770 already carries on the LAN. The deployer refuses a network that does not
exist rather than starting a unit that restart-loops on it.

Co-Authored-By: omp <omp@dankclaw>
2026-10-19 21:27:07 -07:00

1962 lines
93 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# deploy-remote.sh — apply an explicit service plan on one target host.
#
# This script runs on the target checkout. It deliberately does not infer a
# plan, pull Git, or restart an unrelated service. deploy.sh computes the plan
# from the pushed revision and passes the selected service names here.
# Per-service revisions live in /var/lib/serverAppConfigs/deploy-state (or
# DEPLOY_STATE_DIR), outside the checkout, so an unselected change remains
# pending until its service is explicitly deployed.
set -euo pipefail
TARGET="${1:-}"
REVISION="${2:-}"
shift 2 || true
FROM_REVISION=""
PENDING=0
SERVICES=()
usage() {
cat <<'EOF'
Usage: deploy-remote.sh <dankclaw|dankserver|frankbook-pro> <git-revision> [--pending] [service]...
The service list is explicit. An empty list is a successful no-op.
EOF
}
while [[ "$#" -gt 0 ]]; do
case "$1" in
--from-revision)
[[ "$#" -ge 2 ]] || { echo "--from-revision requires a revision" >&2; exit 2; }
FROM_REVISION="$2"
shift 2
;;
--pending)
PENDING=1
shift
;;
--help|-h)
usage
exit 0
;;
*)
SERVICES+=("$1")
shift
;;
esac
done
REMOTE_DIR="${REMOTE_DIR:-/home/dank/serverAppConfigs}"
STATE_DIR="${DEPLOY_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/serverAppConfigs/deploy-state}"
run_privileged() {
if [[ -n "${FRANK_SUDO_PW:-}" ]]; then
printf '%s\n' "$FRANK_SUDO_PW" | sudo -S -- "$@"
else
sudo -- "$@"
fi
}
if [[ "$TARGET" != dankclaw && "$TARGET" != dankserver && "$TARGET" != frankbook-pro ]]; then
usage >&2
exit 2
fi
if [[ -z "$REVISION" ]]; then
usage >&2
exit 2
fi
if [[ "$REVISION" == *[!0-9a-fA-F]* ]]; then
echo "invalid Git revision: $REVISION" >&2
exit 2
fi
if ! git -C "$REMOTE_DIR" cat-file -e "$REVISION^{commit}" 2>/dev/null; then
echo "revision is not a commit in the target checkout: $REVISION" >&2
exit 2
fi
cd "$REMOTE_DIR"
# Keep the state directory outside Git. The file is written only after a
# handler completes successfully; an interrupted handler remains pending.
mkdir -p "$STATE_DIR"
service_is_allowed() {
local service="$1"
case "$TARGET:$service" in
dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\
dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
dankclaw:comfyui|dankclaw:hermes|\
dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\
dankclaw:authentik-postgres|dankclaw:authentik-server|\
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
dankclaw:open-notebook|\
dankclaw:frank-dragoon|dankclaw:frank-zealot|\
dankclaw:frank-tts-engine|dankclaw:frank-tts|\
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
dankserver:frank-archon|dankserver:node-exporter|\
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
frankbook-pro:readarr)
return 0
;;
*)
return 1
;;
esac
}
mark_status() {
local service="$1" status="$2" tmp
tmp="$(mktemp "$STATE_DIR/.${service}.XXXXXX")"
printf 'revision=%s\nstatus=%s\n' "$REVISION" "$status" > "$tmp"
mv -f -- "$tmp" "$STATE_DIR/$service"
}
mark_deployed() {
mark_status "$1" deployed
}
state_revision() {
local service="$1" line value
if [[ -r "$STATE_DIR/$service" ]]; then
while IFS= read -r line; do
case "$line" in
revision=*) value="${line#revision=}" ;;
esac
done < "$STATE_DIR/$service"
printf '%s' "${value:-}"
fi
}
state_status() {
local service="$1" line value=pending
if [[ -r "$STATE_DIR/$service" ]]; then
while IFS= read -r line; do
case "$line" in
status=*) value="${line#status=}" ;;
esac
done < "$STATE_DIR/$service"
fi
printf '%s' "$value"
}
register_changed_services() {
local base="$1" service recorded status
[[ -n "$base" ]] || return 0
while IFS= read -r service; do
[[ -n "$service" ]] || continue
recorded="$(state_revision "$service")"
status="$(state_status "$service")"
if [[ "$status" != deployed || "$recorded" != "$REVISION" ]]; then
mark_status "$service" pending
fi
done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
--base "$base" --head "$REVISION")
}
add_changed_services() {
local base="$1"
local service
[[ -n "$base" ]] || return 0
while IFS= read -r service; do
[[ -n "$service" ]] || continue
SERVICES+=("$service")
done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
--base "$base" --head "$REVISION")
}
add_pending_services() {
local service recorded status changed
for state_file in "$STATE_DIR"/*; do
[[ -f "$state_file" ]] || continue
service="${state_file##*/}"
recorded="$(state_revision "$service")"
status="$(state_status "$service")"
if [[ "$status" != deployed ]]; then
# A pending service was intentionally skipped or its handler
# failed. Retry it even when the checkout has not advanced.
# But a pending entry for a service RETIRED from this target can
# never be deployable: retrying it fails the whole run before
# the e2e gate (the frank-default-sightsound lockout, found
# 2026-09-17). Drop such entries into the host trash instead.
if ! service_is_allowed "$service"; then
local trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-retired-deploy-state"
mkdir -p "$trash"
mv -- "$state_file" "$trash/$service"
echo "WARN: $service is retired on $TARGET; dropped its stale $status state to $trash" >&2
continue
fi
SERVICES+=("$service")
continue
fi
[[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
if [[ "$TARGET" == dankserver && "$service" == frank-dragoon ]]; then continue; fi
if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
--repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
echo "WARN: unable to compare state for $service; retaining it as pending" >&2
SERVICES+=("$service")
continue
fi
if grep -Fxq "$service" <<< "$changed"; then
SERVICES+=("$service")
fi
done
}
copy_searxng_config() {
run_privileged mkdir -p /opt/searxng/config
run_privileged cp shared/searxng/settings.yml /opt/searxng/config/settings.yml
run_privileged cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml
# Bake ${KAGI_API_KEY}/${SEARXNG_SECRET}/${SEARXNG_BASE_URL} into the live
# copy from the host-side /opt/searxng/.env (SearXNG does not expand env
# vars in settings.yml). Fails the deploy if a placeholder is unresolvable.
run_privileged python3 scripts/searxng-render-settings.py \
/opt/searxng/.env /opt/searxng/config/settings.yml
}
copy_prometheus_config() {
run_privileged mkdir -p /opt/prometheus/config /opt/prometheus/config/rules
run_privileged cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml
# Rule files are read from the same mount (--config.file sits beside them),
# so they deploy with the config that references them: a rule file without
# its rule_files entry evaluates nothing, silently.
run_privileged cp shared/prometheus/rules/llm-fleet.yml \
/opt/prometheus/config/rules/llm-fleet.yml
}
# Alertmanager config ships beside the Prometheus config: the two halves of
# the alerting path change together, and a receiver with no route (or a
# route with no receiver) fails silently.
copy_alertmanager_config() {
run_privileged mkdir -p /opt/alertmanager/config /opt/alertmanager/data
run_privileged cp shared/prometheus/alertmanager.yml /opt/alertmanager/config/alertmanager.yml
}
# The Signal relay is Alertmanager's only webhook consumer and the only
# thing on this host that talks to signal-cli. It is a plain systemd USER
# unit (not a quadlet): it runs python3 from the checkout, so a deploy
# updates it without a copy step, and — unlike generated quadlet units,
# which this host's podman 5.x v1 generator rejects — a hand-written user
# unit can be enabled, and Linger=yes for dank starts it at boot.
install_signal_relay() {
local unit_dir="$HOME/.config/systemd/user"
mkdir -p "$unit_dir"
cp scripts/signal-relay.service "$unit_dir/signal-relay.service"
systemctl --user daemon-reload
systemctl --user enable signal-relay.service
systemctl --user restart signal-relay.service
# The relay binds the host's LAN IP (the one address a bridge container
# can reach as host.containers.internal), so keep the unauthenticated
# endpoint off the rest of the LAN: admit loopback, the host itself
# (its own health probe) and the podman bridge subnet only. Idempotent:
# -C before -I.
run_privileged iptables -C INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT 2>/dev/null \
|| run_privileged iptables -I INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT
run_privileged iptables -C INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT 2>/dev/null \
|| run_privileged iptables -I INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT
run_privileged iptables -C INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT 2>/dev/null \
|| run_privileged iptables -I INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT
run_privileged iptables -C INPUT -p tcp --dport 18093 -j DROP 2>/dev/null \
|| run_privileged iptables -I INPUT -p tcp --dport 18093 -j DROP
# The interpreter needs a moment to bind after restart; a one-shot curl
# raced the socket on 2026-09-17 and failed a healthy deploy.
local i
for i in $(seq 1 15); do
if curl -fsS -m 5 http://192.168.0.143:18093/health >/dev/null; then
return 0
fi
sleep 1
done
echo "signal-relay is not answering on 192.168.0.143:18093" >&2
return 1
}
copy_loki_config() {
run_privileged mkdir -p /opt/loki/config
run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
}
copy_alloy_config() {
run_privileged mkdir -p /opt/alloy/config
run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
}
copy_grafana_provisioning() {
# Datasources and dashboards are declared in git, never clicked into
# grafana.db: this Grafana is an empty install, and a UI-configured
# datasource would be invisible to review, lost with the data volume, and
# unknown to anyone reading the repository. The unit already mounts
# /opt/grafana/provisioning read-only, so this only has to populate it.
run_privileged mkdir -p /opt/grafana/provisioning/datasources \
/opt/grafana/provisioning/dashboards
run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \
/opt/grafana/provisioning/datasources/prometheus.yml
run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \
/opt/grafana/provisioning/dashboards/provider.yml
run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \
/opt/grafana/provisioning/dashboards/llm-fleet.json
}
render_caddyfile() {
local hash
hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
if [[ -z "$hash" ]]; then
echo "WARN: CADDY_BASICAUTH_HASH is not set; refusing to overwrite rendered Caddyfile" >&2
return 1
fi
run_privileged mkdir -p /home/dank/Containers/caddy
sed -e "s|REPLACE_WITH_HASH|$hash|g" dankclaw/Caddyfile > /tmp/serverAppConfigs.Caddyfile
run_privileged cp /tmp/serverAppConfigs.Caddyfile /home/dank/Containers/caddy/Caddyfile
}
deploy_homepage() {
if [[ "$TARGET" != dankclaw ]]; then
echo "homepage is owned by dankclaw; refusing to deploy it on $TARGET" >&2
return 2
fi
run_privileged mkdir -p /srv/homepage
run_privileged cp shared/homepage/index.html shared/homepage/admin.html shared/homepage/app.js \
shared/homepage/style.css shared/homepage/services.json shared/homepage/docs.html /srv/homepage/
if [[ -d shared/homepage/fonts ]]; then
run_privileged cp -r shared/homepage/fonts /srv/homepage/
fi
if [[ -d shared/homepage/img ]]; then
run_privileged cp -r shared/homepage/img /srv/homepage/
fi
if [[ -d shared/homepage/sfx ]]; then
run_privileged cp -r shared/homepage/sfx /srv/homepage/
fi
local deployed_at
deployed_at="$(git log -1 --format=%cI)"
printf 'window.DEPLOYED_AT = "%s";\n' "$deployed_at" > /tmp/deployed.js
run_privileged cp /tmp/deployed.js /srv/homepage/deployed.js
run_privileged chmod -R a+rX /srv/homepage
run_privileged cp scripts/homepage-status.py /usr/local/bin/homepage-status.py
run_privileged cp scripts/homepage-status.service scripts/homepage-status.timer /etc/systemd/system/
run_privileged systemctl daemon-reload
run_privileged systemctl enable homepage-status.timer
run_privileged systemctl restart homepage-status.timer
run_privileged python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json
}
stage_legacy_gitea_caddy_state() {
local legacy_state="$STATE_DIR/gitea-caddy"
[[ -e "$legacy_state" ]] || return 0
local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
mkdir -p "$legacy_trash"
mv -- "$legacy_state" "$legacy_trash/gitea-caddy.state"
}
retire_legacy_gitea_caddy() {
# The old split proxy owns ports 80/443 on hosts upgraded in place. Stop
# and disable its generated unit before the canonical Caddy is started.
if systemctl --user is-active --quiet gitea-caddy.service 2>/dev/null; then
systemctl --user stop gitea-caddy.service
fi
if systemctl --user is-enabled --quiet gitea-caddy.service 2>/dev/null; then
# A Quadlet-generated unit may report "generated" without having an
# enablement symlink. Disable is therefore best-effort; moving the
# source out of the generator directory is the durable action.
systemctl --user disable gitea-caddy.service 2>/dev/null || true
fi
local unit_dir="$HOME/.config/containers/systemd"
local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
# Podman generators scan the unit directory recursively on this host, so
# leaving the retired file in systemd/.AGENTTRASH would regenerate it.
# Stage it outside the unit tree in the dated home trash instead.
for unit_file in "$unit_dir/gitea-caddy.container" \
"$unit_dir/.AGENTTRASH/gitea-caddy.container"; do
if [[ -e "$unit_file" ]]; then
mkdir -p "$legacy_trash"
mv -- "$unit_file" "$legacy_trash/"
fi
done
stage_legacy_gitea_caddy_state
systemctl --user daemon-reload
if podman container exists gitea-caddy; then
podman rm --force gitea-caddy
fi
}
deploy_caddy() {
if [[ "$TARGET" != dankclaw ]]; then
echo "caddy is owned by dankclaw; refusing to deploy it on $TARGET" >&2
return 2
fi
render_caddyfile
deploy_homepage
local cfhash oldhash image
cfhash="$(sha256sum dankclaw/caddy/Containerfile | cut -d' ' -f1)"
image=franksdank/caddy-porkbun
oldhash="$(podman image inspect "$image:latest" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
if [[ "$oldhash" != "$cfhash" ]]; then
podman build --no-cache --label "cfhash=$cfhash" -t "$image:latest" \
-f dankclaw/caddy/Containerfile .
fi
# Retire the old split proxy before claiming ports 80/443. This is
# idempotent and leaves the old unit recoverable in the target trash.
retire_legacy_gitea_caddy
# Caddy's env: Porkbun DNS-01 credentials from the checkout's .env. The
# live file holds secrets, so it exists only on the host; the quadlet
# reads it via EnvironmentFile.
mkdir -p /home/dank/Containers/caddy
if [[ ! -f /home/dank/Containers/caddy/env ]]; then
awk -F= '/^PORKBUN_API_KEY=|^PORKBUN_API_SECRET_KEY=/ { print }' \
dankclaw/.env > /home/dank/Containers/caddy/env
fi
local var
for var in PORKBUN_API_KEY PORKBUN_API_SECRET_KEY; do
grep -q "^${var}=." /home/dank/Containers/caddy/env \
|| { echo "caddy env missing ${var}; set it in dankclaw/.env" >&2; return 1; }
done
# Retire the bare compose container: all state (Caddyfile, certs, config)
# lives in host volumes, so removing the container loses nothing.
podman stop caddy >/dev/null 2>&1 || true
podman rm caddy >/dev/null 2>&1 || true
# install_quadlet runs daemon-reload, which re-runs podman's quadlet
# generator: it creates caddy.service AND the default.target.wants symlink
# from the unit's [Install] section (is-enabled reports "generated" —
# that is the healthy state for a quadlet; an explicit systemctl enable
# fails on generated units). Boot persistence comes from the generator,
# same as pihole/gitea/searxng/open-webui.
install_quadlet caddy dankclaw/caddy/quadlet caddy
systemctl --user restart caddy.service
if podman container exists caddy; then
local reloaded=0
for _ in 1 2 3 4 5; do
if podman exec caddy caddy reload --config /etc/caddy/Caddyfile \
--adapter caddyfile >/dev/null 2>&1; then
reloaded=1
echo "Caddy reloaded."
break
fi
sleep 3
done
if [[ "$reloaded" -ne 1 ]]; then
echo "Caddy reload failed; restarting only caddy."
podman restart caddy
fi
fi
}
deploy_compose_service() {
local service="$1"
if [[ "$TARGET" == dankserver ]]; then
# Dankserver's compose file is the source-of-truth inventory, but its
# production containers are individually systemd-managed/bare. Do not
# use compose up here: it can create a second container with a wrong
# name and can interfere with a running workload.
echo "WARN: $service is recorded in dankserver/compose.yaml; production container management is host-specific and no restart was attempted."
return 0
fi
if [[ "$TARGET" == dankclaw ]]; then
# Dankclaw has no compose file since 2026-09-16: every service is a
# Quadlet unit with its own handler above. Reaching this means the
# dispatcher is missing a case, and the old fallback (`podman-compose
# up -d`) would now be a silent no-op or a duplicate container.
echo "no compose services on dankclaw; $service has no quadlet handler" >&2
return 2
fi
case "$service" in
*)
echo "unknown compose service: $service" >&2
return 2
;;
esac
}
# grafana is the one observability service that holds a credential: the
# Authentik OAuth2 client secret. It is seeded into a per-service env file (the
# fleet convention: caddy, pihole, copyparty, romm, seerr) instead of being
# imported from the repo .env wholesale. The repo-root .env that the compose
# service used to pick up is a gluetun env file — it also carries HOME=/root
# and the WireGuard keys, and an EnvironmentFile of that file sets HOME for the
# service, which makes podman look for its config under /root and fail.
seed_grafana_env() {
local dest=/home/dank/Containers/grafana/env
mkdir -p /home/dank/Containers/grafana
if [[ ! -f "$dest" ]]; then
awk -F= '/^GRAFANA_OAUTH_CLIENT_SECRET=/ { print }' dankclaw/.env > "$dest"
chmod 600 "$dest"
fi
grep -q '^GRAFANA_OAUTH_CLIENT_SECRET=.' "$dest" \
|| { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; }
}
# The five observability containers left compose on 2026-09-11 (they were
# started by hand with podman-compose and never came back after a reboot).
# They are ordinary rootless Quadlet units now: same images, same mounts, same
# arguments, same container names, so nothing that addresses them by name —
# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
# compose services published on 0.0.0.0 and none of them authenticates.
deploy_observability() {
local service="$1"
# Observability units live under <target>/observability/quadlet: Dankclaw
# has six, Dankserver one (node-exporter, since 2026-09-16). The config
# copies below are Dankclaw-owned state and run only there.
local unit_dir="$TARGET/observability/quadlet"
case "$service" in
prometheus)
copy_prometheus_config
;;
alertmanager)
copy_alertmanager_config
install_signal_relay
;;
loki)
copy_loki_config
build_compose_image loki
;;
alloy)
copy_alloy_config
build_compose_image alloy
;;
grafana)
copy_grafana_provisioning
seed_grafana_env || return 1
;;
node-exporter)
;;
*)
echo "unknown observability service: $service" >&2
return 2
;;
esac
install_quadlet observability "$unit_dir" "$service"
systemctl --user restart "$service.service"
case "$service" in
prometheus) wait_for_health prometheus http://127.0.0.1:9090/-/healthy 60 ;;
alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
esac
}
build_compose_image() {
local service="$1" image cfhash oldhash containerfile
# Every file the image COPIES IN, beside the Containerfile. The skip below
# compares a hash against the image's `cfhash` label, so anything the build
# bakes must be in that hash or the deploy silently keeps serving the old
# bytes: an edited app.py or warm hook leaves the Containerfile untouched,
# the label matches, `podman build` never runs, and the health gate passes on
# a stale container. Services whose Containerfile pulls everything at
# build time (comfyui, loki, alloy) keep an empty list.
local -a extra=()
case "$service" in
comfyui)
image=comfyui-rocm:gfx1151
containerfile=dankclaw/Containerfile.comfyui
;;
loki|alloy)
image="franksdank/$service:latest"
containerfile="dankclaw/$service/Containerfile"
;;
frank-tts)
# The 7772 façade: stdlib python + ffmpeg, app.py copied in. Listed
# explicitly because build_compose_image's `*)` arm returns 0 — an
# unlisted service builds nothing and the unit's Pull=never then
# fails on a tag that was never made.
image=franksdank/frank-tts:latest
containerfile=dankclaw/voiceservice/Containerfile
extra=(dankclaw/voiceservice/app.py)
;;
frank-tts-engine)
# The arm A TTS engine (qwentts.cpp `tts-server`). Built from
# source at a pinned upstream commit, so the build needs network
# and the cfhash label below is what keeps an unchanged re-deploy
# offline: the clone only happens when this Containerfile changes.
image=franksdank/frank-tts-engine:latest
containerfile=dankclaw/voiceservice/Containerfile.engine
# start.sh is the image's CMD and the warmer is the unit's
# ExecStartPost; both are COPY'd in, so both decide what this image is.
extra=(dankclaw/voiceservice/tts-engine-start.sh
dankclaw/voiceservice/tts-engine-warm.sh)
;;
*)
return 0
;;
esac
cfhash="$(sha256sum "$containerfile" "${extra[@]}" | cut -d' ' -f1 | tr -d '\n')"
oldhash="$(podman image inspect "$image" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
if [[ "$oldhash" != "$cfhash" ]]; then
# --format docker is required, not cosmetic: podman defaults to the OCI
# image format, which has no HEALTHCHECK field at all, so a build in
# the default format silently drops the loki image's healthcheck and
# the container then reports no health instead of failing loudly.
podman build --format docker --no-cache --label "cfhash=$cfhash" \
-t "$image" -f "$containerfile" .
fi
}
quadlet_unit_dir="$HOME/.config/containers/systemd"
# dankserver's user manager had no systemd container generator dir before the
# quadlet migration; every handler that installs a unit must find it present.
mkdir -p "$quadlet_unit_dir"
install_quadlet() {
local service="$1"
local source_dir="$2"
local unit="$3"
mkdir -p "$quadlet_unit_dir"
cp "$source_dir/$unit.container" "$quadlet_unit_dir/"
if [[ -f "$source_dir/$unit.network" ]]; then
cp "$source_dir/$unit.network" "$quadlet_unit_dir/"
fi
systemctl --user daemon-reload
}
deploy_searxng() {
copy_searxng_config
install_quadlet searxng dankclaw/searxng/quadlet searxng
install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
install_quadlet gluetun dankclaw/searxng/quadlet gluetun
# The network is shared by all three units; install it even when only the
# SearXNG service was selected. These copies are user-owned Quadlet state.
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user start searxng-network.service
systemctl --user restart gluetun.service searxng-valkey.service searxng.service
}
deploy_gluetun() {
install_quadlet gluetun dankclaw/searxng/quadlet gluetun
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart gluetun.service searxng.service
}
deploy_searxng_valkey() {
install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart searxng-valkey.service searxng.service
}
# Marreta's secrets are seeded into a per-service env file under the deploy
# user's home (rootless-readable), following the grafana convention rather
# than the /opt/searxng one, so the rootless unit can read it without any
# ownership gymnastics. Only the three MARRETA_* values are copied; the
# quadlet maps them to the app's APP_KEY / ADMIN_EMAIL / ADMIN_PASSWORD.
seed_marreta_env() {
local dest=/home/dank/Containers/marreta/env
mkdir -p /home/dank/Containers/marreta
if [[ ! -f "$dest" ]]; then
awk -F= '/^MARRETA_APP_KEY=/ || /^MARRETA_ADMIN_EMAIL=/ || /^MARRETA_ADMIN_PASSWORD=/ { print }' \
dankclaw/.env > "$dest"
chmod 600 "$dest"
fi
local var
for var in MARRETA_APP_KEY MARRETA_ADMIN_EMAIL MARRETA_ADMIN_PASSWORD; do
grep -q "^${var}=." "$dest" \
|| { echo "marreta env missing ${var}; set it in dankclaw/.env" >&2; return 1; }
done
}
deploy_marreta() {
seed_marreta_env
install_quadlet marreta dankclaw/marreta/quadlet marreta
install_quadlet marreta-browser dankclaw/marreta/quadlet marreta-browser
systemctl --user daemon-reload
# Both containers share gluetun's netns; the tunnel must be up first.
systemctl --user start gluetun.service
# Browser before app: marreta drives the CDP endpoint on loopback.
systemctl --user restart marreta-browser.service
systemctl --user restart marreta.service
}
deploy_open_webui() {
install_quadlet open-webui dankclaw/open-webui/quadlet open-webui
mkdir -p /home/dank/Containers/open-webui/data
systemctl --user daemon-reload
systemctl --user restart open-webui.service
}
# open-notebook is the fleet's podcast generator: one app container (API,
# worker and web UI under supervisord) in front of a SurrealDB container, with
# a private bridge between them. One handler owns the pair, because restarting
# the app without its database yields a UI that answers /health and fails every
# write; scripts/deploy-plan.py therefore maps every path of the pair here.
#
# Credentials follow the marreta convention (per-service file under the deploy
# user's home, mode 600, seeded from dankclaw/.env) with one difference: they
# are not optional. An empty OPEN_NOTEBOOK_PASSWORD is "authentication OFF",
# not "unconfigured" — the API, episode audio included, then answers anyone on
# the LAN — and an empty SURREAL_PASS lets the database come up with different
# root credentials than the app will present. So this handler refuses to start
# either unit until every key carries a value. An existing file is NEVER
# rewritten: regenerating an operator-set password would desync it from the
# credentials already stored in the database.
seed_open_notebook_env() {
local dest=/home/dank/Containers/open-notebook/env
local var missing=0
for var in OPEN_NOTEBOOK_ENCRYPTION_KEY OPEN_NOTEBOOK_PASSWORD SURREAL_PASSWORD SURREAL_PASS; do
grep -q "^${var}=." "$dest" 2>/dev/null || missing=1
done
if [[ "$missing" == 0 ]]; then
return 0
fi
if [[ ! -e "$dest" ]]; then
if [[ ! -f dankclaw/.env ]]; then
echo "open-notebook: neither $dest nor dankclaw/.env exists; put OPEN_NOTEBOOK_ENCRYPTION_KEY, OPEN_NOTEBOOK_PASSWORD and SURREAL_PASSWORD in dankclaw/.env (see dankclaw/.env.example)" >&2
return 1
fi
mkdir -p /home/dank/Containers/open-notebook
{
awk -F= '/^OPEN_NOTEBOOK_ENCRYPTION_KEY=/ || /^OPEN_NOTEBOOK_PASSWORD=/ || /^SURREAL_PASSWORD=/ { print }' \
dankclaw/.env
# The `surreal` CLI reads SURREAL_PASS from the environment while
# the app reads SURREAL_PASSWORD. One value, written twice, so the
# pair can never be seeded with two different root passwords.
sed -n 's/^SURREAL_PASSWORD=/SURREAL_PASS=/p' dankclaw/.env
} > "$dest"
chmod 600 "$dest"
fi
for var in OPEN_NOTEBOOK_ENCRYPTION_KEY OPEN_NOTEBOOK_PASSWORD SURREAL_PASSWORD SURREAL_PASS; do
grep -q "^${var}=." "$dest" \
|| { echo "open-notebook: $dest carries no value for ${var}; fill it in directly (or in dankclaw/.env before the first seed) — a blank value here means an unauthenticated service or a mismatched database password" >&2; return 1; }
done
}
# Both units ship Pull=never: the digest pins in the .container files are the
# contract, and the generator must never re-resolve a moving tag at start. That
# makes fetching a pin a precondition the deployer satisfies once per pin
# change; the existence check keeps every later re-deploy offline-safe.
ensure_pinned_image() {
local image="$1"
if podman image exists "$image"; then
echo "open-notebook: $image is already in local storage"
return 0
fi
echo "open-notebook: pulling pinned image $image"
podman pull "$image"
}
deploy_open_notebook() {
seed_open_notebook_env || return 1
ensure_pinned_image 'docker.io/surrealdb/surrealdb:v2@sha256:e5d7387e3727bbe2f7cd6d869e48cd34777272258572ead37c9656b25f745723' || return 1
ensure_pinned_image 'docker.io/lfnovo/open_notebook:v1-latest@sha256:54b2025a4a4853c188f74b9b8246eca8c5480e5dd1babfa92db1038ff2f30212' || return 1
# install_quadlet copies the sibling open-notebook.network with the app
# unit. That cp is load-bearing, not tidiness: when a referenced Network=
# file is absent the v1 generator refuses to produce the container unit at
# all ("requested Quadlet unit open-notebook.network was not found").
install_quadlet open-notebook-surrealdb dankclaw/open-notebook/quadlet open-notebook-surrealdb
install_quadlet open-notebook dankclaw/open-notebook/quadlet open-notebook
systemctl --user start open-notebook-network.service
# Interim episode destination (Gitea issue #7, `human-only`): finished
# episode audio is bind-mounted out to this host directory. Created here as
# the deploy user so the container's mapped root writes files dank can
# read, and so the dependency is explicit rather than podman's default.
mkdir -p /home/dank/media/podcasts
# No `systemctl --user enable`: generated Quadlet units cannot be enabled
# on this host (see deploy_authentik); boot persistence is the [Install]
# section inside each .container file. Database first, and both on every
# run — a stale app left attached to a recreated database is exactly the
# half-deployed state this handler exists to prevent.
systemctl --user restart open-notebook-surrealdb.service
wait_container_healthy open-notebook-surrealdb 90 || return 1
systemctl --user restart open-notebook.service
# The gate is the API's own health route: upstream serves `GET /health` on
# 5055 with {"status":"healthy"} and exempts it from the password
# middleware, so it stays a real answer once OPEN_NOTEBOOK_PASSWORD is set.
# Host port 5056, because seerr already holds 127.0.0.1:5055. The UI on
# 8502 is deliberately not gated: supervisord starts it only after the API
# answers (the image's wait-for-api.sh), so this gate already implies it.
wait_for_health open-notebook http://127.0.0.1:5056/health 180
}
# frank-tts-engine is the arm A TTS engine behind frank-tts: one Quadlet unit
# plus the private bridge its unit defines (install_quadlet copies the sibling
# frank-tts-engine.network — without that file the v1 generator refuses to
# produce the container unit at all, the same trap documented above). The
# façade that consumes it is a separate service on a separate port and a
# separate handler.
deploy_frank_tts_engine() {
# Weights BEFORE the build, and refuse rather than fetch. The payload is
# 1.4 GiB of GGUF with byte and sha256 checks; silently pulling it inside a
# deploy would turn a stalled network into a deploy that looks hung, and
# would hide which of the two failure modes the operator is in. The fetcher
# is idempotent, so re-running this handler after the fetch is cheap.
if ! scripts/download-frank-tts-engine-qwentts.sh --verify-only >/dev/null; then
echo "frank-tts-engine: the talker/tokenizer GGUFs are missing or wrong-sized under /home/dank/models/Qwen3-TTS-GGUF; run scripts/download-frank-tts-engine-qwentts.sh and re-deploy" >&2
return 1
fi
# Build first: Image= is a locally built tag with Pull=never, so the
# generator cannot start a container from it before it exists.
build_compose_image frank-tts-engine
install_quadlet frank-tts-engine dankclaw/voiceservice/quadlet frank-tts-engine
systemctl --user start frank-tts-engine-network.service
# No `systemctl --user enable` (generated Quadlet units cannot be enabled
# on this host); boot persistence is the [Install] section in the unit.
#
# `restart` blocks until the unit reaches started, which includes the
# ExecStartPost warm pass — a boot of this service is therefore already
# warm by the time the next line runs.
systemctl --user restart frank-tts-engine.service
# NOT wait_for_health: this container publishes nothing, so there is no URL
# the deployer can dial from the host. The unit's HealthCmd runs inside the
# container's own namespace, and podman's recorded health status is what the
# host can actually observe.
wait_container_healthy frank-tts-engine 150 || return 1
# The warmer runs a second time here on purpose. ExecStartPost carries a
# `-` prefix so a failed warm can never crash-loop the endpoint, which also
# means the unit cannot fail the deploy on it. This call is where a voice
# that did not warm becomes a deploy result, and where the per-voice
# cold=/warm= numbers land on the operator's console instead of only in the
# journal. The pass itself is idempotent and the second cold cost is gone:
# the voices are already warm, so this costs about a second per voice. It is
# exec-ed into the container for the same reason the health gate is: no host
# address exists.
podman exec frank-tts-engine /app/warm-voices.sh
}
deploy_frank_tts() {
# The 7772 façade in front of frank-tts-engine. Weights and the engine build
# are not this handler's business, but the BRIDGE FILE is: this unit
# references frank-tts-engine.network, and with a referenced Network= file
# absent the v1 generator refuses to produce the container unit at all. So
# `--only frank-tts` installs the engine's quadlet pair too — idempotent, and
# it is what makes the façade deployable on a host where the engine was
# planned separately or not at all.
#
# The voice packs are host state (the pack builder writes them), but the
# directory is created here as the deploy user on purpose: rootless podman
# would otherwise auto-create a missing bind source as root-owned, and the
# container runs as uid 10000.
mkdir -p /home/dank/voices
build_compose_image frank-tts
install_quadlet frank-tts-engine dankclaw/voiceservice/quadlet frank-tts-engine
install_quadlet frank-tts dankclaw/voiceservice/quadlet frank-tts
systemctl --user start frank-tts-engine-network.service
# No `systemctl --user enable` (generated Quadlet units cannot be enabled on
# this host); boot persistence is the [Install] section in each unit. The
# façade unit carries Wants=/After= on frank-tts-engine.service, so this
# restart pulls the engine — and its ExecStartPost warm pass — ahead of it:
# an engine that will not start delays this gate instead of taking the
# endpoint down with it.
systemctl --user restart frank-tts.service
# The gate is podman's recorded health, driven by the unit's own HealthCmd
# (`wget --spider -q http://127.0.0.1:7772/health`, inside the container's
# namespace). /health answers 200 with the backend name and the resolved
# voice list, so an empty /voices still reads healthy while a dead app does
# not — and note what this gate cannot see: the engine publishes no port on
# purpose, so an engine that is down shows up in the /health payload and in
# the first synthesis, never here.
wait_container_healthy frank-tts 120
}
deploy_comfyui() {
# Build first: the unit's Image= resolves against local storage, so the
# image has to exist before the generator starts a container from it.
build_compose_image comfyui
install_quadlet comfyui dankclaw/comfyui/quadlet comfyui
# Retire the compose-era unit. It owns the same unit name the Quadlet
# generator produces, and it is what kept a stale, device-less container
# alive through `restart: unless-stopped` (9,146 failures on
# "No HIP GPUs are available"). Never `rm`: it moves into the home trash.
local legacy="$HOME/.config/systemd/user/comfyui.service"
if [[ -e "$legacy" ]]; then
local trash="$HOME/.AGENTTRASH/20260916-comfyui-compose-unit"
mkdir -p "$trash"
systemctl --user stop comfyui.service 2>/dev/null || true
systemctl --user disable comfyui.service 2>/dev/null || true
mv -- "$legacy" "$trash/comfyui.service"
echo "retired the compose-era comfyui.service -> $trash"
fi
# The stale container carries the old config (no /dev/kfd, no /dev/dri) and
# would keep the quadlet's container from taking the name. It is replaced
# by the unit in the next step, so nothing is lost.
podman rm -f comfyui >/dev/null 2>&1 || true
systemctl --user daemon-reload
systemctl --user restart comfyui.service
wait_for_health comfyui http://127.0.0.1:8188/ 300
}
deploy_pihole() {
# Rootless port 53 needs net.ipv4.ip_unprivileged_port_start <= 53, and it
# must hold at boot. The repo drop-in is the single source of truth.
# systemd-sysctl applies /etc/sysctl.d/*.conf (lexicographic), then
# /run/sysctl.d, /usr/lib/sysctl.d, and finally /etc/sysctl.conf — later
# files override earlier ones. On 2026-08-26 a later-sorted drop-in (=80)
# AND a stale line in /etc/sysctl.conf (=80) both reset the value after a
# reboot and took LAN DNS down. Neutralize every conflicting source:
# stage sysctl.d files that set a value above 53 in the home trash, and
# comment out an active line in /etc/sysctl.conf (idempotent).
run_privileged install -m 0644 dankclaw/pihole/sysctl/99-dnsmasq.conf \
/etc/sysctl.d/99-dnsmasq.conf
local sysctl_file sysctl_val sysctl_trash
while IFS= read -r sysctl_file; do
[[ "$sysctl_file" == */99-dnsmasq.conf ]] && continue
sysctl_val="$(sed -nE 's/^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=[[:space:]]*([0-9]+).*/\1/p' "$sysctl_file" | head -1)"
if [[ "$sysctl_val" =~ ^[0-9]+$ ]] && (( sysctl_val > 53 )); then
sysctl_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-sysctl-unprivileged-port"
mkdir -p "$sysctl_trash"
run_privileged mv "$sysctl_file" "$sysctl_trash/"
fi
done < <(grep -lE '^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=' /etc/sysctl.d/*.conf 2>/dev/null || true)
if [[ -f /etc/sysctl.conf ]] && grep -qE '^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=' /etc/sysctl.conf; then
run_privileged sed -i -E 's/^[[:space:]]*(net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=)/# \1 managed by dankclaw\/pihole\/sysctl\/99-dnsmasq.conf/' /etc/sysctl.conf
fi
run_privileged sysctl --system >/dev/null
mkdir -p /home/dank/Containers/pihole/etc-pihole /home/dank/Containers/pihole/etc-dockers
if [[ ! -f /home/dank/Containers/pihole/env ]]; then
cp dankclaw/pihole/env.example /home/dank/Containers/pihole/env
echo "WARN: created Pi-hole env from the example; set WEBPASSWORD before using its UI" >&2
fi
cp dankclaw/pihole/addn-hosts /home/dank/Containers/pihole/addn-hosts
mkdir -p "$quadlet_unit_dir"
cp dankclaw/pihole/quadlet/pihole.container "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart pihole.service
}
deploy_gitea_unit() {
local service="$1"
local unit="$2"
mkdir -p "$quadlet_unit_dir"
cp "dankclaw/gitea/quadlet/$unit.container" "$quadlet_unit_dir/"
[[ -f dankclaw/gitea/quadlet/gitea.network ]] && cp dankclaw/gitea/quadlet/gitea.network "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart "$service.service"
}
relink_unit() {
local service="$1"
run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service"
}
# Authentik runs as three boot-persistent Quadlet units (postgres -> server ->
# worker). The trio deploys as one unit: unit files are installed, stale
# compose-era containers holding the claimed names are retired, then the chain
# starts in dependency order with health gates between stages. Re-running the
# handler on an already-active stack is a deliberate no-op (no DB bounce).
wait_container_healthy() {
local name="$1" limit="$2" status="unknown"
for ((i = 1; i <= limit; i += 5)); do
status="$(podman inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null || echo missing)"
[[ "$status" == "healthy" ]] && {
echo "$name healthy after approximately ${i}s"
return 0
}
sleep 5
done
echo "FAIL: $name did not become healthy in ${limit}s (status: $status)" >&2
return 1
}
# --- dankserver media/library stack ----------------------------------------------
# These services migrated from compose-era bare containers to boot-persistent
# Quadlet units installed from this repo (podman 5.x v1 generator honors the
# WantedBy=default.target inside each .container file). The first deploy of a
# service retires the old container and migrates the data root before starting
# the unit; re-running the handler on an already-active unit restarts only the
# selected unit.
migrate_data_root() { # $1=old /opt path, $2=new data root, $3=chown|keep (default chown)
local from="$1" to="$2" mode="${3:-chown}" entry sub base
[[ -d "$from" ]] || return 0
# Renaming a directory needs WRITE permission on the directory being
# renamed itself. Compose-era data roots contain directories owned by
# the containers' mapped host uids (mariadb/redis land in the subuid
# range as 100998), which the deploy user cannot rename. Privileged mv
# preserves numeric ownership — the rootless mapping contract.
mkdir -p "$to"
shopt -s dotglob
for entry in "$from"/*; do
[[ -e "$entry" || -L "$entry" ]] || continue
base="${entry##*/}"
if [[ -e "$to/$base" ]]; then
# Collides with pre-seeded state (env file) or a rescued
# anonymous-volume stub: merge one level deep, never overwriting
# an existing entry.
if [[ -d "$entry" && -d "$to/$base" ]]; then
for sub in "$entry"/*; do
[[ -e "$sub" || -L "$sub" ]] || continue
run_privileged mv -n -- "$sub" "$to/$base/"
done
run_privileged rmdir -- "$entry" 2>/dev/null || \
echo "WARN: $entry not empty after merge; left in place" >&2
else
echo "WARN: collision on $to/$base kept existing; $entry left in place" >&2
fi
continue
fi
run_privileged mv -- "$entry" "$to/"
done
shopt -u dotglob
run_privileged rmdir -- "$from" 2>/dev/null || \
echo "WARN: $from not empty after migration; left in place" >&2
if [[ "$mode" == chown ]]; then
chown -R "$(id -u):$(id -g)" "$to"
fi
}
deploy_romm_stack() {
local service="$1" unit vol tmp
case "$service" in
mariadb) service=romm-mariadb ;;
redis) service=romm-redis ;;
esac
for unit in romm-mariadb romm-redis romm; do
cp "dankserver/romm/quadlet/$unit.container" "$quadlet_unit_dir/"
done
systemctl --user daemon-reload
if ! systemctl --user is-active --quiet romm-mariadb.service; then
# First cutover: the env file must exist before the first start, or
# the units would boot with empty secrets. Fail before stopping anything.
[[ -f /home/dank/Containers/romm/env ]] || {
echo "FAIL: /home/dank/Containers/romm/env does not exist; seed it before the first romm deploy" >&2
return 1
}
# Capture anonymous volume names while the containers are still inspectable,
# then stop and rm the compose-era trio.
local vols
vols="$(podman inspect -f '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}' romm 2>/dev/null || true)"
podman stop -t 60 romm-mariadb romm-redis romm || true
podman rm romm romm-redis romm-mariadb || true
for vol in $vols; do
tmp="$(mktemp -d)"
podman volume export --output "$tmp/v.tar" "$vol"
# --skip-old-files keeps live bind-mounted content (config/, library/,
# resources/, mariadb/, redis/) and rescues only unshadowed entries like assets/.
tar --skip-old-files -C /home/dank/Containers/romm -xf "$tmp/v.tar"
rm -rf -- "$tmp"
done
migrate_data_root /opt/romm /home/dank/Containers/romm keep
systemctl --user start romm-mariadb.service
wait_container_healthy romm-mariadb 120 || return 1
systemctl --user start romm-redis.service
wait_container_healthy romm-redis 60 || return 1
if [[ "$service" == "romm" ]]; then
systemctl --user start romm.service
fi
return 0
fi
systemctl --user restart "$service.service"
}
deploy_jellyfin() {
cp "dankserver/jellyfin/quadlet/jellyfin.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q jellyfin.service; then
podman stop -t 30 jellyfin || true
podman rm jellyfin || true
migrate_data_root /opt/jellyfin /home/dank/Containers/jellyfin
fi
systemctl --user restart jellyfin.service
}
deploy_calibre_web() {
cp "dankserver/calibre-web/quadlet/calibre-web.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q calibre-web.service; then
# The compose-era container ran under a podman-generate-systemd unit; stop
# that unit first so its Restart=always cannot resurrect the old container
# and collide with the new ContainerName.
systemctl --user stop container-calibre-web.service 2>/dev/null || true
podman stop calibre-web || true
podman rm calibre-web || true
fi
systemctl --user restart calibre-web.service
}
# The content server's single admin account and the desktop's basic-auth
# credential (same pair) are bootstrapped exactly once: the env file is the
# plaintext store (mode 600), the userdb's pw column is plaintext too, so
# its file is mode 600 as well. The password is piped on stdin with printf -
# a trailing newline from echo lands in the stored password and breaks every
# login (verified 2026-09-21).
deploy_calibre_server() {
local data_root=/home/dank/Containers/calibre-server
mkdir -p "$data_root"
if [[ ! -f $data_root/env ]]; then
local pw
pw=$(openssl rand -base64 18 | tr -d '/+=' | head -c 20)
printf 'CUSTOM_USER=calibre\nCUSTOM_PASSWORD=%s\n' "$pw" > "$data_root/env"
chmod 600 "$data_root/env"
echo "calibre-server: generated credential (shown once): calibre / $pw"
fi
[[ -f $data_root/env ]] || {
echo "FAIL: $data_root/env does not exist; seed it before the first calibre-server deploy" >&2
return 1
}
if [[ ! -f $data_root/userdb.sqlite ]]; then
local pw
pw=$(grep '^CUSTOM_PASSWORD=' "$data_root/env" | cut -d= -f2)
touch "$data_root/userdb.sqlite"
printf '%s' "$pw" | podman run --rm -i --entrypoint /usr/bin/calibre-server \
-v "$data_root/userdb.sqlite:/userdb.sqlite" \
lscr.io/linuxserver/calibre:9.15.0 \
--userdb /userdb.sqlite --manage-users -- add calibre
chown 1000:1000 "$data_root/userdb.sqlite"
chmod 600 "$data_root/userdb.sqlite"
fi
[[ -f $data_root/userdb.sqlite && -s $data_root/userdb.sqlite ]] || {
echo "FAIL: $data_root/userdb.sqlite missing or empty; the --manage-users bootstrap failed" >&2
return 1
}
cp "dankserver/calibre-server/quadlet/calibre-server.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q calibre-server.service; then
podman stop calibre-server 2>/dev/null || true
podman rm calibre-server 2>/dev/null || true
fi
systemctl --user restart calibre-server.service
}
deploy_copyparty() {
# The admin account is created from ${COPAY_ACCOUNT}:${COPAY_PASSWORD} expanded
# at ExecStart time; a missing env file must fail the unit, not boot copyparty
# with a garbage admin password.
[[ -f /home/dank/Containers/copyparty/env ]] || {
echo "FAIL: /home/dank/Containers/copyparty/env does not exist; seed it before the first copyparty deploy" >&2
return 1
}
cp "dankserver/copyparty/quadlet/copyparty.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q copyparty.service; then
podman stop -t 15 copyparty || true
podman rm copyparty || true
migrate_data_root /opt/copyparty /home/dank/Containers/copyparty
fi
systemctl --user restart copyparty.service
}
deploy_plex() {
cp "dankserver/plex/quadlet/plex.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q plex.service; then
# First cutover: retire the native package unit BEFORE starting the
# container — both cannot own 32400. Mask (not disable): on these hosts
# `systemctl disable` deletes the unit file itself; mask keeps a
# one-command rollback while nothing is signed off yet.
run_privileged systemctl stop plexmediaserver.service
run_privileged systemctl mask plexmediaserver.service
# Rootless identity model: container root maps to the dank user
# (uid 1000) — real uid 997 is not mappable rootless. Adopt the data
# root into the fleet convention (jellyfin et al. run as 1000:1000).
# Rollback chowns it back to 997:984 before restarting the native unit.
run_privileged chown -R 1000:1000 /var/lib/plexmediaserver
fi
systemctl --user restart plex.service
# Readiness gate: /identity answers MediaContainer XML once PMS is up.
#
# HTTPS, NOT HTTP. PMS drops plain HTTP off-loopback: the probe's
# connection arrives on the container's eth0, so an http:// request is
# accepted and then answered with nothing (curl 52, Empty reply) against a
# perfectly healthy server. Measured 2026-10-02 — the gate burned its full
# 600s and failed a plex whose https://127.0.0.1:32400/identity answered
# 200 in 44 ms the whole time. The same law is why the Caddyfile dials the
# plex upstream over TLS (tests/caddyfile.bats pins it).
# -k: PMS presents its own self-signed cert on 32400.
#
# Reality check from the 2026-09 cutover: the FIRST container boot of a
# years-old library ran ~10 min, so the gate still allows 600s.
# Fail the deploy (pending state preserved) rather than mark it deployed blind.
for ((i = 1; i <= 120; i++)); do
curl -fsSk --max-time 3 https://127.0.0.1:32400/identity >/dev/null && return 0
sleep 5
done
echo "FAIL: plex did not answer /identity over HTTPS within 600s" >&2
return 1
}
deploy_tautulli() {
cp "dankserver/tautulli/quadlet/tautulli.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active -q tautulli.service; then
run_privileged snap stop tautulli || true
run_privileged snap disable tautulli || true
mkdir -p /home/dank/Containers/tautulli/config
if [[ ! -f /home/dank/Containers/tautulli/config/config.ini ]]; then
printf '[General]\nPMS_Web_Host = plex\nPMS_Web_Port = 32400\nPMS_Secure = 0\n' \
> /home/dank/Containers/tautulli/config/config.ini
fi
fi
systemctl --user restart tautulli.service
}
deploy_authentik() {
local unit
mkdir -p "$quadlet_unit_dir"
for unit in authentik-postgres authentik-server authentik-worker; do
cp "dankclaw/authentik/quadlet/$unit.container" "$quadlet_unit_dir/"
done
systemctl --user daemon-reload
# Retire compose-era containers only while the units are not yet active;
# unit-managed containers self-heal via Restart=always.
if ! systemctl --user is-active --quiet authentik-postgres.service; then
podman stop authentik-postgres authentik-server authentik-worker 2>/dev/null || true
podman rm authentik-postgres authentik-server authentik-worker 2>/dev/null || true
fi
# No `systemctl --user enable` here: this host's quadlet generator
# (podman 5.x v1) generates units into /run/user/1000/systemd/generator/,
# and generated units cannot be enabled — enable fails with "transient or
# generated". Boot persistence comes from [Install] WantedBy=default.target
# inside each .container file, which the generator honors at user-manager
# start (linger must stay on for the deploy user).
systemctl --user start authentik-postgres.service
wait_container_healthy authentik-postgres 120 || return 1
systemctl --user start authentik-server.service
wait_container_healthy authentik-server 300 || return 1
systemctl --user start authentik-worker.service
}
deploy_authentik_outpost() {
# The outpost is the worker-role binary of the same server image; it dials
# back to core over websocket and serves /outpost.goauthentik.io/* for
# Caddy's forward_auth. Image already in storage — zero pulls. Restarting
# only this unit cycles auth for guarded vhosts for a few seconds; the
# rest of the stack is untouched.
cp dankclaw/authentik/quadlet/authentik-outpost.container "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart authentik-outpost.service
sleep 5
systemctl --user is-active --quiet authentik-outpost.service || \
echo "WARN: authentik-outpost did not come up active" >&2
}
deploy_authentik_outpost_ldap() {
# Dedicated LDAP outpost for Jellyfin's LDAP-Auth plugin. The type-specific
# ghcr.io/goauthentik/ldap image serves 3389/6636 container-side, published
# as host 389/636 for cross-host binds from dankserver. The outpost token
# lives in the host .env, never in this repo. Restarting only this unit
# cycles LDAP auth for a few seconds; the rest of the stack is untouched.
cp dankclaw/authentik/quadlet/authentik-outpost-ldap.container "$quadlet_unit_dir/"
systemctl --user daemon-reload
systemctl --user restart authentik-outpost-ldap.service
sleep 5
systemctl --user is-active --quiet authentik-outpost-ldap.service || \
echo "WARN: authentik-outpost-ldap did not come up active" >&2
}
deploy_hermes() {
# The hermes trio (gateway, dashboard, webui) shares one image and the
# /data volume and deploys as one unit: install the units, retire the
# compose-era containers holding the claimed names while the units are not
# yet active, then restart each unit. An explicit hermes deploy is a
# deliberate action, so restarting an already-running trio is intended.
# Boot persistence comes from [Install] in each unit file (generated units
# cannot be enabled by this host's v1 quadlet generator).
local unit
mkdir -p "$quadlet_unit_dir"
# The CDP browser daemon is a companion service to the trio: install its
# unit and start it if inactive, but never restart it on deploy — a
# restart would kill in-flight browser sessions. Consumers (gateway,
# webui) reconnect with backoff.
cp "dankclaw/hermes/quadlet/chrome-devtools-protocol.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
if ! systemctl --user is-active --quiet chrome-devtools-protocol.service; then
systemctl --user start chrome-devtools-protocol.service
fi
# The units run the image under a localhost/-aliased tag: this host's
# quadlet path can fail to resolve registry-prefixed refs from local
# storage at create time. Tag first, then (re)create via deploy.
podman tag docker.io/nousresearch/hermes-agent:latest localhost/hermes-agent:latest 2>/dev/null || true
for unit in hermes hermes-dashboard hermes-webui; do
cp "dankclaw/hermes/quadlet/$unit.container" "$quadlet_unit_dir/"
done
# The generator must see the freshly-copied unit files before the
# restarts below; without this reload the first deploy after a unit edit
# restarts with the stale generated ExecStart (statfs ENOENT on renamed
# Volume= sources).
systemctl --user daemon-reload
if ! systemctl --user is-active --quiet hermes.service; then
podman stop hermes hermes-dashboard hermes-webui 2>/dev/null || true
podman rm hermes hermes-dashboard hermes-webui 2>/dev/null || true
fi
systemctl --user restart hermes.service
systemctl --user restart hermes-dashboard.service
systemctl --user restart hermes-webui.service
}
deploy_anubis() {
# Anubis is the bot-challenge proxy in front of the guarded vhosts.
# The image lives on ghcr.io; pull once, then the unit runs it from
# local storage (Pull=never keeps boots off the registry).
if ! podman image exists ghcr.io/techarohq/anubis:v1.27.0; then
podman pull ghcr.io/techarohq/anubis:v1.27.0
fi
# The unit runs the localhost/ alias (see the unit file comment): this
# host's podman quadlet path cannot resolve registry-prefixed refs from
# local storage at create time. Tag first, then (re)create via deploy.
podman tag ghcr.io/techarohq/anubis:v1.27.0 localhost/techarohq/anubis:v1.27.0 2>/dev/null || true
cp "dankclaw/anubis/quadlet/anubis.container" "$quadlet_unit_dir/"
systemctl --user daemon-reload
# No enable: generated units cannot be enabled on this host (v1 quadlet
# generator); boot persistence comes from [Install] in the unit file.
# restart, not start: policy.json is parsed at process start only (no
# reload signal exists), so every redeploy must cycle the process.
systemctl --user restart anubis.service
wait_container_healthy anubis 60 || return 1
}
# frank-dragoon is the halogen-flash-server CONTAINER (0.16.2, digest-pinned
# through scripts/frank-dragoon.env) serving Quat3rnion's NATIVE v2
# conversion (2026-10-11) of the abliterated Qwen3.8-Flash-Next —
# qwen38-flash-next-v2-abliterated.hgn (66,687,678,432 B, ~62 GiB resident)
# + the REQUIRED n-gram lookup table + the vision tower — MTP spec decode
# (HALOGEN_SPEC_ADAPT=0), 2 slots over ONE 401,408-position KV pool (2 × ctx,
# raised 2026-10-18 so BOTH slots can hold a maximum-length request) at the
# ctx 200704 ceiling, image input — on the Strix Halo iGPU. It stays a
# ROOT unit because the serve script drives `podman run` directly (the frank-archon pattern):
# relink_unit points /etc/systemd/system/frank-dragoon.service at the repo
# file, boot persistence is that unit's [Install], and LimitMEMLOCK=infinity
# is fleet parity. 2026-10-17 cutover from the Swift1.5-finetune shape —
# rollback = revert the swap commit and redeploy; the swift15 payload
# (~/models/halogen-flashnext-v2) and the 27B one both stay on disk.
#
# The gates mirror serve-frank-dragoon.sh's predicates EXACTLY (the
# 2026-10-01 crash-loop lesson): image digest in dank's podman storage,
# each .hgn payload (checkpoint, n-gram table, vision tower) at the exact
# byte count, the tokenizer directory holding every named file as a
# regular (not symlinked) file — a green deploy against a missing
# artifact would leave a restart-looping unit and a dark endpoint. The
# port gate is the same as every cutover since 2026-09-16: nothing
# starts until 7770 is released.
LLM_DRAGOON_IMAGE="${LLM_DRAGOON_IMAGE:-ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a}"
LLM_DRAGOON_CKPT="${LLM_DRAGOON_CKPT:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-v2-abliterated.hgn}"
LLM_DRAGOON_CKPT_BYTES="${LLM_DRAGOON_CKPT_BYTES:-66687678432}"
LLM_DRAGOON_NGRAM="${LLM_DRAGOON_NGRAM:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-ngram.hgn}"
LLM_DRAGOON_NGRAM_BYTES="${LLM_DRAGOON_NGRAM_BYTES:-51200246144}"
LLM_DRAGOON_VISION="${LLM_DRAGOON_VISION:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-vision.hgn}"
LLM_DRAGOON_VISION_BYTES="${LLM_DRAGOON_VISION_BYTES:-897916416}"
LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-v2-abliterated/tokenizer}"
LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}"
# NPU lane (2026-10-19). HALOGEN_NPU_MODELS is the whole switch and the parity
# twin of the env file's line: empty here AND in the env = the pre-lane
# endpoint. The gates below mirror serve-frank-dragoon.sh, and the vendor's
# fabric-clock host unit (dankclaw/host/, verbatim at tag v0.16.2) is installed
# ONLY while the lane is named — on Strix Halo the iGPU and the NPU co-running
# hang the machine while the fabric clock moves, so halogen refuses to boot with
# the lane armed unless that clock is held.
LLM_DRAGOON_NPU_MODELS="${LLM_DRAGOON_NPU_MODELS:-qwen3-embedding-0.6b}"
LLM_DRAGOON_NPU_DIR="${LLM_DRAGOON_NPU_DIR:-/home/dank/models/halogen-npu}"
LLM_DRAGOON_DEV_ACCEL="${LLM_DRAGOON_DEV_ACCEL:-/dev/accel/accel0}"
LLM_DRAGOON_XRT_DIR="${LLM_DRAGOON_XRT_DIR:-/usr/lib/x86_64-linux-gnu}"
LLM_DRAGOON_NETWORK="${LLM_DRAGOON_NETWORK:-dankclaw_default}"
retire_halogen_dragoon_quadlet() {
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
systemctl --user stop frank-dragoon.service 2>/dev/null || true
if [[ -e "$quadlet_unit_dir/frank-dragoon.container" ]]; then
mkdir -p "$trash"
mv -- "$quadlet_unit_dir/frank-dragoon.container" "$trash/frank-dragoon.container"
echo "retired quadlet frank-dragoon.container -> $trash"
fi
systemctl --user daemon-reload
}
# Wait until a TCP port has no socket on it at all -- listening, accepted, or
# lingering in TIME_WAIT. Rootless podman publishes through pasta, which binds
# the host port at container CREATE, and a stopped LLM runtime keeps its
# listener (and the TIME_WAIT residue of the connections it served) for a
# moment after the process exits. `sport` is the port in question on both the
# listener and the server side of every accepted connection.
wait_port_free() {
local port="$1" limit="${2:-90}" i
for ((i = 1; i <= limit; i++)); do
if [[ -z "$(ss -Htan "sport = :$port" 2>/dev/null)" ]]; then
return 0
fi
sleep 1
done
echo "FAIL: port $port is still held after ${limit}s:" >&2
ss -tan "sport = :$port" >&2 || true
return 1
}
deploy_llm_dragoon() {
# Refuse rather than restart-loop; every gate below has a twin in
# serve-frank-dragoon.sh and they must agree (2026-10-01 lesson).
# Capture podman output before grepping: `podman images | grep -q`
# races under pipefail (SIGPIPE 141 becomes "no match", 2026-09-30).
if ! grep -qF "${LLM_DRAGOON_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then
echo "ERROR: halogen image absent: $LLM_DRAGOON_IMAGE" >&2
echo "frank-dragoon: pull it: podman pull ${LLM_DRAGOON_IMAGE%@*}" >&2
return 1
fi
if [[ ! -f "$LLM_DRAGOON_CKPT" ]]; then
echo "ERROR: halogen checkpoint absent at $LLM_DRAGOON_CKPT" >&2
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-v2abl.sh" >&2
return 1
fi
if [[ "$(stat -c %s "$LLM_DRAGOON_CKPT")" != "$LLM_DRAGOON_CKPT_BYTES" ]]; then
echo "ERROR: halogen checkpoint is not $LLM_DRAGOON_CKPT_BYTES bytes (partial download?)" >&2
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-v2abl.sh --verify-only" >&2
return 1
fi
for gate in "$LLM_DRAGOON_NGRAM:$LLM_DRAGOON_NGRAM_BYTES:n-gram table" \
"$LLM_DRAGOON_VISION:$LLM_DRAGOON_VISION_BYTES:vision tower"; do
IFS=: read -r gpath gbytes gname <<<"$gate"
if [[ ! -f "$gpath" ]]; then
echo "ERROR: halogen $gname absent at $gpath" >&2
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-v2abl.sh" >&2
return 1
fi
if [[ "$(stat -c %s "$gpath")" != "$gbytes" ]]; then
echo "ERROR: halogen $gname is not $gbytes bytes (partial download?)" >&2
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-v2abl.sh --verify-only" >&2
return 1
fi
done
if [[ -L "$LLM_DRAGOON_TOK_DIR" || ! -d "$LLM_DRAGOON_TOK_DIR" ]]; then
echo "ERROR: tokenizer dir missing or a symlink at $LLM_DRAGOON_TOK_DIR" >&2
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-v2abl.sh" >&2
return 1
fi
for tf in $LLM_DRAGOON_TOK_FILES; do
if [[ ! -f "$LLM_DRAGOON_TOK_DIR/$tf" || -L "$LLM_DRAGOON_TOK_DIR/$tf" ]]; then
echo "ERROR: tokenizer file missing or a symlink: $LLM_DRAGOON_TOK_DIR/$tf" >&2
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-v2abl.sh" >&2
return 1
fi
done
# The container network has to exist before the run line names it (a typo
# there is a restart-looping unit, not a config error anyone reads).
if [[ -n "$LLM_DRAGOON_NETWORK" ]] && ! podman network exists "$LLM_DRAGOON_NETWORK"; then
echo "ERROR: podman network $LLM_DRAGOON_NETWORK does not exist" >&2
echo "frank-dragoon: create it or blank FRANK_DRAGOON_NETWORK (pasta default)" >&2
return 1
fi
# --- NPU lane: the same three predicates the serve script checks ---------
if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then
if [[ ! -e "$LLM_DRAGOON_DEV_ACCEL" ]]; then
echo "ERROR: NPU node $LLM_DRAGOON_DEV_ACCEL absent — amdxdna is not bound (is iommu=pt still on the boot line?)" >&2
return 1
fi
for xl in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
if [[ ! -e "$LLM_DRAGOON_XRT_DIR/$xl" ]]; then
echo "ERROR: XRT NPU library $LLM_DRAGOON_XRT_DIR/$xl absent (Ubuntu libxrt2 + libxrt-npu2)" >&2
return 1
fi
done
for nm in ${LLM_DRAGOON_NPU_MODELS//,/ }; do
if [[ ! -d "$LLM_DRAGOON_NPU_DIR/$nm" ]]; then
echo "ERROR: NPU model $nm not staged in $LLM_DRAGOON_NPU_DIR" >&2
echo "frank-dragoon: stage it: bash scripts/download-frank-dragoon-npu.sh" >&2
return 1
fi
done
# The hold must exist BEFORE the engine starts and survive every boot
# (a resume hands the clock back too — the unit's WantedBy covers it).
run_privileged install -m 0755 dankclaw/host/halogen-fabric-clock \
/usr/local/sbin/halogen-fabric-clock
run_privileged install -m 0644 dankclaw/host/halogen-fabric-clock.service \
/etc/systemd/system/halogen-fabric-clock.service
run_privileged systemctl daemon-reload
run_privileged systemctl enable --now halogen-fabric-clock.service
if ! run_privileged /usr/local/sbin/halogen-fabric-clock status; then
echo "ERROR: fabric clock is not held — halogen refuses to start with HALOGEN_NPU_MODELS set" >&2
echo "frank-dragoon: journalctl -u halogen-fabric-clock.service -n 40" >&2
return 1
fi
elif [[ -x /usr/local/sbin/halogen-fabric-clock ]]; then
# Lane rolled back: hand the clock back to the driver and drop the boot
# hook. `disable` deletes the INSTALLED copy of this unit — unlike the
# tracked LLM unit symlinks the fleet's disable law protects, this file
# is repo-owned and the next lane-on deploy reinstalls it.
run_privileged systemctl disable halogen-fabric-clock.service 2>/dev/null || true
run_privileged /usr/local/sbin/halogen-fabric-clock release || true
fi
retire_halogen_dragoon_quadlet
# The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
# the container) before waiting on the port. The stray sweep is keyed
# on the FULL container name, which appears in the argv of a manual
# `podman run --name frank-dragoon-halogen` / `podman wait` but NOT in
# this deployer's own command line (`bash deploy-remote.sh
# frank-dragoon`) — so it cannot shoot itself in the foot (the fleet
# pkill law), and it can never match frank-zealot's llama-server.
run_privileged systemctl stop frank-dragoon.service 2>/dev/null || true
pkill -f 'frank-dragoon-halogen' 2>/dev/null || true
podman rm -f frank-dragoon frank-dragoon-halogen >/dev/null 2>&1 || true
# Nothing may start until the predecessor has released the port
# (2026-09-16: "Deactivated successfully" was not "port free").
wait_port_free 7770 || return 1
relink_unit frank-dragoon
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-dragoon.service
run_privileged systemctl restart frank-dragoon.service
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
# The lane's own gate. /health proves the engine is up, nothing about the
# NPU programs; one real embedding is the observable output. The first
# request after a boot is a warmup artifact (fleet law), so retry before
# calling a booted lane dead.
if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then
for nm in ${LLM_DRAGOON_NPU_MODELS//,/ }; do
lane_ok=0
for _try in 1 2 3 4 5; do
if curl -fsS -m 60 -H 'content-type: application/json' \
-d "{\"model\":\"$nm\",\"input\":[\"frank-dragoon npu lane gate\"]}" \
http://127.0.0.1:7770/v1/embeddings 2>/dev/null \
| python3 -c 'import json,sys
v = json.load(sys.stdin)["data"][0]["embedding"]
raise SystemExit(0 if isinstance(v, list) and len(v) and all(isinstance(x, (int, float)) for x in v) else 1)'; then
lane_ok=1
break
fi
sleep 10
done
if [[ "$lane_ok" != 1 ]]; then
echo "ERROR: NPU model $nm answered no vector on /v1/embeddings" >&2
echo "frank-dragoon: read the boot ledger: tail -100 /home/dank/logs/frank_dragoon.log" >&2
return 1
fi
echo "frank-dragoon: NPU lane verified — $nm embedded"
done
fi
}
# deploy_llm_zealot — frank-zealot is llama.cpp (PrismML fork build) on the
# RTX 3090 as a ROOT unit: the runtime is a native binary tree under
# ~/GitPackages/llama-prism-b10777 (vendor release prism-b10777-e3de715,
# `--version` = 0.2.0-dev build 10777 commit e3de71522 — the ONLY loader class
# that registers the payload's custom tensor type 142 and reads its
# prism.hadamard.* metadata, and since 2026-10-18 ALSO the minimum build for
# this endpoint: the card gates quantized V alongside MTP on
# `prism-b10776-d00e252` or newer, which the 27B-era 10754 tree predates), so
# there is no container image and no generated unit to avoid. relink_unit
# points /etc/systemd/system/frank-zealot.service at the repo file and boot
# persistence is that unit's [Install] section. The former 10754 tree
# (~/GitPackages/llama-prism) stays on disk as the rollback runtime.
#
# It is a SEPARATE endpoint on port 7771; frank-dragoon keeps 7770. The 3090 is
# not shared: frank-dragoon's engine runs on the gfx1151 iGPU.
LLM_ZEALOT_MODEL_FILE="${LLM_ZEALOT_MODEL_FILE:-/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.gguf}"
LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-prism-b10777/llama-server}"
# The projector IS the vision capability here: the Bonsai-2 text file carries
# no vision tensors, so the publisher's own mmproj (Qwen3-VL tower, 27 blocks,
# 768 px, qwen3vl_merger, projection_dim 4096, 624,229,664 B) is the whole
# image path — a deploy that starts without it silently ships a text-only
# endpoint. Checked unconditionally, like frank-dragoon's. Q8_0 is MANDATORY,
# not a preference: the card says the BF16 tower can crash llama-server when
# several image requests run at once, and this endpoint runs --parallel 3. Its
# 868.32 MiB worst-case VRAM (595.29 model + 248.10 compute in the boot ledger)
# lands AFTER the engine's fit check, so it is not inside the q8_0 budget.
LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf}"
# The refusal adapter is part of the payload identity now, so the deployer
# checks it beside the other two files — but it has its OWN fetcher, so a
# missing adapter must not send the operator to the bonsai9 script. Integrity
# (bytes + sha256) is the fetcher's job at download time; this gate only asks
# whether the file the env names exists, exactly like the two above.
LLM_ZEALOT_LORA="${LLM_ZEALOT_LORA:-/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-Uncensored-LoRA.gguf}"
deploy_llm_zealot() {
# Refuse rather than restart-loop: a unit whose runtime or payload is
# missing never becomes healthy, and a green deploy would leave the
# endpoint dark with nothing in the log to say why.
if [[ ! -x "$LLM_ZEALOT_RUNTIME" ]]; then
echo "ERROR: llama.cpp runtime absent at $LLM_ZEALOT_RUNTIME" >&2
echo "frank-zealot: fetch the PrismML fork release build (prism-b10777-e3de715," >&2
echo "frank-zealot: linux-cuda-12.4 asset) into ~/GitPackages/llama-prism-b10777 first:" >&2
echo "frank-zealot: stock llama.cpp refuses the payload's custom tensor type 142, and" >&2
echo "frank-zealot: builds older than b10776 cannot pair MTP with a quantized V cache." >&2
return 1
fi
for f in "$LLM_ZEALOT_MODEL_FILE" "$LLM_ZEALOT_MMPROJ"; do
if [[ ! -f "$f" ]]; then
echo "ERROR: frank-zealot payload file absent at $f" >&2
echo "frank-zealot: run scripts/download-frank-zealot-bonsai9.sh first;" >&2
echo "frank-zealot: refusing to start a unit that cannot load." >&2
return 1
fi
done
if [[ ! -f "$LLM_ZEALOT_LORA" ]]; then
echo "ERROR: frank-zealot refusal adapter absent at $LLM_ZEALOT_LORA" >&2
echo "frank-zealot: run scripts/download-frank-zealot-uncensored-lora.sh first;" >&2
echo "frank-zealot: the env names the adapter, so the unit would refuse to boot" >&2
echo "frank-zealot: without it — starting the unit anyway is a crash loop." >&2
return 1
fi
relink_unit frank-zealot
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-zealot.service
run_privileged systemctl restart frank-zealot.service
wait_for_health frank-zealot http://127.0.0.1:7771/health 900 || return 1
# False-green hardening (frank-dragoon's lesson): the port answering
# proves nothing about WHO answers — the unit must be active with a
# live MainPID, or a stray SGLang holdover is masquerading as deployed.
local main_pid
main_pid="$(run_privileged systemctl show -p MainPID --value frank-zealot.service)"
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
echo "FAIL: :7771 answered but frank-zealot.service has no MainPID — a stray process holds the port" >&2
return 1
fi
echo "frank-zealot: health answered by unit MainPID $main_pid"
}
# deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode,
# digest-pinned stilldeadcode/radiance image) serving the PUBLISHER's
# 4-bit-codebook container of the official Qwen/Qwen3.8-Flash-Next
# (qwen3.8-next-flash-fp8-iq4r-moe.rad, fetched by
# scripts/download-frank-archon-iq4r.sh) on BOTH R9700s (TP=2) as a
# ROOT unit whose serve script drives `podman run` directly (the engine is
# the image entrypoint; the image lives in dank's rootless podman storage).
# 2026-10-15 payload swap from the in-house w4dn8 (payload on disk =
# rollback).
# 2026-10-05 cutover from FreeToken (pre-cutover tag
# frank-archon-freetoken-pr587-20261005); before that, the tcclaviger vLLM
# fork (tag frank-archon-vllm-shapeG-20261003). Same port and alias; the
# harness models.yml id is unchanged.
#
# Gates: the image must exist under its digest ref in dank's rootless
# storage (FRANK_ARCHON_IMAGE), the .rad container file must exist at the
# env's size floor (FRANK_ARCHON_MODEL_FILE >= FRANK_ARCHON_RAD_MIN_MIB —
# a truncated fetch is a guaranteed long-timeout boot), and the unit env
# must be complete. Per-file sha256 verification is the fetch script's job
# at download time and rad-convert/rad-info's job at build time, never
# re-hashed here.
LLM_ARCHON_CONTAINER=frank-archon
LLM_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-probe"
deploy_llm_archon() {
# Refuse rather than restart-loop: a unit whose image or payload is
# missing never becomes healthy, and a green deploy would leave the
# endpoint dark with nothing in the log to say why.
local env_file="/home/dank/serverAppConfigs/scripts/frank-archon.env"
local image model_dir model_file rad_min mib
if [[ ! -f "$env_file" ]]; then
echo "ERROR: frank-archon env file absent: $env_file" >&2
return 1
fi
image="$(sed -n 's/^FRANK_ARCHON_IMAGE=//p' "$env_file" | tail -1)"
model_dir="$(sed -n 's/^FRANK_ARCHON_MODEL_DIR=//p' "$env_file" | tail -1)"
model_file="$(sed -n 's/^FRANK_ARCHON_MODEL_FILE=//p' "$env_file" | tail -1)"
rad_min="$(sed -n 's/^FRANK_ARCHON_RAD_MIN_MIB=//p' "$env_file" | tail -1)"
if [[ -z "$image" || -z "$model_dir" || -z "$model_file" || -z "$rad_min" ]]; then
echo "ERROR: frank-archon.env missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
return 1
fi
# The image gate is the digest ref itself: a name@sha256 ref exists in
# storage only if THAT manifest was pulled. One command — no
# pipefail/grep race (the 2026-09-30 lesson).
if ! podman image exists "$image"; then
echo "ERROR: runtime image absent from storage: $image" >&2
echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest" >&2
return 1
fi
if [[ ! -f "$model_dir/$model_file" ]]; then
echo "ERROR: .rad container missing: $model_dir/$model_file" >&2
echo "frank-archon: run scripts/download-frank-archon-iq4r.sh --fetch" >&2
return 1
fi
mib=$(( $(stat -c %s "$model_dir/$model_file") / 1048576 ))
if [[ "$mib" -lt "$rad_min" ]]; then
echo "ERROR: .rad container too small: $model_dir/$model_file is ${mib} MiB, want >= $rad_min" >&2
echo "frank-archon: re-fetch it (scripts/download-frank-archon-iq4r.sh --fetch; delete the staged file first if its sha256 failed)" >&2
return 1
fi
# A stale container with the same name makes `podman run --name` fail at
# start; remove any leftover from a failed previous attempt.
# A stopped unit can leave its container running: systemd's cgroup kill
# races conmon's --rm cleanup, and the orphan holds the port and GPUs.
# Stop the unit, then remove the containers BEFORE waiting on the port.
# The legacy identities (the retired R9V container, the ad hoc
# experiment container qwen-fp8, the retired glm-probe) hold the same
# GPUs and die here too.
run_privileged systemctl stop frank-archon.service 2>/dev/null || true
local ctr
for ctr in $LLM_ARCHON_LEGACY_CONTAINERS "$LLM_ARCHON_CONTAINER"; do
podman rm -f "$ctr" >/dev/null 2>&1 || true
done
# Nothing starts until the port is released: the lucebox cutover
# (2026-09-26) and the frank-dragoon cutover (2026-09-16) both failed
# exactly here when the next engine bound 7770 a second too early.
wait_port_free 7770 || return 1
relink_unit frank-archon
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-archon.service
run_privileged systemctl restart frank-archon.service
# Cold start: .rad load + PLE pin + expert host-pool upload (the
# publisher's compose healthcheck allows a 15-minute start_period).
# Parity with every predecessor on this port. The HTTP listener binds
# AFTER the full load (main.cpp: engine.start() then engine.serve()),
# so during load the port REFUSES connections — a bare curl is honest
# here, unlike FreeToken's 200-while-loading body. (The MainPID check
# below still guards the stray-port case.)
local i ok=0
for ((i=1; i<=900; i++)); do
if curl -fsS http://127.0.0.1:7770/health 2>/dev/null >/dev/null; then
echo "frank-archon engine healthy after approximately $((i * 2))s"
ok=1
break
fi
sleep 2
done
if [[ "$ok" != 1 ]]; then
echo "FAIL: frank-archon /health never answered 200 in 1800s" >&2
return 1
fi
# False-green hardening (the 2026-09-26 lesson): a bare curl against the
# port cannot tell WHO answers — a stray probe holding :7770 satisfies
# it while the unit is stopped. The deploy is only green if the unit
# itself is active with a live MainPID behind the health answer.
local main_pid
main_pid="$(run_privileged systemctl show -p MainPID --value frank-archon.service)"
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
echo "FAIL: :7770 answered but frank-archon.service has no MainPID — a stray process holds the port" >&2
return 1
fi
echo "frank-archon: health answered by unit MainPID $main_pid"
}
wait_for_health() {
local service="$1" url="$2" limit="$3"
local i
for ((i=1; i<=limit; i++)); do
if curl -fsS "$url" >/dev/null 2>&1; then
echo "$service healthy after approximately $((i * 2))s"
return 0
fi
sleep 2
done
echo "FAIL: $service did not become healthy in $((limit * 2))s" >&2
return 1
}
# ---------------------------------------------------------------------------
# FrankBook-Pro (macOS): quadlet units live INSIDE the podman machine VM
# (/home/core/.config/containers/systemd), not in the macOS user session.
# This handler pipes unit files and seed logic into the VM over
# `podman machine ssh`; it never touches systemctl on the Mac host and never
# uses run_privileged (the Mac account is not passwordless sudo). Unit files
# are re-piped on every call (idempotent, cheap) before the selected unit is
# restarted; first boot also seeds the qBittorrent config once.
FRANKBOOK_SVCS=(gluetun qbittorrent prowlarr sonarr radarr lidarr readarr)
deploy_frankbook_stack() {
local service="$1"
export PATH="/usr/local/bin:/opt/homebrew/bin:$PATH"
if ! podman machine inspect --format '{{.State}}' 2>/dev/null | grep -qx running; then
echo "FAIL: podman machine is not running on frankbook-pro" >&2
return 1
fi
local vm_dir="/home/core/.config/containers/systemd" svc
podman machine ssh "mkdir -p '$vm_dir' /home/core/Containers/gluetun /home/core/Containers/qbittorrent"
for svc in "${FRANKBOOK_SVCS[@]}"; do
podman machine ssh "cat > '$vm_dir/$svc.container'" \
< "frankbook-pro/$svc/quadlet/$svc.container"
done
local tmp
tmp="$(mktemp /tmp/frankbook-vm.XXXXXX)"
cat > "$tmp" <<'VM_SEED'
set -euo pipefail
# qBittorrent first-boot seed: conf + categories + machine-local PBKDF2
# password. qB >= 4.2 ignores plaintext passwords, so the conf carries the
# @ByteArray(salt:hash) form (PBKDF2-HMAC-SHA512, 100k iterations, 64 bytes).
if [ ! -e /home/core/Containers/qbittorrent/qbit-seeded ]; then
mkdir -p /Volumes/COMPLETE/qbittorrent-data/qBittorrent
QBIT_PW="$(openssl rand -base64 18 | tr -d '/+=' | head -c 20)"
QBIT_HASH="$(python3 - "$QBIT_PW" <<'PYH'
import hashlib, base64, os, sys
pw = sys.argv[1].encode()
salt = os.urandom(16)
h = hashlib.pbkdf2_hmac("sha512", pw, salt, 100000, dklen=64)
print("@ByteArray(%s:%s)" % (base64.b64encode(salt).decode(), base64.b64encode(h).decode()))
PYH
)"
umask 077
printf 'QBIT_WEBUI_PASSWORD=%s\n' "$QBIT_PW" > /home/core/Containers/qbittorrent/env
umask 022
cat > /Volumes/COMPLETE/qbittorrent-data/qBittorrent/qBittorrent.conf <<QCONF
[BitTorrent]
Session\\DefaultSavePath=/Volumes/COMPLETE
Session\\Port=6881
Session\\QueueingSystemEnabled=true
Session\\TempPath=/Volumes/INCOMPLETE/incomplete
Session\\TempPathEnabled=true
[LegalNotice]
Accepted=true
[Meta]
MigrationVersion=8
[Network]
Cookies\\Enabled=true
[Preferences]
WebUI\\Address=*
WebUI\\ServerDomains=*
WebUI\\HostHeaderValidation=false
WebUI\\Username=admin
WebUI\\Password_PBKDF2=$QBIT_HASH
QCONF
cat > /Volumes/COMPLETE/qbittorrent-data/qBittorrent/categories.json <<'QCAT'
{
"Finished Torrents": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
"lidarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
"radarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
"tv-sonarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"}
}
QCAT
touch /home/core/Containers/qbittorrent/qbit-seeded
echo " seeded qBittorrent config (WebUI password: /home/core/Containers/qbittorrent/env inside the VM)"
fi
VM_SEED
podman machine ssh bash -s < "$tmp"
rm -f "$tmp"
# Credential gate: without a WireGuard env the tunnel stays parked, and
# qBittorrent (which downloads through it) waits with it. The arrs run
# without the tunnel, so only these two services are gated.
case "$service" in
gluetun|qbittorrent)
if ! podman machine ssh '[ -s /home/core/Containers/gluetun/env ]' >/dev/null 2>&1; then
echo "SKIP: $service waits for gluetun credentials."
echo " Write the WireGuard env into /home/core/Containers/gluetun/env inside the VM"
echo " (podman machine ssh), then re-run: ./scripts/deploy.sh frankbook-pro --service gluetun --service qbittorrent"
return 0
fi
;;
esac
tmp="$(mktemp /tmp/frankbook-vm.XXXXXX)"
cat > "$tmp" <<'VM_START'
set -euo pipefail
SVC="${1:?service required}"
systemctl --user daemon-reload
unit="$(systemctl --user list-units --all --no-legend "${SVC}.service" "container-${SVC}.service" 2>/dev/null | sed 's/^● //' | head -n1 | awk '{print $1}')"
[ -n "$unit" ] || { echo "FAIL: no generated systemd unit for $SVC" >&2; exit 1; }
systemctl --user restart "$unit"
sleep 3
state="$(systemctl --user is-active "$unit" || true)"
if [ "$state" != active ] && [ "$state" != activating ]; then
echo "FAIL: $unit is $state" >&2
systemctl --user status "$unit" --no-pager -l || true
exit 1
fi
VM_START
podman machine ssh bash -s "$service" < "$tmp"
rm -f "$tmp"
local port path
case "$service" in
qbittorrent) port=8080; path=/ ;;
prowlarr) port=9696; path=/ping ;;
sonarr) port=8989; path=/ping ;;
radarr) port=7878; path=/ping ;;
lidarr) port=8686; path=/ping ;;
readarr) port=8787; path=/ping ;;
*) return 0 ;; # gluetun publishes only qBittorrent's ports
esac
printf ' waiting for http://127.0.0.1:%s%s on the Mac...\n' "$port" "$path"
local ok=0 i
for i in $(seq 1 60); do
if curl -fsS -m 4 "http://127.0.0.1:${port}${path}" >/dev/null 2>&1; then
ok=1
break
fi
sleep 5
done
[[ "$ok" -eq 1 ]] || { echo "FAIL: $service did not answer on 127.0.0.1:$port" >&2; return 1; }
echo " $service answers on 127.0.0.1:$port"
}
deploy_seerr() {
install_quadlet seerr dankclaw/seerr/quadlet seerr
systemctl --user restart seerr.service
}
deploy_service() {
local service="$1"
if [[ "$TARGET" == "frankbook-pro" ]]; then
# All frankbook-pro services route through the Mac handler: the name
# case below addresses the Linux hosts, and service names overlap.
deploy_frankbook_stack "$service"
return
fi
case "$service" in
caddy) deploy_caddy ;;
homepage) deploy_homepage ;;
searxng) deploy_searxng ;;
gluetun) deploy_gluetun ;;
searxng-valkey) deploy_searxng_valkey ;;
open-webui) deploy_open_webui ;;
open-notebook) deploy_open_notebook ;;
pihole) deploy_pihole ;;
gitea-postgres) deploy_gitea_unit gitea-postgres gitea-postgres ;;
gitea-server) deploy_gitea_unit gitea-server gitea-server ;;
authentik-postgres|authentik-server|authentik-worker)
deploy_authentik
;;
authentik-outpost)
deploy_authentik_outpost
;;
authentik-outpost-ldap)
deploy_authentik_outpost_ldap
;;
anubis) deploy_anubis ;;
seerr) deploy_seerr ;;
marreta) deploy_marreta ;;
frank-dragoon) deploy_llm_dragoon ;;
frank-zealot) deploy_llm_zealot ;;
frank-archon) deploy_llm_archon ;;
frank-tts-engine) deploy_frank_tts_engine ;;
frank-tts) deploy_frank_tts ;;
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
jellyfin) deploy_jellyfin ;;
calibre-web) deploy_calibre_web ;;
calibre-server) deploy_calibre_server ;;
copyparty) deploy_copyparty ;;
plex) deploy_plex ;;
tautulli) deploy_tautulli ;;
prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
comfyui) deploy_comfyui ;;
*) deploy_compose_service "$service" ;;
esac
}
if [[ -n "$FROM_REVISION" ]]; then
# Register every changed service before applying the requested scope. This
# preserves an explicit --service selection as pending work for the next
# ordinary deployment.
register_changed_services "$FROM_REVISION"
fi
if [[ "$PENDING" -eq 1 ]]; then
add_changed_services "$FROM_REVISION"
add_pending_services
fi
# Remove duplicate service names while preserving caller/planner order.
UNIQUE_SERVICES=()
declare -A SEEN_SERVICES=()
for service in "${SERVICES[@]}"; do
[[ -n "$service" ]] || continue
if [[ -z "${SEEN_SERVICES[$service]:-}" ]]; then
UNIQUE_SERVICES+=("$service")
SEEN_SERVICES[$service]=1
fi
done
SERVICES=("${UNIQUE_SERVICES[@]}")
if [[ "${#SERVICES[@]}" -eq 0 ]]; then
echo "No services selected; nothing to deploy."
exit 0
fi
for service in "${SERVICES[@]}"; do
[[ "$service" =~ ^[a-z0-9][a-z0-9-]*$ ]] || { echo "invalid service: $service" >&2; exit 2; }
service_is_allowed "$service" || { echo "service $service is not deployed on $TARGET" >&2; exit 2; }
echo "=== Deploying $service at $REVISION ==="
mark_status "$service" pending
deploy_service "$service"
mark_deployed "$service"
echo " STATE[$service] = $REVISION (status=deployed)"
done
Reference in New Issue View Git Blame Copy Permalink