530 lines
19 KiB
Bash
Executable File
530 lines
19 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# deploy-remote.sh — apply an explicit service plan on one target host.
|
|
#
|
|
# This script runs on the target checkout. It deliberately does not infer a
|
|
# plan, pull Git, or restart an unrelated service. deploy.sh computes the plan
|
|
# from the pushed revision and passes the selected service names here.
|
|
# Per-service revisions live in /var/lib/serverAppConfigs/deploy-state (or
|
|
# DEPLOY_STATE_DIR), outside the checkout, so an unselected change remains
|
|
# pending until its service is explicitly deployed.
|
|
set -euo pipefail
|
|
|
|
TARGET="${1:-}"
|
|
REVISION="${2:-}"
|
|
shift 2 || true
|
|
FROM_REVISION=""
|
|
PENDING=0
|
|
SERVICES=()
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: deploy-remote.sh <dankclaw|dankserver> <git-revision> [--pending] [service]...
|
|
|
|
The service list is explicit. An empty list is a successful no-op.
|
|
EOF
|
|
}
|
|
|
|
while [[ "$#" -gt 0 ]]; do
|
|
case "$1" in
|
|
--from-revision)
|
|
[[ "$#" -ge 2 ]] || { echo "--from-revision requires a revision" >&2; exit 2; }
|
|
FROM_REVISION="$2"
|
|
shift 2
|
|
;;
|
|
--pending)
|
|
PENDING=1
|
|
shift
|
|
;;
|
|
--help|-h)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
SERVICES+=("$1")
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
REMOTE_DIR="${REMOTE_DIR:-/home/dank/serverAppConfigs}"
|
|
STATE_DIR="${DEPLOY_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/serverAppConfigs/deploy-state}"
|
|
|
|
run_privileged() {
|
|
if [[ -n "${FRANK_SUDO_PW:-}" ]]; then
|
|
printf '%s\n' "$FRANK_SUDO_PW" | sudo -S -- "$@"
|
|
else
|
|
sudo -- "$@"
|
|
fi
|
|
}
|
|
|
|
if [[ "$TARGET" != dankclaw && "$TARGET" != dankserver ]]; then
|
|
usage >&2
|
|
exit 2
|
|
fi
|
|
if [[ -z "$REVISION" ]]; then
|
|
usage >&2
|
|
exit 2
|
|
fi
|
|
if [[ "$REVISION" == *[!0-9a-fA-F]* ]]; then
|
|
echo "invalid Git revision: $REVISION" >&2
|
|
exit 2
|
|
fi
|
|
if ! git -C "$REMOTE_DIR" cat-file -e "$REVISION^{commit}" 2>/dev/null; then
|
|
echo "revision is not a commit in the target checkout: $REVISION" >&2
|
|
exit 2
|
|
fi
|
|
cd "$REMOTE_DIR"
|
|
|
|
# Keep the state directory outside Git. The file is written only after a
|
|
# handler completes successfully; an interrupted handler remains pending.
|
|
mkdir -p "$STATE_DIR"
|
|
|
|
service_is_allowed() {
|
|
local service="$1"
|
|
case "$TARGET:$service" in
|
|
dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\
|
|
dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
|
|
dankclaw:gitea-caddy|dankclaw:comfyui|dankclaw:hermes|\
|
|
dankclaw:hermes-dashboard|dankclaw:prometheus|dankclaw:grafana|\
|
|
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|\
|
|
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
|
dankclaw:authentik-worker|dankclaw:caddy|dankclaw:homepage|\
|
|
dankclaw:frank-default-large|dankclaw:frank-default-npu|\
|
|
dankserver:mariadb|dankserver:redis|dankserver:romm|\
|
|
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|\
|
|
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
|
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
|
|
dankserver:frank-default-small)
|
|
return 0
|
|
;;
|
|
*)
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
mark_status() {
|
|
local service="$1" status="$2" tmp
|
|
tmp="$(mktemp "$STATE_DIR/.${service}.XXXXXX")"
|
|
printf 'revision=%s\nstatus=%s\n' "$REVISION" "$status" > "$tmp"
|
|
mv -f -- "$tmp" "$STATE_DIR/$service"
|
|
}
|
|
|
|
mark_deployed() {
|
|
mark_status "$1" deployed
|
|
}
|
|
|
|
state_revision() {
|
|
local service="$1" line value
|
|
if [[ -r "$STATE_DIR/$service" ]]; then
|
|
while IFS= read -r line; do
|
|
case "$line" in
|
|
revision=*) value="${line#revision=}" ;;
|
|
esac
|
|
done < "$STATE_DIR/$service"
|
|
printf '%s' "${value:-}"
|
|
fi
|
|
}
|
|
|
|
state_status() {
|
|
local service="$1" line value=pending
|
|
if [[ -r "$STATE_DIR/$service" ]]; then
|
|
while IFS= read -r line; do
|
|
case "$line" in
|
|
status=*) value="${line#status=}" ;;
|
|
esac
|
|
done < "$STATE_DIR/$service"
|
|
fi
|
|
printf '%s' "$value"
|
|
}
|
|
|
|
register_changed_services() {
|
|
local base="$1" service recorded status
|
|
[[ -n "$base" ]] || return 0
|
|
while IFS= read -r service; do
|
|
[[ -n "$service" ]] || continue
|
|
recorded="$(state_revision "$service")"
|
|
status="$(state_status "$service")"
|
|
if [[ "$status" != deployed || "$recorded" != "$REVISION" ]]; then
|
|
mark_status "$service" pending
|
|
fi
|
|
done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
|
|
--base "$base" --head "$REVISION")
|
|
}
|
|
|
|
add_changed_services() {
|
|
local base="$1"
|
|
local service
|
|
[[ -n "$base" ]] || return 0
|
|
while IFS= read -r service; do
|
|
[[ -n "$service" ]] || continue
|
|
SERVICES+=("$service")
|
|
done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
|
|
--base "$base" --head "$REVISION")
|
|
}
|
|
|
|
add_pending_services() {
|
|
local service recorded status changed
|
|
for state_file in "$STATE_DIR"/*; do
|
|
[[ -f "$state_file" ]] || continue
|
|
service="${state_file##*/}"
|
|
recorded="$(state_revision "$service")"
|
|
status="$(state_status "$service")"
|
|
if [[ "$status" != deployed ]]; then
|
|
# A pending service was intentionally skipped or its handler
|
|
# failed. Retry it even when the checkout has not advanced.
|
|
SERVICES+=("$service")
|
|
continue
|
|
fi
|
|
[[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
|
|
if [[ "$TARGET" == dankclaw && "$service" == frank-default-small ]]; then continue; fi
|
|
if [[ "$TARGET" == dankserver && "$service" == frank-default-large ]]; then continue; fi
|
|
if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
|
|
--repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
|
|
echo "WARN: unable to compare state for $service; retaining it as pending" >&2
|
|
SERVICES+=("$service")
|
|
continue
|
|
fi
|
|
if grep -Fxq "$service" <<< "$changed"; then
|
|
SERVICES+=("$service")
|
|
fi
|
|
done
|
|
}
|
|
|
|
copy_searxng_config() {
|
|
run_privileged mkdir -p /opt/searxng/config
|
|
run_privileged cp shared/searxng/settings.yml /opt/searxng/config/settings.yml
|
|
run_privileged cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml
|
|
}
|
|
|
|
copy_prometheus_config() {
|
|
run_privileged mkdir -p /opt/prometheus/config
|
|
run_privileged cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml
|
|
}
|
|
|
|
copy_loki_config() {
|
|
run_privileged mkdir -p /opt/loki/config
|
|
run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
|
|
}
|
|
|
|
copy_alloy_config() {
|
|
run_privileged mkdir -p /opt/alloy/config
|
|
run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
|
|
}
|
|
|
|
render_caddyfile() {
|
|
local hash
|
|
hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
|
|
if [[ -z "$hash" ]]; then
|
|
echo "WARN: CADDY_BASICAUTH_HASH is not set; refusing to overwrite rendered Caddyfile" >&2
|
|
return 1
|
|
fi
|
|
run_privileged mkdir -p /home/dank/Containers/caddy
|
|
sed -e "s|REPLACE_WITH_HASH|$hash|g" dankclaw/Caddyfile > /tmp/serverAppConfigs.Caddyfile
|
|
run_privileged cp /tmp/serverAppConfigs.Caddyfile /home/dank/Containers/caddy/Caddyfile
|
|
}
|
|
|
|
deploy_homepage() {
|
|
if [[ "$TARGET" != dankclaw ]]; then
|
|
echo "homepage is owned by dankclaw; refusing to deploy it on $TARGET" >&2
|
|
return 2
|
|
fi
|
|
run_privileged mkdir -p /srv/homepage
|
|
run_privileged cp shared/homepage/index.html shared/homepage/admin.html shared/homepage/app.js \
|
|
shared/homepage/style.css shared/homepage/services.json /srv/homepage/
|
|
if [[ -d shared/homepage/fonts ]]; then
|
|
run_privileged cp -r shared/homepage/fonts /srv/homepage/
|
|
fi
|
|
local deployed_at
|
|
deployed_at="$(git log -1 --format=%cI)"
|
|
printf 'window.DEPLOYED_AT = "%s";\n' "$deployed_at" > /tmp/deployed.js
|
|
run_privileged cp /tmp/deployed.js /srv/homepage/deployed.js
|
|
run_privileged chmod -R a+rX /srv/homepage
|
|
run_privileged cp scripts/homepage-status.py /usr/local/bin/homepage-status.py
|
|
run_privileged cp scripts/homepage-status.service scripts/homepage-status.timer /etc/systemd/system/
|
|
run_privileged systemctl daemon-reload
|
|
run_privileged systemctl enable homepage-status.timer
|
|
run_privileged systemctl restart homepage-status.timer
|
|
run_privileged python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json
|
|
}
|
|
|
|
deploy_caddy() {
|
|
if [[ "$TARGET" != dankclaw ]]; then
|
|
echo "caddy is owned by dankclaw; refusing to deploy it on $TARGET" >&2
|
|
return 2
|
|
fi
|
|
render_caddyfile
|
|
deploy_homepage
|
|
local cfhash oldhash image
|
|
cfhash="$(sha256sum dankclaw/caddy/Containerfile | cut -d' ' -f1)"
|
|
image=franksdank/caddy-porkbun
|
|
oldhash="$(podman image inspect "$image:latest" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
|
|
if [[ "$oldhash" != "$cfhash" ]]; then
|
|
podman build --no-cache --label "cfhash=$cfhash" -t "$image:latest" \
|
|
-f dankclaw/caddy/Containerfile .
|
|
fi
|
|
# The canonical reverse proxy may be gitea-caddy on a host with the old
|
|
# split deployment. Never stop or replace it implicitly. Refuse the Caddy
|
|
# deployment until the operator has explicitly migrated port ownership.
|
|
if podman container exists gitea-caddy && ! podman container exists caddy; then
|
|
echo "FAIL: gitea-caddy owns the reverse-proxy deployment; refusing to replace it with caddy." >&2
|
|
return 1
|
|
fi
|
|
if podman container exists gitea-caddy && podman container exists caddy; then
|
|
local gitea_state caddy_state
|
|
gitea_state="$(podman inspect gitea-caddy --format '{{.State.Status}}' 2>/dev/null || true)"
|
|
caddy_state="$(podman inspect caddy --format '{{.State.Status}}' 2>/dev/null || true)"
|
|
if [[ "$gitea_state" == running && "$caddy_state" != running ]]; then
|
|
echo "FAIL: gitea-caddy currently owns ports 80/443; refusing to start caddy." >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
if podman container exists caddy; then
|
|
podman-compose -f dankclaw/compose.yaml up -d --no-deps caddy
|
|
local reloaded=0
|
|
for _ in 1 2 3 4 5; do
|
|
if podman exec caddy caddy reload --config /etc/caddy/Caddyfile \
|
|
--adapter caddyfile >/dev/null 2>&1; then
|
|
reloaded=1
|
|
echo "Caddy reloaded."
|
|
break
|
|
fi
|
|
sleep 3
|
|
done
|
|
if [[ "$reloaded" -ne 1 ]]; then
|
|
echo "Caddy reload failed; restarting only caddy."
|
|
podman restart caddy
|
|
fi
|
|
else
|
|
echo "WARN: caddy container is absent; no proxy was stopped or replaced."
|
|
fi
|
|
}
|
|
|
|
deploy_compose_service() {
|
|
local service="$1"
|
|
if [[ "$TARGET" == dankserver ]]; then
|
|
# Dankserver's compose file is the source-of-truth inventory, but its
|
|
# production containers are individually systemd-managed/bare. Do not
|
|
# use compose up here: it can create a second container with a wrong
|
|
# name and can interfere with a running workload.
|
|
echo "WARN: $service is recorded in dankserver/compose.yaml; production container management is host-specific and no restart was attempted."
|
|
return 0
|
|
fi
|
|
case "$service" in
|
|
comfyui|hermes|hermes-dashboard|prometheus|grafana|loki|alloy|node-exporter|authentik-postgres|authentik-server|authentik-worker)
|
|
case "$service" in
|
|
comfyui|loki|alloy)
|
|
build_compose_image "$service"
|
|
;;
|
|
prometheus)
|
|
copy_prometheus_config
|
|
;;
|
|
loki)
|
|
copy_loki_config
|
|
;;
|
|
alloy)
|
|
copy_alloy_config
|
|
;;
|
|
esac
|
|
podman-compose -f dankclaw/compose.yaml up -d --no-deps "$service"
|
|
;;
|
|
*)
|
|
echo "unknown compose service: $service" >&2
|
|
return 2
|
|
;;
|
|
esac
|
|
}
|
|
|
|
build_compose_image() {
|
|
local service="$1" image cfhash oldhash containerfile
|
|
case "$service" in
|
|
comfyui)
|
|
image=comfyui-rocm:gfx1151
|
|
containerfile=dankclaw/Containerfile.comfyui
|
|
;;
|
|
loki|alloy)
|
|
image="franksdank/$service:latest"
|
|
containerfile="dankclaw/$service/Containerfile"
|
|
;;
|
|
*)
|
|
return 0
|
|
;;
|
|
esac
|
|
cfhash="$(sha256sum "$containerfile" | cut -d' ' -f1)"
|
|
oldhash="$(podman image inspect "$image" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
|
|
if [[ "$oldhash" != "$cfhash" ]]; then
|
|
podman build --no-cache --label "cfhash=$cfhash" -t "$image" -f "$containerfile" .
|
|
fi
|
|
}
|
|
|
|
quadlet_unit_dir="$HOME/.config/containers/systemd"
|
|
install_quadlet() {
|
|
local service="$1"
|
|
local source_dir="$2"
|
|
local unit="$3"
|
|
mkdir -p "$quadlet_unit_dir"
|
|
cp "$source_dir/$unit.container" "$quadlet_unit_dir/"
|
|
if [[ -f "$source_dir/$unit.network" ]]; then
|
|
cp "$source_dir/$unit.network" "$quadlet_unit_dir/"
|
|
fi
|
|
systemctl --user daemon-reload
|
|
}
|
|
|
|
deploy_searxng() {
|
|
copy_searxng_config
|
|
install_quadlet searxng dankclaw/searxng/quadlet searxng
|
|
install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
|
|
install_quadlet gluetun dankclaw/searxng/quadlet gluetun
|
|
# The network is shared by all three units; install it even when only the
|
|
# SearXNG service was selected. These copies are user-owned Quadlet state.
|
|
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user start searxng-network.service
|
|
systemctl --user restart gluetun.service searxng-valkey.service searxng.service
|
|
}
|
|
|
|
deploy_gluetun() {
|
|
install_quadlet gluetun dankclaw/searxng/quadlet gluetun
|
|
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart gluetun.service searxng.service
|
|
}
|
|
|
|
deploy_searxng_valkey() {
|
|
install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
|
|
cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart searxng-valkey.service searxng.service
|
|
}
|
|
|
|
deploy_open_webui() {
|
|
install_quadlet open-webui dankclaw/open-webui/quadlet open-webui
|
|
mkdir -p /home/dank/Containers/open-webui/data
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart open-webui.service
|
|
}
|
|
|
|
deploy_pihole() {
|
|
mkdir -p /home/dank/Containers/pihole/etc-pihole /home/dank/Containers/pihole/etc-dockers
|
|
if [[ ! -f /home/dank/Containers/pihole/env ]]; then
|
|
cp dankclaw/pihole/env.example /home/dank/Containers/pihole/env
|
|
echo "WARN: created Pi-hole env from the example; set WEBPASSWORD before using its UI" >&2
|
|
fi
|
|
cp dankclaw/pihole/addn-hosts /home/dank/Containers/pihole/addn-hosts
|
|
mkdir -p "$quadlet_unit_dir"
|
|
cp dankclaw/pihole/quadlet/pihole.container "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart pihole.service
|
|
}
|
|
|
|
deploy_gitea_unit() {
|
|
local service="$1"
|
|
local unit="$2"
|
|
mkdir -p "$quadlet_unit_dir"
|
|
cp "dankclaw/gitea/quadlet/$unit.container" "$quadlet_unit_dir/"
|
|
[[ -f dankclaw/gitea/quadlet/gitea.network ]] && cp dankclaw/gitea/quadlet/gitea.network "$quadlet_unit_dir/"
|
|
systemctl --user daemon-reload
|
|
systemctl --user restart "$service.service"
|
|
}
|
|
|
|
relink_unit() {
|
|
local service="$1"
|
|
run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service"
|
|
}
|
|
|
|
deploy_llm_large() {
|
|
relink_unit frank-default-large
|
|
# NPU is intentionally not restarted here. It is a separate, retired
|
|
# endpoint and its unit is only relinked when explicitly selected.
|
|
run_privileged systemctl daemon-reload
|
|
run_privileged systemctl enable frank-default-large
|
|
run_privileged systemctl restart frank-default-large
|
|
wait_for_health frank-default-large http://127.0.0.1:7770/health 1200
|
|
}
|
|
|
|
deploy_llm_small() {
|
|
relink_unit frank-default-small
|
|
run_privileged systemctl daemon-reload
|
|
run_privileged systemctl enable frank-default-small
|
|
run_privileged systemctl restart frank-default-small
|
|
wait_for_health frank-default-small http://127.0.0.1:7770/health 1200
|
|
}
|
|
|
|
deploy_llm_npu() {
|
|
relink_unit frank-default-npu
|
|
run_privileged systemctl daemon-reload
|
|
echo "Relinked frank-default-npu; it was not restarted."
|
|
}
|
|
|
|
wait_for_health() {
|
|
local service="$1" url="$2" limit="$3"
|
|
local i
|
|
for ((i=1; i<=limit; i++)); do
|
|
if curl -fsS "$url" >/dev/null 2>&1; then
|
|
echo "$service healthy after approximately $((i * 2))s"
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "FAIL: $service did not become healthy in $((limit * 2))s" >&2
|
|
return 1
|
|
}
|
|
|
|
deploy_service() {
|
|
local service="$1"
|
|
case "$service" in
|
|
caddy) deploy_caddy ;;
|
|
homepage) deploy_homepage ;;
|
|
searxng) deploy_searxng ;;
|
|
gluetun) deploy_gluetun ;;
|
|
searxng-valkey) deploy_searxng_valkey ;;
|
|
open-webui) deploy_open_webui ;;
|
|
pihole) deploy_pihole ;;
|
|
gitea-postgres) deploy_gitea_unit gitea-postgres gitea-postgres ;;
|
|
gitea-server) deploy_gitea_unit gitea-server gitea-server ;;
|
|
gitea-caddy) deploy_gitea_unit gitea-caddy gitea-caddy ;;
|
|
frank-default-large) deploy_llm_large ;;
|
|
frank-default-small) deploy_llm_small ;;
|
|
frank-default-npu) deploy_llm_npu ;;
|
|
*) deploy_compose_service "$service" ;;
|
|
esac
|
|
}
|
|
|
|
if [[ -n "$FROM_REVISION" ]]; then
|
|
# Register every changed service before applying the requested scope. This
|
|
# preserves an explicit --service selection as pending work for the next
|
|
# ordinary deployment.
|
|
register_changed_services "$FROM_REVISION"
|
|
fi
|
|
if [[ "$PENDING" -eq 1 ]]; then
|
|
add_changed_services "$FROM_REVISION"
|
|
add_pending_services
|
|
fi
|
|
|
|
# Remove duplicate service names while preserving caller/planner order.
|
|
UNIQUE_SERVICES=()
|
|
declare -A SEEN_SERVICES=()
|
|
for service in "${SERVICES[@]}"; do
|
|
[[ -n "$service" ]] || continue
|
|
if [[ -z "${SEEN_SERVICES[$service]:-}" ]]; then
|
|
UNIQUE_SERVICES+=("$service")
|
|
SEEN_SERVICES[$service]=1
|
|
fi
|
|
done
|
|
SERVICES=("${UNIQUE_SERVICES[@]}")
|
|
|
|
if [[ "${#SERVICES[@]}" -eq 0 ]]; then
|
|
echo "No services selected; nothing to deploy."
|
|
exit 0
|
|
fi
|
|
|
|
for service in "${SERVICES[@]}"; do
|
|
[[ "$service" =~ ^[a-z0-9][a-z0-9-]*$ ]] || { echo "invalid service: $service" >&2; exit 2; }
|
|
service_is_allowed "$service" || { echo "service $service is not deployed on $TARGET" >&2; exit 2; }
|
|
echo "=== Deploying $service at $REVISION ==="
|
|
mark_status "$service" pending
|
|
deploy_service "$service"
|
|
mark_deployed "$service"
|
|
echo " STATE[$service] = $REVISION (status=deployed)"
|
|
done
|