303 lines
12 KiB
Bash
Executable File
303 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# deploy.sh — Deploy the serverAppConfigs repo to a target and restart its
|
|
# podman stack and LLM fleet services.
|
|
#
|
|
# Implements the git deployment workflow from AGENTS.md. Assumes local
|
|
# changes are committed and pushed, then:
|
|
# 1. Pushes main to origin
|
|
# 2. Pulls the repo on the target
|
|
# 3. Copies shared configs into place and restarts the podman stack
|
|
# (dankclaw only; the dankserver media stack runs as bare containers)
|
|
# 4. Relinks the LLM systemd units and restarts the active LLM service
|
|
# 5. Waits for the Caddy homepage and the LLM health endpoint
|
|
#
|
|
# Sudo: the fleet hosts allow passwordless sudo for the deploy user. For
|
|
# headless runs on a host without that, set FRANK_SUDO_PW so remote sudo
|
|
# reads the password from stdin (`sudo -S`). The password is never stored
|
|
# in this repo.
|
|
#
|
|
# Usage:
|
|
# ./scripts/deploy.sh <target> [--dry-run]
|
|
#
|
|
# Targets:
|
|
# dankclaw - AI/search/dev/observability stack + frank-default-large
|
|
# (CIRU ROCmFPX llama.cpp, Qwen3.8-27B, port 7770)
|
|
# dankserver - media/library stack (bare containers) + frank-default-small
|
|
# (upstream llama.cpp, Qwen3.6-35B-A3B, port 7770)
|
|
#
|
|
# Example:
|
|
# ./scripts/deploy.sh dankclaw
|
|
# ./scripts/deploy.sh dankserver --dry-run
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Target configuration
|
|
# ---------------------------------------------------------------------------
|
|
declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR COMPOSE HEALTH_URL COMPOSE_UP COPY_SHARED
|
|
declare -A LLM_SERVICES LLM_UNITS LLM_HEALTH LLM_WAIT
|
|
|
|
# Caddy is consolidated on dankclaw: it fronts both hosts' services. The
|
|
# dankserver media stack runs as bare containers (not this repo's compose),
|
|
# so its stack deploy is pull-only; health is checked against the shared Caddy.
|
|
REMOTE_USER[dankclaw]=dank
|
|
REMOTE_HOST[dankclaw]=dankclaw
|
|
REMOTE_DIR[dankclaw]="/home/dank/serverAppConfigs"
|
|
COMPOSE[dankclaw]="dankclaw/compose.yaml"
|
|
HEALTH_URL[dankclaw]="http://127.0.0.1/"
|
|
COMPOSE_UP[dankclaw]=1
|
|
COPY_SHARED[dankclaw]=1
|
|
# LLM: frank-default-large is the active service (restart + health wait).
|
|
# frank-default-npu is relinked only (not enabled on the host); its unit
|
|
# picks up the new paths at its next manual start.
|
|
LLM_SERVICES[dankclaw]="frank-default-large"
|
|
LLM_UNITS[dankclaw]="frank-default-large frank-default-npu"
|
|
LLM_HEALTH[dankclaw]="http://127.0.0.1:7770/health"
|
|
# CIRU llama.cpp (ROCm 7.15) loads the v3 five-file payload (GGUF + FFN +
|
|
# GDN + Output-K8 .pfs + mmproj, ~36.7 GiB) + MTP draft/Output-K8 proxy and
|
|
# initializes PromptForge views; allow 20 minutes.
|
|
LLM_WAIT[dankclaw]=600
|
|
|
|
REMOTE_USER[dankserver]=dank
|
|
REMOTE_HOST[dankserver]=dankserver
|
|
REMOTE_DIR[dankserver]="/home/dank/serverAppConfigs"
|
|
COMPOSE[dankserver]="dankserver/compose.yaml"
|
|
HEALTH_URL[dankserver]="http://192.168.0.143/"
|
|
COMPOSE_UP[dankserver]=0
|
|
COPY_SHARED[dankserver]=0
|
|
# LLM: frank-default-small is the active service (restart + health wait).
|
|
LLM_SERVICES[dankserver]="frank-default-small"
|
|
LLM_UNITS[dankserver]="frank-default-small"
|
|
LLM_HEALTH[dankserver]="http://127.0.0.1:7770/health"
|
|
# Cold load: --no-mmap reads the 30.6 GB Q6_K_P through RAM (MoE experts on
|
|
# CPU); allow 20 minutes.
|
|
LLM_WAIT[dankserver]=600
|
|
|
|
TARGET="${1:-}"
|
|
DRY_RUN=0
|
|
if [ "${2:-}" = "--dry-run" ]; then
|
|
DRY_RUN=1
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Validation
|
|
# ---------------------------------------------------------------------------
|
|
if [ -z "${TARGET:-}" ]; then
|
|
echo "Usage: $0 <dankclaw|dankserver> [--dry-run]"
|
|
exit 1
|
|
fi
|
|
if [ -z "${REMOTE_HOST[$TARGET]:-}" ]; then
|
|
echo "Unknown target: $TARGET"
|
|
exit 1
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Sudo wrapper (headless-friendly)
|
|
# ---------------------------------------------------------------------------
|
|
SUDO="sudo"
|
|
if [ -n "${FRANK_SUDO_PW:-}" ]; then
|
|
SUDO="printf '%s\n' '$FRANK_SUDO_PW' | sudo -S"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Local checks (pre-flight)
|
|
# ---------------------------------------------------------------------------
|
|
cd "$(dirname "$0")/.."
|
|
if ! git diff --quiet HEAD; then
|
|
echo "FAIL: Local working tree has uncommitted changes."
|
|
echo "Commit or stash them before deploying."
|
|
exit 1
|
|
fi
|
|
if [ "$(git rev-parse --abbrev-ref HEAD)" != "main" ]; then
|
|
echo "WARN: Not on 'main'; deploying a non-main branch."
|
|
fi
|
|
|
|
echo "=== Deploying to ${TARGET} ==="
|
|
echo " host: ${REMOTE_HOST[$TARGET]}"
|
|
echo " dir: ${REMOTE_DIR[$TARGET]}"
|
|
echo " compose: ${COMPOSE[$TARGET]}"
|
|
echo " llm svc: ${LLM_SERVICES[$TARGET]}"
|
|
echo " llm health: ${LLM_HEALTH[$TARGET]}"
|
|
echo " homepage: ${HEALTH_URL[$TARGET]}"
|
|
echo ""
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Push (unless dry-run)
|
|
# ---------------------------------------------------------------------------
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
echo "[dry-run] Would push local main to origin"
|
|
echo "[dry-run] Would SSH to ${REMOTE_HOST[$TARGET]} and git pull"
|
|
else
|
|
echo "[1/5] Pushing local main to origin..."
|
|
git push origin main
|
|
fi
|
|
|
|
REMOTE="$(
|
|
printf '%s@%s' "${REMOTE_USER[$TARGET]}" "${REMOTE_HOST[$TARGET]}"
|
|
)"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Remote: pull + copy shared configs + restart stack + restart LLM service
|
|
# ---------------------------------------------------------------------------
|
|
REMOTE_SCRIPT="
|
|
set -euo pipefail
|
|
cd '${REMOTE_DIR[$TARGET]}'
|
|
|
|
echo '[2/5] Pulling repo on ${TARGET}...'
|
|
git pull origin main
|
|
|
|
if [ -f .env ]; then
|
|
echo ' .env present, using it'
|
|
else
|
|
echo ' WARN: no .env on target; copy the .env.example and fill it in'
|
|
fi
|
|
|
|
if [ '${COPY_SHARED[$TARGET]}' = '1' ]; then
|
|
echo '[3/5] Copying shared configs into place...'
|
|
mkdir -p /opt/searxng/config /opt/prometheus/config /opt/loki/config /opt/alloy/config
|
|
${SUDO} cp shared/searxng/settings.yml /opt/searxng/config/settings.yml
|
|
${SUDO} cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml
|
|
${SUDO} cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml
|
|
${SUDO} cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
|
|
${SUDO} cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
|
|
|
|
echo ' Rendering Caddyfile (basicauth hash from .env)...'
|
|
# Extract the hash with grep/cut rather than sourcing .env: sourcing a
|
|
# bcrypt value in a set -u shell dies on its dollar-sign expansion.
|
|
# '|| true' so a missing hash warns below instead of aborting under set -e.
|
|
# Only the shared family hash remains: the homepage's per-user hashes were
|
|
# retired in favor of Authentik forward_auth (the /admin/ page and the user
|
|
# page are now behind the same SSO session).
|
|
CADDY_BASICAUTH_HASH=\"\$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)\"
|
|
if [ -n \"\${CADDY_BASICAUTH_HASH}\" ]; then
|
|
# '|' delimiter: the bcrypt hash contains '/' characters.
|
|
mkdir -p /home/dank/Containers/caddy
|
|
sed -e \"s|REPLACE_WITH_HASH|\${CADDY_BASICAUTH_HASH}|g\" \\
|
|
dankclaw/Caddyfile > /home/dank/Containers/caddy/Caddyfile
|
|
else
|
|
echo ' WARN: CADDY_BASICAUTH_HASH is not set in .env; keeping the placeholder(s)'
|
|
fi
|
|
|
|
echo ' Deploying the homepage...'
|
|
${SUDO} mkdir -p /srv/homepage
|
|
${SUDO} cp shared/homepage/index.html shared/homepage/admin.html shared/homepage/app.js \\
|
|
shared/homepage/style.css shared/homepage/services.json /srv/homepage/
|
|
if [ -d shared/homepage/fonts ]; then ${SUDO} cp -r shared/homepage/fonts /srv/homepage/; fi
|
|
# The last master deployment datetime (Q43): the deployed commit's datetime.
|
|
DEPLOYED_AT=\"\$(git log -1 --format=%cI)\"
|
|
printf 'window.DEPLOYED_AT = \"%s\";\\n' \"\$DEPLOYED_AT\" > /tmp/deployed.js
|
|
${SUDO} cp /tmp/deployed.js /srv/homepage/deployed.js
|
|
${SUDO} chmod -R a+rX /srv/homepage
|
|
# The status checker: install the timer + service, run it once, enable it.
|
|
${SUDO} cp scripts/homepage-status.py /usr/local/bin/homepage-status.py
|
|
${SUDO} cp scripts/homepage-status.service scripts/homepage-status.timer /etc/systemd/system/
|
|
${SUDO} systemctl daemon-reload
|
|
${SUDO} systemctl enable homepage-status.timer
|
|
${SUDO} systemctl restart homepage-status.timer
|
|
${SUDO} python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json
|
|
fi
|
|
|
|
if [ '${COMPOSE_UP[$TARGET]}' = '1' ]; then
|
|
echo '[4/5] Restarting stack...'
|
|
# Locally built images: caddy + porkbun module, and loki/alloy + a static
|
|
# busybox (their upstream images lack the wget the healthchecks need).
|
|
for dir in caddy loki alloy; do
|
|
img=franksdank/\$dir
|
|
[ \"\$dir\" = caddy ] && img=franksdank/caddy-porkbun
|
|
# Rebuild when the Containerfile changed. buildah's layer cache can
|
|
# serve a stale COPY --from layer (keyed on stage name), so a rebuild
|
|
# always runs with --no-cache.
|
|
cfhash=\$(sha256sum dankclaw/\$dir/Containerfile | cut -d' ' -f1)
|
|
oldhash=\$(podman image inspect \$img:latest --format '{{index .Config.Labels \"cfhash\"}}' 2>/dev/null || echo none)
|
|
if [ \"\$oldhash\" != \"\$cfhash\" ]; then
|
|
echo \" \$img image missing or stale; building it...\"
|
|
podman build --no-cache --label cfhash=\$cfhash -t \$img:latest -f dankclaw/\$dir/Containerfile .
|
|
fi
|
|
done
|
|
podman-compose -f '${COMPOSE[$TARGET]}' up -d
|
|
|
|
# A Caddyfile render is a bind-mount content change, not a container
|
|
# config change: the caddy container often survives 'up -d' and keeps
|
|
# serving its in-memory config. Reload explicitly; the file watcher
|
|
# (admin on) is the backstop if the API is not up yet. If the reload
|
|
# cannot be applied at all (e.g. the running config has the admin API
|
|
# disabled -- the fix cannot ride the API it is meant to enable),
|
|
# restart the container so it reads the file on start.
|
|
caddy_reloaded=0
|
|
for i in 1 2 3 4 5; do
|
|
if podman exec caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile >/dev/null 2>&1; then
|
|
caddy_reloaded=1
|
|
echo \" Caddy reloaded.\"
|
|
break
|
|
fi
|
|
sleep 3
|
|
done
|
|
if [ \"\$caddy_reloaded\" -ne 1 ]; then
|
|
echo \" Caddy reload failed; restarting the container to load the file.\"
|
|
podman restart caddy
|
|
fi
|
|
else
|
|
echo '[4/5] Skipping compose restart (stack runs as bare containers on this host)'
|
|
fi
|
|
|
|
echo '[4/5] Relinking LLM units and restarting LLM services...'
|
|
for unit in ${LLM_UNITS[$TARGET]}; do
|
|
# The loop variable must expand on the remote, so it is escaped here and
|
|
# the path is unquoted (it has no spaces). Quoting the source path would
|
|
# ship a literal loop-variable token and create a dangling symlink
|
|
# (the 2026-08-22 relink bug).
|
|
${SUDO} ln -sf ${REMOTE_DIR[$TARGET]}/scripts/\${unit}.service /etc/systemd/system/\${unit}.service
|
|
done
|
|
${SUDO} systemctl daemon-reload
|
|
for svc in ${LLM_SERVICES[$TARGET]}; do
|
|
${SUDO} systemctl enable \"\${svc}\"
|
|
${SUDO} systemctl restart \"\${svc}\"
|
|
echo \" restarted \${svc}\"
|
|
done
|
|
"
|
|
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
echo "[dry-run] Remote commands would run:"
|
|
echo "$REMOTE_SCRIPT"
|
|
else
|
|
# shellcheck disable=SC2087
|
|
ssh "$REMOTE" bash -s <<REMOTE_EOF
|
|
${REMOTE_SCRIPT}
|
|
REMOTE_EOF
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Health checks: homepage, then the LLM endpoint
|
|
# ---------------------------------------------------------------------------
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
echo "[dry-run] Would wait for health at ${HEALTH_URL[$TARGET]} and ${LLM_HEALTH[$TARGET]}"
|
|
else
|
|
echo "[5/5] Waiting for homepage ${HEALTH_URL[$TARGET]} (via ssh)..."
|
|
HEALTH="${HEALTH_URL[$TARGET]}"
|
|
for i in $(seq 1 60); do
|
|
if ssh "$REMOTE" "curl -sf '$HEALTH'" > /dev/null 2>&1; then
|
|
echo " Homepage healthy after approx $((i * 2))s"
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
if ! ssh "$REMOTE" "curl -sf '$HEALTH'" > /dev/null 2>&1; then
|
|
echo "FAIL: Homepage did not become healthy in 120s"
|
|
exit 1
|
|
fi
|
|
|
|
echo "[5/5] Waiting for LLM health ${LLM_HEALTH[$TARGET]} (via ssh)..."
|
|
LLM_URL="${LLM_HEALTH[$TARGET]}"
|
|
LLM_ITERS="${LLM_WAIT[$TARGET]}"
|
|
for i in $(seq 1 "$LLM_ITERS"); do
|
|
if ssh "$REMOTE" "curl -sf '$LLM_URL'" > /dev/null 2>&1; then
|
|
echo " LLM healthy after approx $((i * 2))s"
|
|
echo ""
|
|
echo "=== Deploy complete: ${TARGET} ==="
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "FAIL: LLM service did not become healthy in $((LLM_ITERS * 2))s"
|
|
exit 1
|
|
fi
|