197 lines
7.1 KiB
Bash
Executable File
197 lines
7.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# deploy.sh — Deploy the vllmScripts repo to a target and restart its service.
|
|
#
|
|
# This script implements the second half of the git deployment workflow from
|
|
# AGENTS.md. It assumes the local changes are committed and pushed, then:
|
|
# 1. Pulls the repo on the target
|
|
# 2. Installs / enables the systemd unit
|
|
# 3. Drops page caches (dankclaw) and restarts the service
|
|
# 4. Waits for the health endpoint and reports status
|
|
#
|
|
# Sudo: when FRANK_SUDO_PW is set, remote sudo pipes it via `sudo -S` for
|
|
# headless runs. Otherwise plain `sudo` is used (tty or passwordless config).
|
|
# Prefer the passwordless restart helpers installed on each target
|
|
# (/usr/local/sbin/frank-restart-<large|small>.sh); they cover the unit
|
|
# symlink, cache drop (dankclaw), daemon-reload, and restart in one root
|
|
# script. See docs/deployment.md -> Passwordless restart.
|
|
#
|
|
# Usage:
|
|
# ./deploy.sh <target> [--dry-run]
|
|
# FRANK_SUDO_PW=... ./deploy.sh <target>
|
|
#
|
|
# Targets:
|
|
# dankclaw - llama.cpp server (frank-default-large.service, port 7770)
|
|
# dankserver - llama.cpp server (frank-default-small.service, port 7770)
|
|
#
|
|
# Example:
|
|
# ./deploy.sh dankclaw
|
|
# ./deploy.sh dankserver --dry-run
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Target configuration
|
|
# ---------------------------------------------------------------------------
|
|
declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR SERVICE SERVICE_HEALTH DROP_CACHES HELPER HEALTH_WAIT
|
|
|
|
REMOTE_USER[dankclaw]=dank
|
|
REMOTE_HOST[dankclaw]=dankclaw
|
|
REMOTE_DIR[dankclaw]="/home/dank/vllmScripts"
|
|
SERVICE[dankclaw]="frank-default-large"
|
|
SERVICE_HEALTH[dankclaw]="http://127.0.0.1:7770/health"
|
|
# Drop page caches before restart: with the model's pages fully cached, the
|
|
# KV/graph allocation stalls at load (VRAM stuck at 28 GiB). See AGENTS.md.
|
|
DROP_CACHES[dankclaw]=1
|
|
# Passwordless restart helper (root-owned, installed via /etc/sudoers.d/frank-large).
|
|
# When present, deploy.sh uses it instead of inline sudo commands.
|
|
HELPER[dankclaw]="/usr/local/sbin/frank-restart-large.sh"
|
|
# Health wait in iterations of 2s. Cold loads after the cache drop measured
|
|
# at up to 17 minutes on 2026-08-13; 600 iterations = 20 minutes.
|
|
HEALTH_WAIT[dankclaw]=600
|
|
|
|
REMOTE_USER[dankserver]=dank
|
|
REMOTE_HOST[dankserver]=dankserver
|
|
REMOTE_DIR[dankserver]="/home/dank/vLLM"
|
|
SERVICE[dankserver]="frank-default-small"
|
|
SERVICE_HEALTH[dankserver]="http://127.0.0.1:7770/health"
|
|
DROP_CACHES[dankserver]=0
|
|
HELPER[dankserver]="/usr/local/sbin/frank-restart-small.sh"
|
|
HEALTH_WAIT[dankserver]=300
|
|
|
|
TARGET="${1:-}"
|
|
DRY_RUN=0
|
|
if [ "${2:-}" = "--dry-run" ]; then
|
|
DRY_RUN=1
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Validation
|
|
# ---------------------------------------------------------------------------
|
|
if [ -z "${TARGET:-}" ]; then
|
|
echo "Usage: $0 <dankclaw|dankserver> [--dry-run]"
|
|
exit 1
|
|
fi
|
|
if [ -z "${REMOTE_HOST[$TARGET]:-}" ]; then
|
|
echo "Unknown target: $TARGET"
|
|
exit 1
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Sudo wrapper (headless-friendly)
|
|
# ---------------------------------------------------------------------------
|
|
# When FRANK_SUDO_PW is set, remote sudo reads the password from stdin via
|
|
# `sudo -S`. Otherwise plain `sudo` is used (tty or passwordless config).
|
|
SUDO_PREFIX="sudo"
|
|
if [ -n "${FRANK_SUDO_PW:-}" ]; then
|
|
SUDO_PREFIX="printf '%s\n' '$FRANK_SUDO_PW' | sudo -S"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Local checks (pre-flight)
|
|
# ---------------------------------------------------------------------------
|
|
cd "$(dirname "$0")/.."
|
|
if ! git diff --quiet HEAD; then
|
|
echo "FAIL: Local working tree has uncommitted changes."
|
|
echo "Commit or stash them before deploying."
|
|
exit 1
|
|
fi
|
|
if [ "$(git rev-parse --abbrev-ref HEAD)" != "main" ]; then
|
|
echo "WARN: Not on 'main'; deploying a non-main branch."
|
|
fi
|
|
|
|
echo "=== Deploying to ${TARGET} ==="
|
|
echo " host: ${REMOTE_HOST[$TARGET]}"
|
|
echo " dir: ${REMOTE_DIR[$TARGET]}"
|
|
echo " service: ${SERVICE[$TARGET]}"
|
|
echo " health: ${SERVICE_HEALTH[$TARGET]}"
|
|
echo ""
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Push (unless dry-run)
|
|
# ---------------------------------------------------------------------------
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
echo "[dry-run] Would push local main to origin"
|
|
echo "[dry-run] Would SSH to ${REMOTE_HOST[$TARGET]} and git pull"
|
|
else
|
|
echo "[1/4] Pushing local main to origin..."
|
|
git push origin main
|
|
fi
|
|
|
|
REMOTE="$(
|
|
printf '%s@%s' "${REMOTE_USER[$TARGET]}" "${REMOTE_HOST[$TARGET]}"
|
|
)"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Remote: pull + install + restart
|
|
# ---------------------------------------------------------------------------
|
|
DROP_CACHES_CMD=""
|
|
if [ "${DROP_CACHES[$TARGET]}" = "1" ]; then
|
|
DROP_CACHES_CMD="
|
|
echo 'Dropping page caches for reliable model load...'
|
|
${SUDO_PREFIX} sh -c 'sync; echo 3 > /proc/sys/vm/drop_caches'
|
|
"
|
|
fi
|
|
|
|
HELPER_PATH="${HELPER[$TARGET]:-}"
|
|
|
|
REMOTE_SCRIPT="
|
|
set -euo pipefail
|
|
cd '${REMOTE_DIR[$TARGET]}'
|
|
|
|
echo '[2/4] Pulling repo on ${TARGET}...'
|
|
git pull origin main
|
|
|
|
if [ -x '${HELPER_PATH}' ]; then
|
|
# Passwordless restart helper (sudoers NOPASSWD) - preferred path.
|
|
echo '[3/4] Restarting via passwordless helper ${HELPER_PATH}...'
|
|
${SUDO_PREFIX} '${HELPER_PATH}'
|
|
else
|
|
echo '[3/4] Helper not installed; using inline sudo commands...'
|
|
# Install / symlink the systemd unit
|
|
case '${TARGET}' in
|
|
dankclaw)
|
|
${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-large.service' /etc/systemd/system/frank-default-large.service
|
|
;;
|
|
dankserver)
|
|
${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-small.service' /etc/systemd/system/frank-default-small.service
|
|
;;
|
|
esac
|
|
${DROP_CACHES_CMD}
|
|
echo '[3/4] Reload and restart ${SERVICE[$TARGET]}...'
|
|
${SUDO_PREFIX} systemctl daemon-reload
|
|
${SUDO_PREFIX} systemctl enable '${SERVICE[$TARGET]}'
|
|
${SUDO_PREFIX} systemctl restart '${SERVICE[$TARGET]}'
|
|
fi
|
|
"
|
|
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
echo "[dry-run] Remote commands would run:"
|
|
echo "$REMOTE_SCRIPT"
|
|
else
|
|
# shellcheck disable=SC2087
|
|
ssh "$REMOTE" bash -s <<REMOTE_EOF
|
|
${REMOTE_SCRIPT}
|
|
REMOTE_EOF
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Health check
|
|
# ---------------------------------------------------------------------------
|
|
if [ "$DRY_RUN" = "1" ]; then
|
|
echo "[dry-run] Would wait for health at ${SERVICE_HEALTH[$TARGET]}"
|
|
else
|
|
echo "[4/4] Waiting for health endpoint ${SERVICE_HEALTH[$TARGET]} (via ssh)..."
|
|
HEALTH_URL="${SERVICE_HEALTH[$TARGET]}"
|
|
HEALTH_ITERS="${HEALTH_WAIT[$TARGET]:-300}"
|
|
for i in $(seq 1 "$HEALTH_ITERS"); do
|
|
if ssh "$REMOTE" "curl -sf '$HEALTH_URL'" > /dev/null 2>&1; then
|
|
echo " Service healthy after approx $((i * 2))s"
|
|
echo ""
|
|
echo "=== Deploy complete: ${TARGET} ==="
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "FAIL: Service did not become healthy in $((HEALTH_ITERS * 2))s"
|
|
exit 1
|
|
fi
|