diff --git a/AGENTS.md b/AGENTS.md index 3498668..53d17cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,7 +85,7 @@ graph TD - `sudo` on both boxes needs a tty. The remote sudo steps in `deploy.sh` fail non-interactively. Use `echo '' | sudo -S `. - Never run `pkill -f` with a pattern that appears in your own command line. It kills your ssh session. Use `pkill -x `. - Model load is slow: about 3 minutes warm, more than 8 minutes cold. Health returns `503 Loading model` the whole time. That is normal. Do not kill the process. -- Page-cache state confounds crash tests. Cold-cache loads are slow and survive. Warm-cache loads crash fast. Control for it when testing. +- Page-cache state confounds crash tests. Cold-cache loads are slow and survive. Warm-cache loads crash fast. Control for it when testing. `deploy.sh` drops caches on Dankclaw before restart (see `DROP_CACHES`); manual restarts must run `sync && echo 3 | sudo tee /proc/sys/vm/drop_caches` first. ### Dankclaw memory model diff --git a/docs/deployment.md b/docs/deployment.md index aee9795..437c4f8 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -59,18 +59,22 @@ The script does these actions: 2. Pushes the `main` branch to origin. 3. Pulls the repository on the target machine. 4. Installs or updates the systemd unit. -5. Reloads systemd and restarts the service. -6. Waits for the health endpoint to respond. +5. Drops page caches on Dankclaw before the restart. This prevents the load stall that happens when the model's pages are fully cached. +6. Reloads systemd and restarts the service. +7. Waits for the health endpoint to respond (over ssh). -## Manual Update +Set `FRANK_SUDO_PW` when running headless, so that remote sudo reads the password from stdin: -If you cannot run the deploy script, update a server manually. Complete each step on the target machine. +``` +FRANK_SUDO_PW=... ./scripts/deploy.sh dankclaw +``` ### Update Dankclaw 1. Pull the repository. 2. Symlink the unit file, if it changed. -3. Reload and restart the service. +3. Drop the page caches. +4. Reload and restart the service. Run these commands on Dankclaw: @@ -78,6 +82,7 @@ Run these commands on Dankclaw: cd /home/dank/vllmScripts git pull origin main sudo ln -sf /home/dank/vllmScripts/scripts/frank-default-large.service /etc/systemd/system/frank-default-large.service +sync && echo 3 | sudo tee /proc/sys/vm/drop_caches sudo systemctl daemon-reload sudo systemctl restart frank-default-large curl -sf http://127.0.0.1:7770/health diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 31ec1b6..5361d12 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -5,14 +5,18 @@ # AGENTS.md. It assumes the local changes are committed and pushed, then: # 1. Pulls the repo on the target # 2. Installs / enables the systemd unit -# 3. Reloads systemd and restarts the service +# 3. Drops page caches (dankclaw) and restarts the service # 4. Waits for the health endpoint and reports status # +# Sudo: when FRANK_SUDO_PW is set, remote sudo pipes it via `sudo -S` for +# headless runs. Otherwise plain `sudo` is used (tty or passwordless config). +# # Usage: # ./deploy.sh [--dry-run] +# FRANK_SUDO_PW=... ./deploy.sh # # Targets: -# dankclaw - vLLM-ROCm server (frank-default-large.service, port 7770) +# dankclaw - llama.cpp server (frank-default-large.service, port 7770) # dankserver - llama.cpp server (frank-default-small.service, port 7770) # # Example: @@ -23,19 +27,23 @@ set -euo pipefail # --------------------------------------------------------------------------- # Target configuration # --------------------------------------------------------------------------- -declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR SERVICE SERVICE_HEALTH +declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR SERVICE SERVICE_HEALTH DROP_CACHES REMOTE_USER[dankclaw]=dank REMOTE_HOST[dankclaw]=dankclaw REMOTE_DIR[dankclaw]="/home/dank/vllmScripts" SERVICE[dankclaw]="frank-default-large" SERVICE_HEALTH[dankclaw]="http://127.0.0.1:7770/health" +# Drop page caches before restart: with the model's pages fully cached, the +# KV/graph allocation stalls at load (VRAM stuck at 28 GiB). See AGENTS.md. +DROP_CACHES[dankclaw]=1 REMOTE_USER[dankserver]=dank REMOTE_HOST[dankserver]=dankserver REMOTE_DIR[dankserver]="/home/dank/vLLM" SERVICE[dankserver]="frank-default-small" SERVICE_HEALTH[dankserver]="http://127.0.0.1:7770/health" +DROP_CACHES[dankserver]=0 TARGET="${1:-}" DRY_RUN=0 @@ -55,6 +63,16 @@ if [ -z "${REMOTE_HOST[$TARGET]:-}" ]; then exit 1 fi +# --------------------------------------------------------------------------- +# Sudo wrapper (headless-friendly) +# --------------------------------------------------------------------------- +# When FRANK_SUDO_PW is set, remote sudo reads the password from stdin via +# `sudo -S`. Otherwise plain `sudo` is used (tty or passwordless config). +SUDO_PREFIX="sudo" +if [ -n "${FRANK_SUDO_PW:-}" ]; then + SUDO_PREFIX="printf '%s\n' '$FRANK_SUDO_PW' | sudo -S" +fi + # --------------------------------------------------------------------------- # Local checks (pre-flight) # --------------------------------------------------------------------------- @@ -93,6 +111,14 @@ REMOTE="$( # --------------------------------------------------------------------------- # Remote: pull + install + restart # --------------------------------------------------------------------------- +DROP_CACHES_CMD="" +if [ "${DROP_CACHES[$TARGET]}" = "1" ]; then + DROP_CACHES_CMD=" +echo 'Dropping page caches for reliable model load...' +${SUDO_PREFIX} sh -c 'sync; echo 3 > /proc/sys/vm/drop_caches' +" +fi + REMOTE_SCRIPT=" set -euo pipefail cd '${REMOTE_DIR[$TARGET]}' @@ -103,24 +129,27 @@ git pull origin main # Install / symlink the systemd unit case '${TARGET}' in dankclaw) - sudo ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-large.service' /etc/systemd/system/frank-default-large.service + ${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-large.service' /etc/systemd/system/frank-default-large.service ;; dankserver) - sudo ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-small.service' /etc/systemd/system/frank-default-small.service + ${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-small.service' /etc/systemd/system/frank-default-small.service ;; esac - +${DROP_CACHES_CMD} echo '[3/4] Reload and restart ${SERVICE[$TARGET]}...' -sudo systemctl daemon-reload -sudo systemctl enable '${SERVICE[$TARGET]}' -sudo systemctl restart '${SERVICE[$TARGET]}' +${SUDO_PREFIX} systemctl daemon-reload +${SUDO_PREFIX} systemctl enable '${SERVICE[$TARGET]}' +${SUDO_PREFIX} systemctl restart '${SERVICE[$TARGET]}' " if [ "$DRY_RUN" = "1" ]; then echo "[dry-run] Remote commands would run:" echo "$REMOTE_SCRIPT" else - ssh "$REMOTE" "bash -c '$REMOTE_SCRIPT'" + # shellcheck disable=SC2087 + ssh "$REMOTE" bash -s < /dev/null 2>&1; then + if ssh "$REMOTE" "curl -sf '$HEALTH_URL'" > /dev/null 2>&1; then echo " Service healthy after approx $((i * 2))s" echo "" echo "=== Deploy complete: ${TARGET} ==="