harness: llama-server lifecycle in pre-clean and health guard

This commit is contained in:
Frank
2026-10-01 21:58:51 -07:00
parent cde91c1a00
commit 01ba41cab4
2 changed files with 13 additions and 6 deletions
+3 -2
View File
@@ -38,7 +38,8 @@ rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \
# 2. Free the GPU and the port: stop the deployed unit (mask is unnecessary—
# the unit only starts via systemd, and we never enable more of it).
ssh "$HOST" 'sudo -n systemctl stop frank-large.service;
podman rm -f frank-large-mimo 2>/dev/null; true' >/dev/null
podman rm -f frank-large-mimo 2>/dev/null;
pkill -x llama-server 2>/dev/null; true' >/dev/null
# 3. Boot the candidate detached; the launcher's own output IS the serve
# stage output (gate failures print there before any container exists).
@@ -55,7 +56,7 @@ if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do
http://127.0.0.1:7770/health 2>/dev/null || echo 000)
[ \"\$code\" = 200 ] && echo UP && exit 0
st=\$(podman inspect -f '{{.State.Status}}' frank-large-mimo 2>/dev/null || echo gone)
if [ \"\$st\" = exited ] || { [ \"\$st\" = gone ] && ! pgrep -f serve-frank-large >/dev/null; }; then echo DOWN; exit 1; fi
if [ "\$st" = exited ] || { [ "\$st" = gone ] && ! pgrep -f serve-frank-large >/dev/null && ! pgrep -x llama-server >/dev/null; }; then echo DOWN; exit 1; fi
sleep 5
done; exit 1" | grep -q UP; then
echo "ERROR: candidate never became healthy; serve.log tail:" >&2