deploy: never start the replacement while the predecessor holds the port

The 2026-09-16 halogen restore failed exactly here. The ik root unit stopped,
systemd logged "Deactivated successfully", pasta tried to bind 7770 about a
second later, reported "Address already in use", and the quadlet exited 126.
Only Restart=on-failure recovered it -- minutes later, which is not a deploy.

The handler now waits for every socket on the port to clear (listener, accepted
and TIME_WAIT) before installing and starting the quadlet, and fails loudly with
the socket table attached if it never does. The retirement also reports when the
unit file was already gone, because this host's `disable` deletes it.
This commit is contained in:
Frank
2026-09-16 14:06:03 -07:00
parent 79cca19919
commit 7403bd3d50
3 changed files with 43 additions and 1 deletions
+31
View File
@@ -993,10 +993,34 @@ retire_root_llm_large() {
run_privileged mv -- /etc/systemd/system/frank-large.service \ run_privileged mv -- /etc/systemd/system/frank-large.service \
"$trash/frank-large.service" "$trash/frank-large.service"
echo "retired root unit frank-large.service -> $trash" echo "retired root unit frank-large.service -> $trash"
else
# This host's `systemctl disable` is allowed to delete the unit file
# outright (2026-08-26 incident), so there is often nothing left to
# move. Say so: previously no message meant no evidence either way.
echo "root unit frank-large.service was already removed by disable"
fi fi
run_privileged systemctl daemon-reload run_privileged systemctl daemon-reload
} }
# Wait until a TCP port has no socket on it at all -- listening, accepted, or
# lingering in TIME_WAIT. Rootless podman publishes through pasta, which binds
# the host port at container CREATE, and a stopped LLM runtime keeps its
# listener (and the TIME_WAIT residue of the connections it served) for a
# moment after the process exits. `sport` is the port in question on both the
# listener and the server side of every accepted connection.
wait_port_free() {
local port="$1" limit="${2:-90}" i
for ((i = 1; i <= limit; i++)); do
if [[ -z "$(ss -Htan "sport = :$port" 2>/dev/null)" ]]; then
return 0
fi
sleep 1
done
echo "FAIL: port $port is still held after ${limit}s:" >&2
ss -tan "sport = :$port" >&2 || true
return 1
}
deploy_llm_large() { deploy_llm_large() {
retire_root_llm_large retire_root_llm_large
# Pull once, then run the localhost/-aliased tag: this host's quadlet path # Pull once, then run the localhost/-aliased tag: this host's quadlet path
@@ -1010,6 +1034,13 @@ deploy_llm_large() {
# The weights volume must exist, and the container needs it read-write: # The weights volume must exist, and the container needs it read-write:
# that is where a first start downloads into. # that is where a first start downloads into.
mkdir -p /home/dank/models/halogen mkdir -p /home/dank/models/halogen
# Nothing may start until the predecessor has released the port. On
# 2026-09-16 this sequence failed exactly here: the ik server logged
# "Deactivated successfully", pasta tried to bind 7770 a second later and
# reported "Address already in use", and the unit exited 126. Restart=
# on-failure recovered it minutes later, but a deploy that starts a doomed
# container has not deployed anything -- so gate on the port, not on hope.
wait_port_free 7770 || return 1
install_quadlet frank-large dankclaw/frank-large/quadlet frank-large install_quadlet frank-large dankclaw/frank-large/quadlet frank-large
# No enable: generated units cannot be enabled on this host (v1 quadlet # No enable: generated units cannot be enabled on this host (v1 quadlet
# generator); boot persistence comes from [Install] in the unit file. # generator); boot persistence comes from [Install] in the unit file.
+9
View File
@@ -102,3 +102,12 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
[ ! -e "$REPO_ROOT/scripts/frank-large.service" ] [ ! -e "$REPO_ROOT/scripts/frank-large.service" ]
[ ! -e "$REPO_ROOT/scripts/serve-frank-large.sh" ] [ ! -e "$REPO_ROOT/scripts/serve-frank-large.sh" ]
} }
@test "the deploy handler waits for the predecessor to release the port" {
# The 2026-09-16 cutover failed at exactly this: the ik server's socket was
# still on 7770 when pasta tried to bind it ("Address already in use"), and
# the container died on it. Restart=on-failure recovered minutes later, but
# a deploy that starts a doomed container has not deployed anything.
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
grep -q '^wait_port_free() {' "$REMOTE_DEPLOY"
}
+3 -1
View File
@@ -52,7 +52,9 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
# MTP tensors at all, so MTP is a property of the file, not of the flags. # MTP tensors at all, so MTP is a property of the file, not of the flags.
grep -q '^MODEL_FILE=.*-mtp\.gguf$' "$ENV_FILE" grep -q '^MODEL_FILE=.*-mtp\.gguf$' "$ENV_FILE"
# A separate draft model would load a second 12 GiB copy of a 12 GiB model. # A separate draft model would load a second 12 GiB copy of a 12 GiB model.
! grep -q -- '--model-draft' "$SERVE" # Only the effective configuration counts: the script's own comment explains
# why there is no draft path, and a textual grep would match that prose.
! grep -vE '^[[:space:]]*#' "$SERVE" | grep -q -- '--model-draft'
} }
@test "frank-small carries the fleet's image path, and only it" { @test "frank-small carries the fleet's image path, and only it" {