deploy: never start the replacement while the predecessor holds the port
The 2026-09-16 halogen restore failed exactly here. The ik root unit stopped, systemd logged "Deactivated successfully", pasta tried to bind 7770 about a second later, reported "Address already in use", and the quadlet exited 126. Only Restart=on-failure recovered it -- minutes later, which is not a deploy. The handler now waits for every socket on the port to clear (listener, accepted and TIME_WAIT) before installing and starting the quadlet, and fails loudly with the socket table attached if it never does. The retirement also reports when the unit file was already gone, because this host's `disable` deletes it.
This commit is contained in:
@@ -993,10 +993,34 @@ retire_root_llm_large() {
|
||||
run_privileged mv -- /etc/systemd/system/frank-large.service \
|
||||
"$trash/frank-large.service"
|
||||
echo "retired root unit frank-large.service -> $trash"
|
||||
else
|
||||
# This host's `systemctl disable` is allowed to delete the unit file
|
||||
# outright (2026-08-26 incident), so there is often nothing left to
|
||||
# move. Say so: previously no message meant no evidence either way.
|
||||
echo "root unit frank-large.service was already removed by disable"
|
||||
fi
|
||||
run_privileged systemctl daemon-reload
|
||||
}
|
||||
|
||||
# Wait until a TCP port has no socket on it at all -- listening, accepted, or
|
||||
# lingering in TIME_WAIT. Rootless podman publishes through pasta, which binds
|
||||
# the host port at container CREATE, and a stopped LLM runtime keeps its
|
||||
# listener (and the TIME_WAIT residue of the connections it served) for a
|
||||
# moment after the process exits. `sport` is the port in question on both the
|
||||
# listener and the server side of every accepted connection.
|
||||
wait_port_free() {
|
||||
local port="$1" limit="${2:-90}" i
|
||||
for ((i = 1; i <= limit; i++)); do
|
||||
if [[ -z "$(ss -Htan "sport = :$port" 2>/dev/null)" ]]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "FAIL: port $port is still held after ${limit}s:" >&2
|
||||
ss -tan "sport = :$port" >&2 || true
|
||||
return 1
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
retire_root_llm_large
|
||||
# Pull once, then run the localhost/-aliased tag: this host's quadlet path
|
||||
@@ -1010,6 +1034,13 @@ deploy_llm_large() {
|
||||
# The weights volume must exist, and the container needs it read-write:
|
||||
# that is where a first start downloads into.
|
||||
mkdir -p /home/dank/models/halogen
|
||||
# Nothing may start until the predecessor has released the port. On
|
||||
# 2026-09-16 this sequence failed exactly here: the ik server logged
|
||||
# "Deactivated successfully", pasta tried to bind 7770 a second later and
|
||||
# reported "Address already in use", and the unit exited 126. Restart=
|
||||
# on-failure recovered it minutes later, but a deploy that starts a doomed
|
||||
# container has not deployed anything -- so gate on the port, not on hope.
|
||||
wait_port_free 7770 || return 1
|
||||
install_quadlet frank-large dankclaw/frank-large/quadlet frank-large
|
||||
# No enable: generated units cannot be enabled on this host (v1 quadlet
|
||||
# generator); boot persistence comes from [Install] in the unit file.
|
||||
|
||||
@@ -102,3 +102,12 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
[ ! -e "$REPO_ROOT/scripts/frank-large.service" ]
|
||||
[ ! -e "$REPO_ROOT/scripts/serve-frank-large.sh" ]
|
||||
}
|
||||
|
||||
@test "the deploy handler waits for the predecessor to release the port" {
|
||||
# The 2026-09-16 cutover failed at exactly this: the ik server's socket was
|
||||
# still on 7770 when pasta tried to bind it ("Address already in use"), and
|
||||
# the container died on it. Restart=on-failure recovered minutes later, but
|
||||
# a deploy that starts a doomed container has not deployed anything.
|
||||
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
||||
grep -q '^wait_port_free() {' "$REMOTE_DEPLOY"
|
||||
}
|
||||
@@ -52,7 +52,9 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
|
||||
# MTP tensors at all, so MTP is a property of the file, not of the flags.
|
||||
grep -q '^MODEL_FILE=.*-mtp\.gguf$' "$ENV_FILE"
|
||||
# A separate draft model would load a second 12 GiB copy of a 12 GiB model.
|
||||
! grep -q -- '--model-draft' "$SERVE"
|
||||
# Only the effective configuration counts: the script's own comment explains
|
||||
# why there is no draft path, and a textual grep would match that prose.
|
||||
! grep -vE '^[[:space:]]*#' "$SERVE" | grep -q -- '--model-draft'
|
||||
}
|
||||
|
||||
@test "frank-small carries the fleet's image path, and only it" {
|
||||
|
||||
Reference in New Issue
Block a user