diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index e268c60..f4122d7 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -993,10 +993,34 @@ retire_root_llm_large() { run_privileged mv -- /etc/systemd/system/frank-large.service \ "$trash/frank-large.service" echo "retired root unit frank-large.service -> $trash" + else + # This host's `systemctl disable` is allowed to delete the unit file + # outright (2026-08-26 incident), so there is often nothing left to + # move. Say so: previously no message meant no evidence either way. + echo "root unit frank-large.service was already removed by disable" fi run_privileged systemctl daemon-reload } +# Wait until a TCP port has no socket on it at all -- listening, accepted, or +# lingering in TIME_WAIT. Rootless podman publishes through pasta, which binds +# the host port at container CREATE, and a stopped LLM runtime keeps its +# listener (and the TIME_WAIT residue of the connections it served) for a +# moment after the process exits. `sport` is the port in question on both the +# listener and the server side of every accepted connection. +wait_port_free() { + local port="$1" limit="${2:-90}" i + for ((i = 1; i <= limit; i++)); do + if [[ -z "$(ss -Htan "sport = :$port" 2>/dev/null)" ]]; then + return 0 + fi + sleep 1 + done + echo "FAIL: port $port is still held after ${limit}s:" >&2 + ss -tan "sport = :$port" >&2 || true + return 1 +} + deploy_llm_large() { retire_root_llm_large # Pull once, then run the localhost/-aliased tag: this host's quadlet path @@ -1010,6 +1034,13 @@ deploy_llm_large() { # The weights volume must exist, and the container needs it read-write: # that is where a first start downloads into. mkdir -p /home/dank/models/halogen + # Nothing may start until the predecessor has released the port. On + # 2026-09-16 this sequence failed exactly here: the ik server logged + # "Deactivated successfully", pasta tried to bind 7770 a second later and + # reported "Address already in use", and the unit exited 126. Restart= + # on-failure recovered it minutes later, but a deploy that starts a doomed + # container has not deployed anything -- so gate on the port, not on hope. + wait_port_free 7770 || return 1 install_quadlet frank-large dankclaw/frank-large/quadlet frank-large # No enable: generated units cannot be enabled on this host (v1 quadlet # generator); boot persistence comes from [Install] in the unit file. diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 4a7491b..5991dd8 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -102,3 +102,12 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" [ ! -e "$REPO_ROOT/scripts/frank-large.service" ] [ ! -e "$REPO_ROOT/scripts/serve-frank-large.sh" ] } + +@test "the deploy handler waits for the predecessor to release the port" { + # The 2026-09-16 cutover failed at exactly this: the ik server's socket was + # still on 7770 when pasta tried to bind it ("Address already in use"), and + # the container died on it. Restart=on-failure recovered minutes later, but + # a deploy that starts a doomed container has not deployed anything. + grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" + grep -q '^wait_port_free() {' "$REMOTE_DEPLOY" +} \ No newline at end of file diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 8569ae6..a982279 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -52,7 +52,9 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" # MTP tensors at all, so MTP is a property of the file, not of the flags. grep -q '^MODEL_FILE=.*-mtp\.gguf$' "$ENV_FILE" # A separate draft model would load a second 12 GiB copy of a 12 GiB model. - ! grep -q -- '--model-draft' "$SERVE" + # Only the effective configuration counts: the script's own comment explains + # why there is no draft path, and a textual grep would match that prose. + ! grep -vE '^[[:space:]]*#' "$SERVE" | grep -q -- '--model-draft' } @test "frank-small carries the fleet's image path, and only it" {