refact(deployer): one MainPID gate for all three LLM endpoints
frank-dragoon's health gate never checked WHO answered :7770 — the false-green the fleet learned about on 2026-09-26 (a stray probe holding the port satisfies a bare curl while the unit is stopped). The other two handlers carried that check as inlined copies; all three now call assert_unit_owns_port, and dragoon gets the protection it was missing. The seam follows: the endpoints' text-grep deployer tests are deleted, because tests/llm_deploy_gates.bats EXECUTES the same handlers end to end (stub systemctl/podman/sudo/ss/curl) and asserts refusals, restarts, unit links, the pending marker, and the MainPID hardening. GATE_PREFIX is exported at each handler for the same reason the serve scripts export it.
This commit is contained in:
+21
-25
@@ -1166,6 +1166,20 @@ wait_port_free() {
|
||||
ss -tan "sport = :$port" >&2 || true
|
||||
return 1
|
||||
}
|
||||
# assert_unit_owns_port SERVICE PORT — the false-green gate every LLM
|
||||
# endpoint shares. A health answer proves the PORT replied, never WHO
|
||||
# replied: on 2026-09-26 a leftover probe held :7770, satisfied the curl,
|
||||
# and the deploy reported green over a stopped unit. Green requires the
|
||||
# unit's own MainPID to be alive behind the answer.
|
||||
assert_unit_owns_port() {
|
||||
local service="$1" port="$2" main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value "$service.service")"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :$port answered but $service.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "$service: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
deploy_llm_dragoon() {
|
||||
# Refuse rather than restart-loop; every gate here is the SAME library
|
||||
@@ -1173,7 +1187,7 @@ deploy_llm_dragoon() {
|
||||
# 2026-10-01 crash-loop lesson).
|
||||
# Capture podman output before grepping is moot: require_image is one
|
||||
# command, no pipe (the 2026-09-30 SIGPIPE lesson).
|
||||
GATE_PREFIX=frank-dragoon
|
||||
export GATE_PREFIX=frank-dragoon
|
||||
require_image "$LLM_DRAGOON_IMAGE" \
|
||||
"pull it: podman pull docker://${LLM_DRAGOON_IMAGE%@*}" || return 1
|
||||
require_bytes "$LLM_DRAGOON_CKPT" checkpoint "$LLM_DRAGOON_CKPT_BYTES" \
|
||||
@@ -1205,7 +1219,8 @@ deploy_llm_dragoon() {
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-dragoon.service
|
||||
run_privileged systemctl restart frank-dragoon.service
|
||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
|
||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200 || return 1
|
||||
assert_unit_owns_port frank-dragoon 7770
|
||||
}
|
||||
|
||||
# deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on
|
||||
@@ -1231,7 +1246,7 @@ deploy_llm_zealot() {
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why. Same library calls
|
||||
# as serve-frank-zealot.sh (the 2026-10-01 law).
|
||||
GATE_PREFIX=frank-zealot
|
||||
export GATE_PREFIX=frank-zealot
|
||||
require_exec "$LLM_ZEALOT_RUNTIME" runtime \
|
||||
"build it: cmake -B build-cuda -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86" \
|
||||
|| return 1
|
||||
@@ -1245,16 +1260,7 @@ deploy_llm_zealot() {
|
||||
run_privileged systemctl enable frank-zealot.service
|
||||
run_privileged systemctl restart frank-zealot.service
|
||||
wait_for_health frank-zealot http://127.0.0.1:7771/health 900 || return 1
|
||||
# False-green hardening (frank-dragoon's lesson): the port answering
|
||||
# proves nothing about WHO answers — the unit must be active with a
|
||||
# live MainPID, or a stray SGLang holdover is masquerading as deployed.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-zealot.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7771 answered but frank-zealot.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-zealot: health answered by unit MainPID $main_pid"
|
||||
assert_unit_owns_port frank-zealot 7771
|
||||
}
|
||||
|
||||
# deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode,
|
||||
@@ -1285,7 +1291,7 @@ deploy_llm_archon() {
|
||||
# shared library's, called with the values the unit's own env file
|
||||
# declares — so serve-frank-archon.sh and this handler cannot disagree,
|
||||
# and the payload path lives in exactly one file.
|
||||
GATE_PREFIX=frank-archon
|
||||
export GATE_PREFIX=frank-archon
|
||||
local env_file="${LLM_ARCHON_ENV_FILE:-$REMOTE_DIR/scripts/frank-archon.env}"
|
||||
local image model_dir model_file rad_min
|
||||
if [[ ! -f "$env_file" ]]; then
|
||||
@@ -1350,17 +1356,7 @@ deploy_llm_archon() {
|
||||
echo "FAIL: frank-archon /health never answered 200 in 1800s" >&2
|
||||
return 1
|
||||
fi
|
||||
# False-green hardening (the 2026-09-26 lesson): a bare curl against the
|
||||
# port cannot tell WHO answers — a stray probe holding :7770 satisfies
|
||||
# it while the unit is stopped. The deploy is only green if the unit
|
||||
# itself is active with a live MainPID behind the health answer.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-archon.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7770 answered but frank-archon.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-archon: health answered by unit MainPID $main_pid"
|
||||
assert_unit_owns_port frank-archon 7770
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
|
||||
@@ -32,7 +32,10 @@
|
||||
# writes the error body as the artifact), and the declared byte count is a
|
||||
# contract in both directions — a bigger file is a DIFFERENT artifact.
|
||||
|
||||
# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon).
|
||||
# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon). The
|
||||
# caller exports it; shellcheck cannot follow it across the source boundary,
|
||||
# so the read site carries the directive.
|
||||
# shellcheck disable=SC2034
|
||||
: "${GATE_PREFIX:=gate}"
|
||||
|
||||
# Two bash laws make the `return 1` at every call site load-bearing:
|
||||
|
||||
Reference in New Issue
Block a user