refact(deployer): one MainPID gate for all three LLM endpoints
frank-dragoon's health gate never checked WHO answered :7770 — the false-green the fleet learned about on 2026-09-26 (a stray probe holding the port satisfies a bare curl while the unit is stopped). The other two handlers carried that check as inlined copies; all three now call assert_unit_owns_port, and dragoon gets the protection it was missing. The seam follows: the endpoints' text-grep deployer tests are deleted, because tests/llm_deploy_gates.bats EXECUTES the same handlers end to end (stub systemctl/podman/sudo/ss/curl) and asserts refusals, restarts, unit links, the pending marker, and the MainPID hardening. GATE_PREFIX is exported at each handler for the same reason the serve scripts export it.
This commit is contained in:
+21
-25
@@ -1166,6 +1166,20 @@ wait_port_free() {
|
||||
ss -tan "sport = :$port" >&2 || true
|
||||
return 1
|
||||
}
|
||||
# assert_unit_owns_port SERVICE PORT — the false-green gate every LLM
|
||||
# endpoint shares. A health answer proves the PORT replied, never WHO
|
||||
# replied: on 2026-09-26 a leftover probe held :7770, satisfied the curl,
|
||||
# and the deploy reported green over a stopped unit. Green requires the
|
||||
# unit's own MainPID to be alive behind the answer.
|
||||
assert_unit_owns_port() {
|
||||
local service="$1" port="$2" main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value "$service.service")"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :$port answered but $service.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "$service: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
deploy_llm_dragoon() {
|
||||
# Refuse rather than restart-loop; every gate here is the SAME library
|
||||
@@ -1173,7 +1187,7 @@ deploy_llm_dragoon() {
|
||||
# 2026-10-01 crash-loop lesson).
|
||||
# Capture podman output before grepping is moot: require_image is one
|
||||
# command, no pipe (the 2026-09-30 SIGPIPE lesson).
|
||||
GATE_PREFIX=frank-dragoon
|
||||
export GATE_PREFIX=frank-dragoon
|
||||
require_image "$LLM_DRAGOON_IMAGE" \
|
||||
"pull it: podman pull docker://${LLM_DRAGOON_IMAGE%@*}" || return 1
|
||||
require_bytes "$LLM_DRAGOON_CKPT" checkpoint "$LLM_DRAGOON_CKPT_BYTES" \
|
||||
@@ -1205,7 +1219,8 @@ deploy_llm_dragoon() {
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-dragoon.service
|
||||
run_privileged systemctl restart frank-dragoon.service
|
||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
|
||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200 || return 1
|
||||
assert_unit_owns_port frank-dragoon 7770
|
||||
}
|
||||
|
||||
# deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on
|
||||
@@ -1231,7 +1246,7 @@ deploy_llm_zealot() {
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why. Same library calls
|
||||
# as serve-frank-zealot.sh (the 2026-10-01 law).
|
||||
GATE_PREFIX=frank-zealot
|
||||
export GATE_PREFIX=frank-zealot
|
||||
require_exec "$LLM_ZEALOT_RUNTIME" runtime \
|
||||
"build it: cmake -B build-cuda -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86" \
|
||||
|| return 1
|
||||
@@ -1245,16 +1260,7 @@ deploy_llm_zealot() {
|
||||
run_privileged systemctl enable frank-zealot.service
|
||||
run_privileged systemctl restart frank-zealot.service
|
||||
wait_for_health frank-zealot http://127.0.0.1:7771/health 900 || return 1
|
||||
# False-green hardening (frank-dragoon's lesson): the port answering
|
||||
# proves nothing about WHO answers — the unit must be active with a
|
||||
# live MainPID, or a stray SGLang holdover is masquerading as deployed.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-zealot.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7771 answered but frank-zealot.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-zealot: health answered by unit MainPID $main_pid"
|
||||
assert_unit_owns_port frank-zealot 7771
|
||||
}
|
||||
|
||||
# deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode,
|
||||
@@ -1285,7 +1291,7 @@ deploy_llm_archon() {
|
||||
# shared library's, called with the values the unit's own env file
|
||||
# declares — so serve-frank-archon.sh and this handler cannot disagree,
|
||||
# and the payload path lives in exactly one file.
|
||||
GATE_PREFIX=frank-archon
|
||||
export GATE_PREFIX=frank-archon
|
||||
local env_file="${LLM_ARCHON_ENV_FILE:-$REMOTE_DIR/scripts/frank-archon.env}"
|
||||
local image model_dir model_file rad_min
|
||||
if [[ ! -f "$env_file" ]]; then
|
||||
@@ -1350,17 +1356,7 @@ deploy_llm_archon() {
|
||||
echo "FAIL: frank-archon /health never answered 200 in 1800s" >&2
|
||||
return 1
|
||||
fi
|
||||
# False-green hardening (the 2026-09-26 lesson): a bare curl against the
|
||||
# port cannot tell WHO answers — a stray probe holding :7770 satisfies
|
||||
# it while the unit is stopped. The deploy is only green if the unit
|
||||
# itself is active with a live MainPID behind the health answer.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-archon.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7770 answered but frank-archon.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-archon: health answered by unit MainPID $main_pid"
|
||||
assert_unit_owns_port frank-archon 7770
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
|
||||
@@ -32,7 +32,10 @@
|
||||
# writes the error body as the artifact), and the declared byte count is a
|
||||
# contract in both directions — a bigger file is a DIFFERENT artifact.
|
||||
|
||||
# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon).
|
||||
# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon). The
|
||||
# caller exports it; shellcheck cannot follow it across the source boundary,
|
||||
# so the read site carries the directive.
|
||||
# shellcheck disable=SC2034
|
||||
: "${GATE_PREFIX:=gate}"
|
||||
|
||||
# Two bash laws make the `return 1` at every call site load-bearing:
|
||||
|
||||
@@ -64,10 +64,13 @@ sandbox_setup() {
|
||||
|
||||
# $HOME/serverAppConfigs is the checkout the handler works in: a git
|
||||
# WORKTREE of the caller's checkout, so the handler runs the EXACT files
|
||||
# under test (the suite lives in worktrees, where uncommitted state is
|
||||
# the norm), and it is a real git repo: the deployer refuses a revision
|
||||
# under test, and it is a real git repo: the deployer refuses a revision
|
||||
# it cannot resolve. The handler cds into REMOTE_DIR, so every relative
|
||||
# path it uses resolves there.
|
||||
#
|
||||
# HEAD is committed state, so the seam asserts on what is in history —
|
||||
# the same rule as the whole suite, and the reason an uncommitted
|
||||
# deployer change is invisible here until it is committed.
|
||||
git -C "$repo_root" worktree add -q --detach "$HOME/serverAppConfigs" HEAD
|
||||
export REMOTE_DIR="$HOME/serverAppConfigs"
|
||||
|
||||
|
||||
@@ -419,36 +419,3 @@ serve_run() {
|
||||
! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
|
||||
! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
|
||||
}
|
||||
|
||||
@test "the deployer gates the image, payload and containers, then health-checks" {
|
||||
# The deployer refuses when the image is absent from dank's rootless
|
||||
# storage or the .rad is missing/too small — then clears stale
|
||||
# containers (including the legacy identities), waits out the port,
|
||||
# relinks/enables/restarts the root unit and health-gates it.
|
||||
grep -q 'LLM_ARCHON_CONTAINER=frank-archon$' "$REMOTE_DEPLOY"
|
||||
grep -q 'FRANK_ARCHON_IMAGE' "$REMOTE_DEPLOY"
|
||||
grep -q 'podman image exists' "$REMOTE_DEPLOY"
|
||||
grep -q 'FRANK_ARCHON_MODEL_FILE' "$REMOTE_DEPLOY"
|
||||
grep -q 'FRANK_ARCHON_RAD_MIN_MIB' "$REMOTE_DEPLOY"
|
||||
grep -q 'frank-archon-qwen' "$REMOTE_DEPLOY"
|
||||
grep -q 'qwen-fp8' "$REMOTE_DEPLOY"
|
||||
grep -q 'glm-probe' "$REMOTE_DEPLOY"
|
||||
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
||||
grep -q 'relink_unit frank-archon' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl enable frank-archon.service' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl restart frank-archon.service' "$REMOTE_DEPLOY"
|
||||
# The health gate is a bare curl: the HTTP listener binds AFTER the
|
||||
# full load (main.cpp — engine.start() then engine.serve()), so during
|
||||
# load the port refuses connections and a bare curl is honest. The
|
||||
# FreeToken-era body read ('status ok') is gone with that engine.
|
||||
grep -q 'curl -fsS http://127.0.0.1:7770/health' "$REMOTE_DEPLOY"
|
||||
! grep -q '"status":"ok"' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "the deployer will not report green on a stray port-holder" {
|
||||
# The 2026-09-26 false green: a leftover probe held :7770 and satisfied
|
||||
# the health curl while the real unit was stopped. The deploy is only
|
||||
# green when the unit's own MainPID is live behind the health answer.
|
||||
grep -q 'systemctl show -p MainPID --value frank-archon.service' "$REMOTE_DEPLOY"
|
||||
grep -q 'stray process holds the port' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@@ -335,11 +335,6 @@ $(cat "$TEST_TMP/serve.log")"
|
||||
grep -qF "LLM_DRAGOON_CKPT_BYTES=\"\${LLM_DRAGOON_CKPT_BYTES:-$bytes}\"" "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the deployer points a missing artifact at the flashnext fetch script" {
|
||||
grep -qE 'download-frank-dragoon-flashnext\.sh' "$DEPLOYER"
|
||||
grep -qE 'podman pull' "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the stray sweep cannot take frank-zealot or the deployer down" {
|
||||
# frank-zealot also listens on this host (7771, llama-server) and the
|
||||
# deployer's own command line contains "frank-dragoon"; only the FULL
|
||||
|
||||
@@ -269,17 +269,6 @@ run_stage() {
|
||||
[[ "$output" == *"build-llama-master-cuda.sh"* ]]
|
||||
}
|
||||
|
||||
@test "the deploy handler gates on binary, GGUF and projector, then MainPID-hardens health" {
|
||||
grep -q 'deploy_llm_zealot' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_ZEALOT_RUNTIME' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_ZEALOT_MODEL_FILE' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_ZEALOT_MMPROJ' "$REMOTE_DEPLOY"
|
||||
grep -q 'relink_unit frank-zealot' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl enable frank-zealot.service' "$REMOTE_DEPLOY"
|
||||
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
|
||||
grep -q 'frank-zealot.service has no MainPID' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "the two endpoints keep separate ports and the large handler never pkills llama-server" {
|
||||
# frank-dragoon's sweeps must not take down frank-zealot, whose
|
||||
# process is llama-server on the SAME host.
|
||||
|
||||
@@ -23,11 +23,14 @@ teardown() { sandbox_teardown; }
|
||||
[ "$(restarts_requested)" = "0" ]
|
||||
}
|
||||
|
||||
@test "the dragoon handler refuses a truncated checkpoint, naming both sizes" {
|
||||
@test "the dragoon handler refuses a truncated checkpoint, pointing at the fetch" {
|
||||
truncate -s 9 "$STAGE/models/ck.hgn"
|
||||
run_handler dankclaw frank-dragoon
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"checkpoint: is 9 bytes, expected 10"* ]]
|
||||
# The refusal is only useful if it names the remedy: the fetch script
|
||||
# re-verifies byte counts and hashes.
|
||||
[[ "$output" == *"download-frank-dragoon-flashnext.sh"* ]]
|
||||
[ "$(restarts_requested)" = "0" ]
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user