refact(deployer): one MainPID gate for all three LLM endpoints

frank-dragoon's health gate never checked WHO answered :7770 — the
false-green the fleet learned about on 2026-09-26 (a stray probe holding
the port satisfies a bare curl while the unit is stopped). The other two
handlers carried that check as inlined copies; all three now call
assert_unit_owns_port, and dragoon gets the protection it was missing.

The seam follows: the endpoints' text-grep deployer tests are deleted,
because tests/llm_deploy_gates.bats EXECUTES the same handlers end to end
(stub systemctl/podman/sudo/ss/curl) and asserts refusals, restarts, unit
links, the pending marker, and the MainPID hardening. GATE_PREFIX is
exported at each handler for the same reason the serve scripts export it.
This commit is contained in:
Frank
2026-10-13 22:02:48 -07:00
parent c5a816e6a8
commit ab80c1366d
7 changed files with 34 additions and 78 deletions
+21 -25
View File
@@ -1166,6 +1166,20 @@ wait_port_free() {
ss -tan "sport = :$port" >&2 || true
return 1
}
# assert_unit_owns_port SERVICE PORT — the false-green gate every LLM
# endpoint shares. A health answer proves the PORT replied, never WHO
# replied: on 2026-09-26 a leftover probe held :7770, satisfied the curl,
# and the deploy reported green over a stopped unit. Green requires the
# unit's own MainPID to be alive behind the answer.
assert_unit_owns_port() {
local service="$1" port="$2" main_pid
main_pid="$(run_privileged systemctl show -p MainPID --value "$service.service")"
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
echo "FAIL: :$port answered but $service.service has no MainPID — a stray process holds the port" >&2
return 1
fi
echo "$service: health answered by unit MainPID $main_pid"
}
deploy_llm_dragoon() {
# Refuse rather than restart-loop; every gate here is the SAME library
@@ -1173,7 +1187,7 @@ deploy_llm_dragoon() {
# 2026-10-01 crash-loop lesson).
# Capture podman output before grepping is moot: require_image is one
# command, no pipe (the 2026-09-30 SIGPIPE lesson).
GATE_PREFIX=frank-dragoon
export GATE_PREFIX=frank-dragoon
require_image "$LLM_DRAGOON_IMAGE" \
"pull it: podman pull docker://${LLM_DRAGOON_IMAGE%@*}" || return 1
require_bytes "$LLM_DRAGOON_CKPT" checkpoint "$LLM_DRAGOON_CKPT_BYTES" \
@@ -1205,7 +1219,8 @@ deploy_llm_dragoon() {
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-dragoon.service
run_privileged systemctl restart frank-dragoon.service
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200 || return 1
assert_unit_owns_port frank-dragoon 7770
}
# deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on
@@ -1231,7 +1246,7 @@ deploy_llm_zealot() {
# missing never becomes healthy, and a green deploy would leave the
# endpoint dark with nothing in the log to say why. Same library calls
# as serve-frank-zealot.sh (the 2026-10-01 law).
GATE_PREFIX=frank-zealot
export GATE_PREFIX=frank-zealot
require_exec "$LLM_ZEALOT_RUNTIME" runtime \
"build it: cmake -B build-cuda -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86" \
|| return 1
@@ -1245,16 +1260,7 @@ deploy_llm_zealot() {
run_privileged systemctl enable frank-zealot.service
run_privileged systemctl restart frank-zealot.service
wait_for_health frank-zealot http://127.0.0.1:7771/health 900 || return 1
# False-green hardening (frank-dragoon's lesson): the port answering
# proves nothing about WHO answers — the unit must be active with a
# live MainPID, or a stray SGLang holdover is masquerading as deployed.
local main_pid
main_pid="$(run_privileged systemctl show -p MainPID --value frank-zealot.service)"
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
echo "FAIL: :7771 answered but frank-zealot.service has no MainPID — a stray process holds the port" >&2
return 1
fi
echo "frank-zealot: health answered by unit MainPID $main_pid"
assert_unit_owns_port frank-zealot 7771
}
# deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode,
@@ -1285,7 +1291,7 @@ deploy_llm_archon() {
# shared library's, called with the values the unit's own env file
# declares — so serve-frank-archon.sh and this handler cannot disagree,
# and the payload path lives in exactly one file.
GATE_PREFIX=frank-archon
export GATE_PREFIX=frank-archon
local env_file="${LLM_ARCHON_ENV_FILE:-$REMOTE_DIR/scripts/frank-archon.env}"
local image model_dir model_file rad_min
if [[ ! -f "$env_file" ]]; then
@@ -1350,17 +1356,7 @@ deploy_llm_archon() {
echo "FAIL: frank-archon /health never answered 200 in 1800s" >&2
return 1
fi
# False-green hardening (the 2026-09-26 lesson): a bare curl against the
# port cannot tell WHO answers — a stray probe holding :7770 satisfies
# it while the unit is stopped. The deploy is only green if the unit
# itself is active with a live MainPID behind the health answer.
local main_pid
main_pid="$(run_privileged systemctl show -p MainPID --value frank-archon.service)"
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
echo "FAIL: :7770 answered but frank-archon.service has no MainPID — a stray process holds the port" >&2
return 1
fi
echo "frank-archon: health answered by unit MainPID $main_pid"
assert_unit_owns_port frank-archon 7770
}
wait_for_health() {
+4 -1
View File
@@ -32,7 +32,10 @@
# writes the error body as the artifact), and the declared byte count is a
# contract in both directions — a bigger file is a DIFFERENT artifact.
# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon).
# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon). The
# caller exports it; shellcheck cannot follow it across the source boundary,
# so the read site carries the directive.
# shellcheck disable=SC2034
: "${GATE_PREFIX:=gate}"
# Two bash laws make the `return 1` at every call site load-bearing:
+5 -2
View File
@@ -64,10 +64,13 @@ sandbox_setup() {
# $HOME/serverAppConfigs is the checkout the handler works in: a git
# WORKTREE of the caller's checkout, so the handler runs the EXACT files
# under test (the suite lives in worktrees, where uncommitted state is
# the norm), and it is a real git repo: the deployer refuses a revision
# under test, and it is a real git repo: the deployer refuses a revision
# it cannot resolve. The handler cds into REMOTE_DIR, so every relative
# path it uses resolves there.
#
# HEAD is committed state, so the seam asserts on what is in history —
# the same rule as the whole suite, and the reason an uncommitted
# deployer change is invisible here until it is committed.
git -C "$repo_root" worktree add -q --detach "$HOME/serverAppConfigs" HEAD
export REMOTE_DIR="$HOME/serverAppConfigs"
-33
View File
@@ -419,36 +419,3 @@ serve_run() {
! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
}
@test "the deployer gates the image, payload and containers, then health-checks" {
# The deployer refuses when the image is absent from dank's rootless
# storage or the .rad is missing/too small — then clears stale
# containers (including the legacy identities), waits out the port,
# relinks/enables/restarts the root unit and health-gates it.
grep -q 'LLM_ARCHON_CONTAINER=frank-archon$' "$REMOTE_DEPLOY"
grep -q 'FRANK_ARCHON_IMAGE' "$REMOTE_DEPLOY"
grep -q 'podman image exists' "$REMOTE_DEPLOY"
grep -q 'FRANK_ARCHON_MODEL_FILE' "$REMOTE_DEPLOY"
grep -q 'FRANK_ARCHON_RAD_MIN_MIB' "$REMOTE_DEPLOY"
grep -q 'frank-archon-qwen' "$REMOTE_DEPLOY"
grep -q 'qwen-fp8' "$REMOTE_DEPLOY"
grep -q 'glm-probe' "$REMOTE_DEPLOY"
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
grep -q 'relink_unit frank-archon' "$REMOTE_DEPLOY"
grep -q 'systemctl enable frank-archon.service' "$REMOTE_DEPLOY"
grep -q 'systemctl restart frank-archon.service' "$REMOTE_DEPLOY"
# The health gate is a bare curl: the HTTP listener binds AFTER the
# full load (main.cpp — engine.start() then engine.serve()), so during
# load the port refuses connections and a bare curl is honest. The
# FreeToken-era body read ('status ok') is gone with that engine.
grep -q 'curl -fsS http://127.0.0.1:7770/health' "$REMOTE_DEPLOY"
! grep -q '"status":"ok"' "$REMOTE_DEPLOY"
}
@test "the deployer will not report green on a stray port-holder" {
# The 2026-09-26 false green: a leftover probe held :7770 and satisfied
# the health curl while the real unit was stopped. The deploy is only
# green when the unit's own MainPID is live behind the health answer.
grep -q 'systemctl show -p MainPID --value frank-archon.service' "$REMOTE_DEPLOY"
grep -q 'stray process holds the port' "$REMOTE_DEPLOY"
}
-5
View File
@@ -335,11 +335,6 @@ $(cat "$TEST_TMP/serve.log")"
grep -qF "LLM_DRAGOON_CKPT_BYTES=\"\${LLM_DRAGOON_CKPT_BYTES:-$bytes}\"" "$DEPLOYER"
}
@test "the deployer points a missing artifact at the flashnext fetch script" {
grep -qE 'download-frank-dragoon-flashnext\.sh' "$DEPLOYER"
grep -qE 'podman pull' "$DEPLOYER"
}
@test "the stray sweep cannot take frank-zealot or the deployer down" {
# frank-zealot also listens on this host (7771, llama-server) and the
# deployer's own command line contains "frank-dragoon"; only the FULL
-11
View File
@@ -269,17 +269,6 @@ run_stage() {
[[ "$output" == *"build-llama-master-cuda.sh"* ]]
}
@test "the deploy handler gates on binary, GGUF and projector, then MainPID-hardens health" {
grep -q 'deploy_llm_zealot' "$REMOTE_DEPLOY"
grep -q 'LLM_ZEALOT_RUNTIME' "$REMOTE_DEPLOY"
grep -q 'LLM_ZEALOT_MODEL_FILE' "$REMOTE_DEPLOY"
grep -q 'LLM_ZEALOT_MMPROJ' "$REMOTE_DEPLOY"
grep -q 'relink_unit frank-zealot' "$REMOTE_DEPLOY"
grep -q 'systemctl enable frank-zealot.service' "$REMOTE_DEPLOY"
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
grep -q 'frank-zealot.service has no MainPID' "$REMOTE_DEPLOY"
}
@test "the two endpoints keep separate ports and the large handler never pkills llama-server" {
# frank-dragoon's sweeps must not take down frank-zealot, whose
# process is llama-server on the SAME host.
+4 -1
View File
@@ -23,11 +23,14 @@ teardown() { sandbox_teardown; }
[ "$(restarts_requested)" = "0" ]
}
@test "the dragoon handler refuses a truncated checkpoint, naming both sizes" {
@test "the dragoon handler refuses a truncated checkpoint, pointing at the fetch" {
truncate -s 9 "$STAGE/models/ck.hgn"
run_handler dankclaw frank-dragoon
[ "$status" -ne 0 ]
[[ "$output" == *"checkpoint: is 9 bytes, expected 10"* ]]
# The refusal is only useful if it names the remedy: the fetch script
# re-verifies byte counts and hashes.
[[ "$output" == *"download-frank-dragoon-flashnext.sh"* ]]
[ "$(restarts_requested)" = "0" ]
}