From ab80c1366ddfb05cb874ff4af5f670ffc2368583 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 13 Oct 2026 22:02:48 -0700 Subject: [PATCH] refact(deployer): one MainPID gate for all three LLM endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit frank-dragoon's health gate never checked WHO answered :7770 — the false-green the fleet learned about on 2026-09-26 (a stray probe holding the port satisfies a bare curl while the unit is stopped). The other two handlers carried that check as inlined copies; all three now call assert_unit_owns_port, and dragoon gets the protection it was missing. The seam follows: the endpoints' text-grep deployer tests are deleted, because tests/llm_deploy_gates.bats EXECUTES the same handlers end to end (stub systemctl/podman/sudo/ss/curl) and asserts refusals, restarts, unit links, the pending marker, and the MainPID hardening. GATE_PREFIX is exported at each handler for the same reason the serve scripts export it. --- scripts/deploy-remote.sh | 46 ++++++++++++++++------------------- scripts/lib/llm-gates.sh | 5 +++- tests/deploy_sandbox.bash | 7 ++++-- tests/frank_archon_unit.bats | 33 ------------------------- tests/frank_dragoon_unit.bats | 5 ---- tests/frank_zealot_unit.bats | 11 --------- tests/llm_deploy_gates.bats | 5 +++- 7 files changed, 34 insertions(+), 78 deletions(-) diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 0d64784..c148dde 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1166,6 +1166,20 @@ wait_port_free() { ss -tan "sport = :$port" >&2 || true return 1 } +# assert_unit_owns_port SERVICE PORT — the false-green gate every LLM +# endpoint shares. A health answer proves the PORT replied, never WHO +# replied: on 2026-09-26 a leftover probe held :7770, satisfied the curl, +# and the deploy reported green over a stopped unit. Green requires the +# unit's own MainPID to be alive behind the answer. +assert_unit_owns_port() { + local service="$1" port="$2" main_pid + main_pid="$(run_privileged systemctl show -p MainPID --value "$service.service")" + if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then + echo "FAIL: :$port answered but $service.service has no MainPID — a stray process holds the port" >&2 + return 1 + fi + echo "$service: health answered by unit MainPID $main_pid" +} deploy_llm_dragoon() { # Refuse rather than restart-loop; every gate here is the SAME library @@ -1173,7 +1187,7 @@ deploy_llm_dragoon() { # 2026-10-01 crash-loop lesson). # Capture podman output before grepping is moot: require_image is one # command, no pipe (the 2026-09-30 SIGPIPE lesson). - GATE_PREFIX=frank-dragoon + export GATE_PREFIX=frank-dragoon require_image "$LLM_DRAGOON_IMAGE" \ "pull it: podman pull docker://${LLM_DRAGOON_IMAGE%@*}" || return 1 require_bytes "$LLM_DRAGOON_CKPT" checkpoint "$LLM_DRAGOON_CKPT_BYTES" \ @@ -1205,7 +1219,8 @@ deploy_llm_dragoon() { run_privileged systemctl daemon-reload run_privileged systemctl enable frank-dragoon.service run_privileged systemctl restart frank-dragoon.service - wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200 + wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200 || return 1 + assert_unit_owns_port frank-dragoon 7770 } # deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on @@ -1231,7 +1246,7 @@ deploy_llm_zealot() { # missing never becomes healthy, and a green deploy would leave the # endpoint dark with nothing in the log to say why. Same library calls # as serve-frank-zealot.sh (the 2026-10-01 law). - GATE_PREFIX=frank-zealot + export GATE_PREFIX=frank-zealot require_exec "$LLM_ZEALOT_RUNTIME" runtime \ "build it: cmake -B build-cuda -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86" \ || return 1 @@ -1245,16 +1260,7 @@ deploy_llm_zealot() { run_privileged systemctl enable frank-zealot.service run_privileged systemctl restart frank-zealot.service wait_for_health frank-zealot http://127.0.0.1:7771/health 900 || return 1 - # False-green hardening (frank-dragoon's lesson): the port answering - # proves nothing about WHO answers — the unit must be active with a - # live MainPID, or a stray SGLang holdover is masquerading as deployed. - local main_pid - main_pid="$(run_privileged systemctl show -p MainPID --value frank-zealot.service)" - if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then - echo "FAIL: :7771 answered but frank-zealot.service has no MainPID — a stray process holds the port" >&2 - return 1 - fi - echo "frank-zealot: health answered by unit MainPID $main_pid" + assert_unit_owns_port frank-zealot 7771 } # deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode, @@ -1285,7 +1291,7 @@ deploy_llm_archon() { # shared library's, called with the values the unit's own env file # declares — so serve-frank-archon.sh and this handler cannot disagree, # and the payload path lives in exactly one file. - GATE_PREFIX=frank-archon + export GATE_PREFIX=frank-archon local env_file="${LLM_ARCHON_ENV_FILE:-$REMOTE_DIR/scripts/frank-archon.env}" local image model_dir model_file rad_min if [[ ! -f "$env_file" ]]; then @@ -1350,17 +1356,7 @@ deploy_llm_archon() { echo "FAIL: frank-archon /health never answered 200 in 1800s" >&2 return 1 fi - # False-green hardening (the 2026-09-26 lesson): a bare curl against the - # port cannot tell WHO answers — a stray probe holding :7770 satisfies - # it while the unit is stopped. The deploy is only green if the unit - # itself is active with a live MainPID behind the health answer. - local main_pid - main_pid="$(run_privileged systemctl show -p MainPID --value frank-archon.service)" - if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then - echo "FAIL: :7770 answered but frank-archon.service has no MainPID — a stray process holds the port" >&2 - return 1 - fi - echo "frank-archon: health answered by unit MainPID $main_pid" + assert_unit_owns_port frank-archon 7770 } wait_for_health() { diff --git a/scripts/lib/llm-gates.sh b/scripts/lib/llm-gates.sh index 5f540ac..ed7827c 100755 --- a/scripts/lib/llm-gates.sh +++ b/scripts/lib/llm-gates.sh @@ -32,7 +32,10 @@ # writes the error body as the artifact), and the declared byte count is a # contract in both directions — a bigger file is a DIFFERENT artifact. -# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon). +# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon). The +# caller exports it; shellcheck cannot follow it across the source boundary, +# so the read site carries the directive. +# shellcheck disable=SC2034 : "${GATE_PREFIX:=gate}" # Two bash laws make the `return 1` at every call site load-bearing: diff --git a/tests/deploy_sandbox.bash b/tests/deploy_sandbox.bash index e83a79d..60acbba 100755 --- a/tests/deploy_sandbox.bash +++ b/tests/deploy_sandbox.bash @@ -64,10 +64,13 @@ sandbox_setup() { # $HOME/serverAppConfigs is the checkout the handler works in: a git # WORKTREE of the caller's checkout, so the handler runs the EXACT files - # under test (the suite lives in worktrees, where uncommitted state is - # the norm), and it is a real git repo: the deployer refuses a revision + # under test, and it is a real git repo: the deployer refuses a revision # it cannot resolve. The handler cds into REMOTE_DIR, so every relative # path it uses resolves there. + # + # HEAD is committed state, so the seam asserts on what is in history — + # the same rule as the whole suite, and the reason an uncommitted + # deployer change is invisible here until it is committed. git -C "$repo_root" worktree add -q --detach "$HOME/serverAppConfigs" HEAD export REMOTE_DIR="$HOME/serverAppConfigs" diff --git a/tests/frank_archon_unit.bats b/tests/frank_archon_unit.bats index 02a4feb..7de937a 100755 --- a/tests/frank_archon_unit.bats +++ b/tests/frank_archon_unit.bats @@ -419,36 +419,3 @@ serve_run() { ! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE" ! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE" } - -@test "the deployer gates the image, payload and containers, then health-checks" { - # The deployer refuses when the image is absent from dank's rootless - # storage or the .rad is missing/too small — then clears stale - # containers (including the legacy identities), waits out the port, - # relinks/enables/restarts the root unit and health-gates it. - grep -q 'LLM_ARCHON_CONTAINER=frank-archon$' "$REMOTE_DEPLOY" - grep -q 'FRANK_ARCHON_IMAGE' "$REMOTE_DEPLOY" - grep -q 'podman image exists' "$REMOTE_DEPLOY" - grep -q 'FRANK_ARCHON_MODEL_FILE' "$REMOTE_DEPLOY" - grep -q 'FRANK_ARCHON_RAD_MIN_MIB' "$REMOTE_DEPLOY" - grep -q 'frank-archon-qwen' "$REMOTE_DEPLOY" - grep -q 'qwen-fp8' "$REMOTE_DEPLOY" - grep -q 'glm-probe' "$REMOTE_DEPLOY" - grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" - grep -q 'relink_unit frank-archon' "$REMOTE_DEPLOY" - grep -q 'systemctl enable frank-archon.service' "$REMOTE_DEPLOY" - grep -q 'systemctl restart frank-archon.service' "$REMOTE_DEPLOY" - # The health gate is a bare curl: the HTTP listener binds AFTER the - # full load (main.cpp — engine.start() then engine.serve()), so during - # load the port refuses connections and a bare curl is honest. The - # FreeToken-era body read ('status ok') is gone with that engine. - grep -q 'curl -fsS http://127.0.0.1:7770/health' "$REMOTE_DEPLOY" - ! grep -q '"status":"ok"' "$REMOTE_DEPLOY" -} - -@test "the deployer will not report green on a stray port-holder" { - # The 2026-09-26 false green: a leftover probe held :7770 and satisfied - # the health curl while the real unit was stopped. The deploy is only - # green when the unit's own MainPID is live behind the health answer. - grep -q 'systemctl show -p MainPID --value frank-archon.service' "$REMOTE_DEPLOY" - grep -q 'stray process holds the port' "$REMOTE_DEPLOY" -} diff --git a/tests/frank_dragoon_unit.bats b/tests/frank_dragoon_unit.bats index 79daae4..c207ef9 100755 --- a/tests/frank_dragoon_unit.bats +++ b/tests/frank_dragoon_unit.bats @@ -335,11 +335,6 @@ $(cat "$TEST_TMP/serve.log")" grep -qF "LLM_DRAGOON_CKPT_BYTES=\"\${LLM_DRAGOON_CKPT_BYTES:-$bytes}\"" "$DEPLOYER" } -@test "the deployer points a missing artifact at the flashnext fetch script" { - grep -qE 'download-frank-dragoon-flashnext\.sh' "$DEPLOYER" - grep -qE 'podman pull' "$DEPLOYER" -} - @test "the stray sweep cannot take frank-zealot or the deployer down" { # frank-zealot also listens on this host (7771, llama-server) and the # deployer's own command line contains "frank-dragoon"; only the FULL diff --git a/tests/frank_zealot_unit.bats b/tests/frank_zealot_unit.bats index f03fc3a..c7a8782 100755 --- a/tests/frank_zealot_unit.bats +++ b/tests/frank_zealot_unit.bats @@ -269,17 +269,6 @@ run_stage() { [[ "$output" == *"build-llama-master-cuda.sh"* ]] } -@test "the deploy handler gates on binary, GGUF and projector, then MainPID-hardens health" { - grep -q 'deploy_llm_zealot' "$REMOTE_DEPLOY" - grep -q 'LLM_ZEALOT_RUNTIME' "$REMOTE_DEPLOY" - grep -q 'LLM_ZEALOT_MODEL_FILE' "$REMOTE_DEPLOY" - grep -q 'LLM_ZEALOT_MMPROJ' "$REMOTE_DEPLOY" - grep -q 'relink_unit frank-zealot' "$REMOTE_DEPLOY" - grep -q 'systemctl enable frank-zealot.service' "$REMOTE_DEPLOY" - grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY" - grep -q 'frank-zealot.service has no MainPID' "$REMOTE_DEPLOY" -} - @test "the two endpoints keep separate ports and the large handler never pkills llama-server" { # frank-dragoon's sweeps must not take down frank-zealot, whose # process is llama-server on the SAME host. diff --git a/tests/llm_deploy_gates.bats b/tests/llm_deploy_gates.bats index 7d0a5b2..75426f1 100755 --- a/tests/llm_deploy_gates.bats +++ b/tests/llm_deploy_gates.bats @@ -23,11 +23,14 @@ teardown() { sandbox_teardown; } [ "$(restarts_requested)" = "0" ] } -@test "the dragoon handler refuses a truncated checkpoint, naming both sizes" { +@test "the dragoon handler refuses a truncated checkpoint, pointing at the fetch" { truncate -s 9 "$STAGE/models/ck.hgn" run_handler dankclaw frank-dragoon [ "$status" -ne 0 ] [[ "$output" == *"checkpoint: is 9 bytes, expected 10"* ]] + # The refusal is only useful if it names the remedy: the fetch script + # re-verifies byte counts and hashes. + [[ "$output" == *"download-frank-dragoon-flashnext.sh"* ]] [ "$(restarts_requested)" = "0" ] }