refact(deployer): the LLM handlers gate through the shared library

deploy_llm_dragoon/zealot/arkon now call require_image / require_bytes /
require_min_mib / require_file / require_dir_files, so the deployer and the
serve scripts refuse the same missing payload with the same words — the
2026-10-01 crash-loop law, enforced by construction instead of by duplicated
shell. require_exec joins the library for the native-runtime gate (a
compiled tree that lost its +x refuses like a missing one). relink_unit
takes SYSTEMD_SYSTEM_DIR so a handler can be driven end to end in a sandbox.

The library's own refusals now carry their status: a bash function that
ends inside an if returns the CONDITION's status, so a gate whose last
statement is its refusal printed the message and returned SUCCESS — the
new seam caught it, and the gate tests pin it.

New seam tests/llm_deploy_gates.bash + tests/stubs/: deploy-remote.sh's LLM
handlers EXECUTED against stub systemctl/podman/sudo/ss/curl, asserting
exit status, refusal text, no restart, and the pending state marker.
This commit is contained in:
Frank committed 2026-10-13 21:58:13 -07:00
1 parent cb25796a6b
commit c5a816e6a8
12 files changed
+403 -97

No files matched your search

+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env bash
# deploy_sandbox.bash — the deploy-handler seam's fixture builder.
#
# The seam (tests/llm_deploy_gates.bats) EXECUTES scripts/deploy-remote.sh's
# LLM handlers with every host touchpoint stubbed (tests/stubs/) and every
# artifact pointed at a fixture, so the assertions are the handler's exit
# status, its refusal text, and what it ASKED systemd/podman to do.
#
# Usage:
# source tests/deploy_sandbox.bash
# sandbox_setup
# run_handler dankserver frank-archon
# sandbox_teardown
#
# After sandbox_setup the caller owns:
# $STAGE the temp root (payload/, bin/, systemd/, state/)
# $REMOTE_DIR the sandbox checkout (a git clone; the handler
# cds into it, so relative paths resolve there)
# $STAGE/systemctl.log every systemctl invocation
# $STAGE/podman.log every podman invocation
# $STAGE/systemd/<unit>.service the unit link a green deploy leaves behind
# Env the caller sets per case: STUB_IMAGE ("" = empty storage),
# STUB_MAIN_PID (0 = a stray process owns the port), the LLM_* overrides.
sandbox_setup() {
STAGE="$(mktemp -d)"
export HOME="$STAGE/home"
mkdir -p "$HOME" "$STAGE/state" "$STAGE/systemd" "$STAGE/bin"
# Host touchpoints: the seam's stubs, on PATH under their real names.
local stub repo_root
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
for stub in "$repo_root"/tests/stubs/*; do
ln -s "$stub" "$STAGE/bin/$(basename "$stub")"
done
export PATH="$STAGE/bin:$PATH"
export STUB_SYSTEMCTL_LOG="$STAGE/systemctl.log"
export STUB_PODMAN_LOG="$STAGE/podman.log"
# A deploy marks state; a sandbox with no git identity would make the
# identity, not the handler, the thing under test.
export GIT_AUTHOR_NAME=seam GIT_AUTHOR_EMAIL=seam@localhost
export GIT_COMMITTER_NAME=seam GIT_COMMITTER_EMAIL=seam@localhost
export SYSTEMD_SYSTEM_DIR="$STAGE/systemd"
export DEPLOY_STATE_DIR="$STAGE/state"
# run_privileged pipes the password from FRANK_SUDO_PW into `sudo -S --`;
# the stub drops it and execs, so no privilege gate hides a handler bug.
export FRANK_SUDO_PW=test
# Fixture payload: 3 MiB where a size floor applies, 10 bytes elsewhere.
mkdir -p "$STAGE/models/tokenizer"
truncate -s 3145728 "$STAGE/models/payload.rad"
truncate -s 10 "$STAGE/models/ck.hgn"
truncate -s 10 "$STAGE/models/ngram.hgn"
truncate -s 10 "$STAGE/models/vision.hgn"
printf 'x\n' > "$STAGE/models/llama-server"
chmod +x "$STAGE/models/llama-server"
truncate -s 10 "$STAGE/models/model.gguf"
truncate -s 10 "$STAGE/models/mmproj.gguf"
local tf
for tf in tokenizer.json chat_template.jinja tokenizer_config.json \
merges.txt vocab.json generation_config.json; do
printf 'x\n' > "$STAGE/models/tokenizer/$tf"
done
# $HOME/serverAppConfigs is the checkout the handler works in: a git
# WORKTREE of the caller's checkout, so the handler runs the EXACT files
# under test (the suite lives in worktrees, where uncommitted state is
# the norm), and it is a real git repo: the deployer refuses a revision
# it cannot resolve. The handler cds into REMOTE_DIR, so every relative
# path it uses resolves there.
git -C "$repo_root" worktree add -q --detach "$HOME/serverAppConfigs" HEAD
export REMOTE_DIR="$HOME/serverAppConfigs"
# Every endpoint variable is env-overridable, so the handlers see the
# fixtures and never the real /home/dank paths.
export STUB_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:deadbeef
export LLM_DRAGOON_IMAGE="$STUB_IMAGE"
export LLM_DRAGOON_CKPT="$STAGE/models/ck.hgn" LLM_DRAGOON_CKPT_BYTES=10
export LLM_DRAGOON_NGRAM="$STAGE/models/ngram.hgn" LLM_DRAGOON_NGRAM_BYTES=10
export LLM_DRAGOON_VISION="$STAGE/models/vision.hgn" LLM_DRAGOON_VISION_BYTES=10
export LLM_DRAGOON_TOK_DIR="$STAGE/models/tokenizer"
export LLM_ZEALOT_RUNTIME="$STAGE/models/llama-server"
export LLM_ZEALOT_MODEL_FILE="$STAGE/models/model.gguf"
export LLM_ZEALOT_MMPROJ="$STAGE/models/mmproj.gguf"
export LLM_ARCHON_ENV_FILE="$REMOTE_DIR/scripts/frank-archon.env"
# The archon handler reads the served image out of the unit's own env
# file, so the stub's storage must hold exactly that ref.
ARCHON_IMAGE="$(sed -nE 's/^FRANK_ARCHON_IMAGE=(.+)$/\1/p' "$LLM_ARCHON_ENV_FILE")"
}
sandbox_teardown() {
[[ -z "${STAGE:-}" ]] && return 0
# A worktree leaves a registration in the caller's .git/worktrees, so
# remove it through git, not just the directory.
[[ -d "$STAGE/home/serverAppConfigs" ]] &&
git -C "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" \
worktree remove --force "$STAGE/home/serverAppConfigs"
rm -rf "$STAGE"
}
run_handler() {
local target="$1" service="$2"
run bash "$REMOTE_DIR/scripts/deploy-remote.sh" \
"$target" "$(git -C "$REMOTE_DIR" rev-parse HEAD)" "$service"
}
# archon_payload_env — the unit's env file with the payload coordinates
# redirected into the sandbox (and the size floor optionally lowered, which
# is how a test builds the truncated-.rad case).
archon_payload_env() {
local re="s#^FRANK_ARCHON_MODEL_DIR=.*#FRANK_ARCHON_MODEL_DIR=$STAGE/models#;"
re+=" s#^FRANK_ARCHON_MODEL_FILE=.*#FRANK_ARCHON_MODEL_FILE=payload.rad#"
if [[ -n "${1:-}" ]]; then
re+="; s#^FRANK_ARCHON_RAD_MIN_MIB=.*#FRANK_ARCHON_RAD_MIN_MIB=$1#"
fi
sed -E "$re" "$LLM_ARCHON_ENV_FILE" > "$STAGE/archon.env"
export LLM_ARCHON_ENV_FILE="$STAGE/archon.env"
}
restarts_requested() {
# grep -c prints 0 and exits 1 on no match; the count, not the status,
# is what the assertions read.
local n=0
[[ -f "$STAGE/systemctl.log" ]] && n="$(grep -c 'restart ' "$STAGE/systemctl.log")"
printf '%s\n' "$n"
}
+2 -2
View File
@@ -204,7 +204,7 @@ good_env() {
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_DRAGOON_NGRAM_TABLE=absent.hgn "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"n-gram-table"*"missing"* ]]
[[ "$output" == *"frank-dragoon: n-gram: missing at $TEST_TMP/absent.hgn"* ]]
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_DRAGOON_NGRAM_BYTES=42 "$SERVE"
[ "$status" -ne 0 ]
@@ -220,7 +220,7 @@ good_env() {
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_DRAGOON_VISION_TOWER=absent.hgn "$SERVE"
[ "$status" -ne 0 ]
[[ "$output" == *"vision-tower"*"missing"* ]]
[[ "$output" == *"frank-dragoon: vision: missing at $TEST_TMP/absent.hgn"* ]]
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
FRANK_DRAGOON_VISION_BYTES=42 "$SERVE"
[ "$status" -ne 0 ]
+133
View File
@@ -0,0 +1,133 @@
#!/usr/bin/env bats
# Seam: the deployer's LLM handlers are the twin of the serve scripts'
# artifact gates (the 2026-10-01 crash-loop: a green deploy against a
# missing payload left a restart-looping unit and a dark endpoint).
#
# These tests EXECUTE scripts/deploy-remote.sh's LLM handlers in a sandbox
# (tests/deploy_sandbox.bash): every host touchpoint is a stub, the artifacts
# are fixtures, and the assertions are exit status, the refusal on stderr,
# and what the handler asked systemd to do. A handler that started the unit
# before checking its artifacts would pass a text-grep seam and fail here.
load helpers
load deploy_sandbox
setup() { sandbox_setup; }
teardown() { sandbox_teardown; }
@test "the dragoon handler refuses when the engine image is not in storage" {
export STUB_IMAGE=
run_handler dankclaw frank-dragoon
[ "$status" -ne 0 ]
[[ "$output" == *"frank-dragoon: image:"*"not in podman storage"* ]]
[ "$(restarts_requested)" = "0" ]
}
@test "the dragoon handler refuses a truncated checkpoint, naming both sizes" {
truncate -s 9 "$STAGE/models/ck.hgn"
run_handler dankclaw frank-dragoon
[ "$status" -ne 0 ]
[[ "$output" == *"checkpoint: is 9 bytes, expected 10"* ]]
[ "$(restarts_requested)" = "0" ]
}
@test "the dragoon handler refuses a symlinked tokenizer file" {
# The flat-tokenizer law: a HuggingFace cache snapshot links its
# members, and the link dangles inside the container mount.
rm "$STAGE/models/tokenizer/vocab.json"
ln -s /nowhere/vocab.json "$STAGE/models/tokenizer/vocab.json"
run_handler dankclaw frank-dragoon
[ "$status" -ne 0 ]
[[ "$output" == *"frank-dragoon: tokenizer: $STAGE/models/tokenizer/vocab.json is missing or a symlink"* ]]
[ "$(restarts_requested)" = "0" ]
}
@test "the dragoon handler starts the unit once every artifact gate passes" {
run_handler dankclaw frank-dragoon
[ "$status" -eq 0 ]
grep -q 'restart frank-dragoon.service' "$STAGE/systemctl.log"
grep -qx 'rm -f frank-dragoon frank-dragoon-halogen' "$STAGE/podman.log"
}
@test "the zealot handler refuses when the llama.cpp runtime is not executable" {
# The runtime is an operator-built binary; a tree that compiled but lost
# its exec bit must refuse exactly like a missing one.
rm "$STAGE/models/llama-server"
run_handler dankclaw frank-zealot
[ "$status" -ne 0 ]
[[ "$output" == *"frank-zealot: runtime: missing at $STAGE/models/llama-server"* ]]
[[ "$output" == *"cmake"* ]]
[ "$(restarts_requested)" = "0" ]
}
@test "the zealot handler refuses a missing projector — a text-only endpoint is not deployable" {
# The projector IS the vision capability: deploying without it silently
# ships a text-only endpoint.
rm "$STAGE/models/mmproj.gguf"
run_handler dankclaw frank-zealot
[ "$status" -ne 0 ]
[[ "$output" == *"frank-zealot: mmproj: missing at $STAGE/models/mmproj.gguf"* ]]
[ "$(restarts_requested)" = "0" ]
}
@test "the zealot handler starts the unit once every artifact gate passes" {
run_handler dankclaw frank-zealot
[ "$status" -eq 0 ]
grep -q 'restart frank-zealot.service' "$STAGE/systemctl.log"
}
@test "the archon handler refuses when the radiance image is not in storage" {
export STUB_IMAGE=
run_handler dankserver frank-archon
[ "$status" -ne 0 ]
[[ "$output" == *"frank-archon: image:"*"not in podman storage"* ]]
[[ "$output" == *"podman pull"* ]]
[ "$(restarts_requested)" = "0" ]
}
@test "the archon handler refuses a .rad below the env's size floor" {
truncate -s 1048576 "$STAGE/models/payload.rad" # 1 MiB, far under the floor
archon_payload_env
export STUB_IMAGE="$ARCHON_IMAGE"
run_handler dankserver frank-archon
[ "$status" -ne 0 ]
[[ "$output" == *"frank-archon: .rad: is 1 MiB, expected >= "* ]]
[[ "$output" == *"convert-swift15-w4dn8.sh"* ]]
[ "$(restarts_requested)" = "0" ]
}
@test "the archon handler clears the legacy container identities and relinks the unit" {
# The retired identities (the R9V container, the ad hoc probe, the glm
# experiment) hold the same GPUs, so a deploy that leaves one running
# leaves the new engine unable to bind. The unit link is the root-unit
# shape's whole install.
archon_payload_env 2
export STUB_IMAGE="$ARCHON_IMAGE"
run_handler dankserver frank-archon
[ "$status" -eq 0 ]
grep -q 'restart frank-archon.service' "$STAGE/systemctl.log"
[ -L "$STAGE/systemd/frank-archon.service" ]
local ctr
for ctr in frank-archon frank-archon-qwen frank-archon-r9v qwen-fp8 glm-probe; do
grep -qx "rm -f $ctr" "$STAGE/podman.log"
done
}
@test "no LLM handler reports green when its unit has no MainPID behind the port" {
# The 2026-09-26 false green: a leftover probe held :7770 and satisfied
# the health curl while the real unit was stopped.
archon_payload_env 2
export STUB_IMAGE="$ARCHON_IMAGE"
export STUB_MAIN_PID=0
run_handler dankserver frank-archon
[ "$status" -ne 0 ]
[[ "$output" == *"frank-archon.service has no MainPID"* ]]
}
@test "a refused LLM handler leaves its service pending in the deploy state" {
export STUB_IMAGE=
run_handler dankserver frank-archon
[ "$status" -ne 0 ]
grep -q 'status=pending' "$STAGE/state/frank-archon"
[ ! -e "$STAGE/state/frank-zealot" ]
}
+2
View File
@@ -0,0 +1,2 @@
#!/usr/bin/env bash
exit 0
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
# Stub podman for the deploy-handler seam: `image exists` succeeds only for
# $STUB_IMAGE (empty = nothing in storage); every other verb is a no-op.
# The invocations are logged so a test can read what the handler executed.
printf '%s\n' "$*" >> "${STUB_PODMAN_LOG:-/dev/null}"
case "$1 $2" in
"image exists")
[[ -n "${STUB_IMAGE:-}" && "$3" == "$STUB_IMAGE" ]] || exit 1
;;
esac
exit 0
+2
View File
@@ -0,0 +1,2 @@
#!/usr/bin/env bash
exit 0
Executable
+2
View File
@@ -0,0 +1,2 @@
#!/usr/bin/env bash
exit 0
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env bash
# Stub sudo for the deploy-handler seam: run_privileged pipes the password
# on stdin and calls `sudo -S -- <cmd>`. The stub drops the password and the
# flags and execs the command, so no privilege gate hides a handler bug.
cat >/dev/null 2>&1 || true
while [ "$#" -gt 0 ]; do
case "$1" in
-S|--prompt) shift ;;
--) shift; break ;;
*) break ;;
esac
done
exec "$@"
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
# Stub systemctl for the deploy-handler seam: every invocation is logged, so
# a test asserts on what the handler ASKED systemd to do. `is-active` answers
# "inactive" (the state a cutover finds the unit in) and MainPID answers
# $STUB_MAIN_PID — 0 is the stray-port-holder shape, the false-green gate
# every LLM handler reads after a health answer.
printf '%s\n' "$*" >> "${STUB_SYSTEMCTL_LOG:-/dev/null}"
case "$*" in
"is-active "*) echo inactive ;;
*"MainPID --value "*) echo "${STUB_MAIN_PID:-4242}" ;;
esac
exit 0