refact(deployer): the LLM handlers gate through the shared library
deploy_llm_dragoon/zealot/arkon now call require_image / require_bytes / require_min_mib / require_file / require_dir_files, so the deployer and the serve scripts refuse the same missing payload with the same words — the 2026-10-01 crash-loop law, enforced by construction instead of by duplicated shell. require_exec joins the library for the native-runtime gate (a compiled tree that lost its +x refuses like a missing one). relink_unit takes SYSTEMD_SYSTEM_DIR so a handler can be driven end to end in a sandbox. The library's own refusals now carry their status: a bash function that ends inside an if returns the CONDITION's status, so a gate whose last statement is its refusal printed the message and returned SUCCESS — the new seam caught it, and the gate tests pin it. New seam tests/llm_deploy_gates.bash + tests/stubs/: deploy-remote.sh's LLM handlers EXECUTED against stub systemctl/podman/sudo/ss/curl, asserting exit status, refusal text, no restart, and the pending state marker.
This commit is contained in:
1 parent
cb25796a6b
commit
c5a816e6a8
12 files changed
+403
-97
No files matched your search
Executable
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env bash
|
||||
# deploy_sandbox.bash — the deploy-handler seam's fixture builder.
|
||||
#
|
||||
# The seam (tests/llm_deploy_gates.bats) EXECUTES scripts/deploy-remote.sh's
|
||||
# LLM handlers with every host touchpoint stubbed (tests/stubs/) and every
|
||||
# artifact pointed at a fixture, so the assertions are the handler's exit
|
||||
# status, its refusal text, and what it ASKED systemd/podman to do.
|
||||
#
|
||||
# Usage:
|
||||
# source tests/deploy_sandbox.bash
|
||||
# sandbox_setup
|
||||
# run_handler dankserver frank-archon
|
||||
# sandbox_teardown
|
||||
#
|
||||
# After sandbox_setup the caller owns:
|
||||
# $STAGE the temp root (payload/, bin/, systemd/, state/)
|
||||
# $REMOTE_DIR the sandbox checkout (a git clone; the handler
|
||||
# cds into it, so relative paths resolve there)
|
||||
# $STAGE/systemctl.log every systemctl invocation
|
||||
# $STAGE/podman.log every podman invocation
|
||||
# $STAGE/systemd/<unit>.service the unit link a green deploy leaves behind
|
||||
# Env the caller sets per case: STUB_IMAGE ("" = empty storage),
|
||||
# STUB_MAIN_PID (0 = a stray process owns the port), the LLM_* overrides.
|
||||
|
||||
sandbox_setup() {
|
||||
STAGE="$(mktemp -d)"
|
||||
export HOME="$STAGE/home"
|
||||
mkdir -p "$HOME" "$STAGE/state" "$STAGE/systemd" "$STAGE/bin"
|
||||
|
||||
# Host touchpoints: the seam's stubs, on PATH under their real names.
|
||||
local stub repo_root
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
for stub in "$repo_root"/tests/stubs/*; do
|
||||
ln -s "$stub" "$STAGE/bin/$(basename "$stub")"
|
||||
done
|
||||
export PATH="$STAGE/bin:$PATH"
|
||||
export STUB_SYSTEMCTL_LOG="$STAGE/systemctl.log"
|
||||
export STUB_PODMAN_LOG="$STAGE/podman.log"
|
||||
# A deploy marks state; a sandbox with no git identity would make the
|
||||
# identity, not the handler, the thing under test.
|
||||
export GIT_AUTHOR_NAME=seam GIT_AUTHOR_EMAIL=seam@localhost
|
||||
export GIT_COMMITTER_NAME=seam GIT_COMMITTER_EMAIL=seam@localhost
|
||||
export SYSTEMD_SYSTEM_DIR="$STAGE/systemd"
|
||||
export DEPLOY_STATE_DIR="$STAGE/state"
|
||||
# run_privileged pipes the password from FRANK_SUDO_PW into `sudo -S --`;
|
||||
# the stub drops it and execs, so no privilege gate hides a handler bug.
|
||||
export FRANK_SUDO_PW=test
|
||||
|
||||
# Fixture payload: 3 MiB where a size floor applies, 10 bytes elsewhere.
|
||||
mkdir -p "$STAGE/models/tokenizer"
|
||||
truncate -s 3145728 "$STAGE/models/payload.rad"
|
||||
truncate -s 10 "$STAGE/models/ck.hgn"
|
||||
truncate -s 10 "$STAGE/models/ngram.hgn"
|
||||
truncate -s 10 "$STAGE/models/vision.hgn"
|
||||
printf 'x\n' > "$STAGE/models/llama-server"
|
||||
chmod +x "$STAGE/models/llama-server"
|
||||
truncate -s 10 "$STAGE/models/model.gguf"
|
||||
truncate -s 10 "$STAGE/models/mmproj.gguf"
|
||||
local tf
|
||||
for tf in tokenizer.json chat_template.jinja tokenizer_config.json \
|
||||
merges.txt vocab.json generation_config.json; do
|
||||
printf 'x\n' > "$STAGE/models/tokenizer/$tf"
|
||||
done
|
||||
|
||||
# $HOME/serverAppConfigs is the checkout the handler works in: a git
|
||||
# WORKTREE of the caller's checkout, so the handler runs the EXACT files
|
||||
# under test (the suite lives in worktrees, where uncommitted state is
|
||||
# the norm), and it is a real git repo: the deployer refuses a revision
|
||||
# it cannot resolve. The handler cds into REMOTE_DIR, so every relative
|
||||
# path it uses resolves there.
|
||||
git -C "$repo_root" worktree add -q --detach "$HOME/serverAppConfigs" HEAD
|
||||
export REMOTE_DIR="$HOME/serverAppConfigs"
|
||||
|
||||
# Every endpoint variable is env-overridable, so the handlers see the
|
||||
# fixtures and never the real /home/dank paths.
|
||||
export STUB_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:deadbeef
|
||||
export LLM_DRAGOON_IMAGE="$STUB_IMAGE"
|
||||
export LLM_DRAGOON_CKPT="$STAGE/models/ck.hgn" LLM_DRAGOON_CKPT_BYTES=10
|
||||
export LLM_DRAGOON_NGRAM="$STAGE/models/ngram.hgn" LLM_DRAGOON_NGRAM_BYTES=10
|
||||
export LLM_DRAGOON_VISION="$STAGE/models/vision.hgn" LLM_DRAGOON_VISION_BYTES=10
|
||||
export LLM_DRAGOON_TOK_DIR="$STAGE/models/tokenizer"
|
||||
export LLM_ZEALOT_RUNTIME="$STAGE/models/llama-server"
|
||||
export LLM_ZEALOT_MODEL_FILE="$STAGE/models/model.gguf"
|
||||
export LLM_ZEALOT_MMPROJ="$STAGE/models/mmproj.gguf"
|
||||
export LLM_ARCHON_ENV_FILE="$REMOTE_DIR/scripts/frank-archon.env"
|
||||
# The archon handler reads the served image out of the unit's own env
|
||||
# file, so the stub's storage must hold exactly that ref.
|
||||
ARCHON_IMAGE="$(sed -nE 's/^FRANK_ARCHON_IMAGE=(.+)$/\1/p' "$LLM_ARCHON_ENV_FILE")"
|
||||
}
|
||||
|
||||
sandbox_teardown() {
|
||||
[[ -z "${STAGE:-}" ]] && return 0
|
||||
# A worktree leaves a registration in the caller's .git/worktrees, so
|
||||
# remove it through git, not just the directory.
|
||||
[[ -d "$STAGE/home/serverAppConfigs" ]] &&
|
||||
git -C "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" \
|
||||
worktree remove --force "$STAGE/home/serverAppConfigs"
|
||||
rm -rf "$STAGE"
|
||||
}
|
||||
|
||||
run_handler() {
|
||||
local target="$1" service="$2"
|
||||
run bash "$REMOTE_DIR/scripts/deploy-remote.sh" \
|
||||
"$target" "$(git -C "$REMOTE_DIR" rev-parse HEAD)" "$service"
|
||||
}
|
||||
|
||||
# archon_payload_env — the unit's env file with the payload coordinates
|
||||
# redirected into the sandbox (and the size floor optionally lowered, which
|
||||
# is how a test builds the truncated-.rad case).
|
||||
archon_payload_env() {
|
||||
local re="s#^FRANK_ARCHON_MODEL_DIR=.*#FRANK_ARCHON_MODEL_DIR=$STAGE/models#;"
|
||||
re+=" s#^FRANK_ARCHON_MODEL_FILE=.*#FRANK_ARCHON_MODEL_FILE=payload.rad#"
|
||||
if [[ -n "${1:-}" ]]; then
|
||||
re+="; s#^FRANK_ARCHON_RAD_MIN_MIB=.*#FRANK_ARCHON_RAD_MIN_MIB=$1#"
|
||||
fi
|
||||
sed -E "$re" "$LLM_ARCHON_ENV_FILE" > "$STAGE/archon.env"
|
||||
export LLM_ARCHON_ENV_FILE="$STAGE/archon.env"
|
||||
}
|
||||
|
||||
restarts_requested() {
|
||||
# grep -c prints 0 and exits 1 on no match; the count, not the status,
|
||||
# is what the assertions read.
|
||||
local n=0
|
||||
[[ -f "$STAGE/systemctl.log" ]] && n="$(grep -c 'restart ' "$STAGE/systemctl.log")"
|
||||
printf '%s\n' "$n"
|
||||
}
|
||||
@@ -204,7 +204,7 @@ good_env() {
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||||
FRANK_DRAGOON_NGRAM_TABLE=absent.hgn "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"n-gram-table"*"missing"* ]]
|
||||
[[ "$output" == *"frank-dragoon: n-gram: missing at $TEST_TMP/absent.hgn"* ]]
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||||
FRANK_DRAGOON_NGRAM_BYTES=42 "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
@@ -220,7 +220,7 @@ good_env() {
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||||
FRANK_DRAGOON_VISION_TOWER=absent.hgn "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"vision-tower"*"missing"* ]]
|
||||
[[ "$output" == *"frank-dragoon: vision: missing at $TEST_TMP/absent.hgn"* ]]
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||||
FRANK_DRAGOON_VISION_BYTES=42 "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
|
||||
Executable
+133
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the deployer's LLM handlers are the twin of the serve scripts'
|
||||
# artifact gates (the 2026-10-01 crash-loop: a green deploy against a
|
||||
# missing payload left a restart-looping unit and a dark endpoint).
|
||||
#
|
||||
# These tests EXECUTE scripts/deploy-remote.sh's LLM handlers in a sandbox
|
||||
# (tests/deploy_sandbox.bash): every host touchpoint is a stub, the artifacts
|
||||
# are fixtures, and the assertions are exit status, the refusal on stderr,
|
||||
# and what the handler asked systemd to do. A handler that started the unit
|
||||
# before checking its artifacts would pass a text-grep seam and fail here.
|
||||
|
||||
load helpers
|
||||
load deploy_sandbox
|
||||
|
||||
setup() { sandbox_setup; }
|
||||
teardown() { sandbox_teardown; }
|
||||
|
||||
@test "the dragoon handler refuses when the engine image is not in storage" {
|
||||
export STUB_IMAGE=
|
||||
run_handler dankclaw frank-dragoon
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-dragoon: image:"*"not in podman storage"* ]]
|
||||
[ "$(restarts_requested)" = "0" ]
|
||||
}
|
||||
|
||||
@test "the dragoon handler refuses a truncated checkpoint, naming both sizes" {
|
||||
truncate -s 9 "$STAGE/models/ck.hgn"
|
||||
run_handler dankclaw frank-dragoon
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"checkpoint: is 9 bytes, expected 10"* ]]
|
||||
[ "$(restarts_requested)" = "0" ]
|
||||
}
|
||||
|
||||
@test "the dragoon handler refuses a symlinked tokenizer file" {
|
||||
# The flat-tokenizer law: a HuggingFace cache snapshot links its
|
||||
# members, and the link dangles inside the container mount.
|
||||
rm "$STAGE/models/tokenizer/vocab.json"
|
||||
ln -s /nowhere/vocab.json "$STAGE/models/tokenizer/vocab.json"
|
||||
run_handler dankclaw frank-dragoon
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-dragoon: tokenizer: $STAGE/models/tokenizer/vocab.json is missing or a symlink"* ]]
|
||||
[ "$(restarts_requested)" = "0" ]
|
||||
}
|
||||
|
||||
@test "the dragoon handler starts the unit once every artifact gate passes" {
|
||||
run_handler dankclaw frank-dragoon
|
||||
[ "$status" -eq 0 ]
|
||||
grep -q 'restart frank-dragoon.service' "$STAGE/systemctl.log"
|
||||
grep -qx 'rm -f frank-dragoon frank-dragoon-halogen' "$STAGE/podman.log"
|
||||
}
|
||||
|
||||
@test "the zealot handler refuses when the llama.cpp runtime is not executable" {
|
||||
# The runtime is an operator-built binary; a tree that compiled but lost
|
||||
# its exec bit must refuse exactly like a missing one.
|
||||
rm "$STAGE/models/llama-server"
|
||||
run_handler dankclaw frank-zealot
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-zealot: runtime: missing at $STAGE/models/llama-server"* ]]
|
||||
[[ "$output" == *"cmake"* ]]
|
||||
[ "$(restarts_requested)" = "0" ]
|
||||
}
|
||||
|
||||
@test "the zealot handler refuses a missing projector — a text-only endpoint is not deployable" {
|
||||
# The projector IS the vision capability: deploying without it silently
|
||||
# ships a text-only endpoint.
|
||||
rm "$STAGE/models/mmproj.gguf"
|
||||
run_handler dankclaw frank-zealot
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-zealot: mmproj: missing at $STAGE/models/mmproj.gguf"* ]]
|
||||
[ "$(restarts_requested)" = "0" ]
|
||||
}
|
||||
|
||||
@test "the zealot handler starts the unit once every artifact gate passes" {
|
||||
run_handler dankclaw frank-zealot
|
||||
[ "$status" -eq 0 ]
|
||||
grep -q 'restart frank-zealot.service' "$STAGE/systemctl.log"
|
||||
}
|
||||
|
||||
@test "the archon handler refuses when the radiance image is not in storage" {
|
||||
export STUB_IMAGE=
|
||||
run_handler dankserver frank-archon
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-archon: image:"*"not in podman storage"* ]]
|
||||
[[ "$output" == *"podman pull"* ]]
|
||||
[ "$(restarts_requested)" = "0" ]
|
||||
}
|
||||
|
||||
@test "the archon handler refuses a .rad below the env's size floor" {
|
||||
truncate -s 1048576 "$STAGE/models/payload.rad" # 1 MiB, far under the floor
|
||||
archon_payload_env
|
||||
export STUB_IMAGE="$ARCHON_IMAGE"
|
||||
run_handler dankserver frank-archon
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-archon: .rad: is 1 MiB, expected >= "* ]]
|
||||
[[ "$output" == *"convert-swift15-w4dn8.sh"* ]]
|
||||
[ "$(restarts_requested)" = "0" ]
|
||||
}
|
||||
|
||||
@test "the archon handler clears the legacy container identities and relinks the unit" {
|
||||
# The retired identities (the R9V container, the ad hoc probe, the glm
|
||||
# experiment) hold the same GPUs, so a deploy that leaves one running
|
||||
# leaves the new engine unable to bind. The unit link is the root-unit
|
||||
# shape's whole install.
|
||||
archon_payload_env 2
|
||||
export STUB_IMAGE="$ARCHON_IMAGE"
|
||||
run_handler dankserver frank-archon
|
||||
[ "$status" -eq 0 ]
|
||||
grep -q 'restart frank-archon.service' "$STAGE/systemctl.log"
|
||||
[ -L "$STAGE/systemd/frank-archon.service" ]
|
||||
local ctr
|
||||
for ctr in frank-archon frank-archon-qwen frank-archon-r9v qwen-fp8 glm-probe; do
|
||||
grep -qx "rm -f $ctr" "$STAGE/podman.log"
|
||||
done
|
||||
}
|
||||
|
||||
@test "no LLM handler reports green when its unit has no MainPID behind the port" {
|
||||
# The 2026-09-26 false green: a leftover probe held :7770 and satisfied
|
||||
# the health curl while the real unit was stopped.
|
||||
archon_payload_env 2
|
||||
export STUB_IMAGE="$ARCHON_IMAGE"
|
||||
export STUB_MAIN_PID=0
|
||||
run_handler dankserver frank-archon
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-archon.service has no MainPID"* ]]
|
||||
}
|
||||
|
||||
@test "a refused LLM handler leaves its service pending in the deploy state" {
|
||||
export STUB_IMAGE=
|
||||
run_handler dankserver frank-archon
|
||||
[ "$status" -ne 0 ]
|
||||
grep -q 'status=pending' "$STAGE/state/frank-archon"
|
||||
[ ! -e "$STAGE/state/frank-zealot" ]
|
||||
}
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
# Stub podman for the deploy-handler seam: `image exists` succeeds only for
|
||||
# $STUB_IMAGE (empty = nothing in storage); every other verb is a no-op.
|
||||
# The invocations are logged so a test can read what the handler executed.
|
||||
printf '%s\n' "$*" >> "${STUB_PODMAN_LOG:-/dev/null}"
|
||||
case "$1 $2" in
|
||||
"image exists")
|
||||
[[ -n "${STUB_IMAGE:-}" && "$3" == "$STUB_IMAGE" ]] || exit 1
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# Stub sudo for the deploy-handler seam: run_privileged pipes the password
|
||||
# on stdin and calls `sudo -S -- <cmd>`. The stub drops the password and the
|
||||
# flags and execs the command, so no privilege gate hides a handler bug.
|
||||
cat >/dev/null 2>&1 || true
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
-S|--prompt) shift ;;
|
||||
--) shift; break ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
exec "$@"
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# Stub systemctl for the deploy-handler seam: every invocation is logged, so
|
||||
# a test asserts on what the handler ASKED systemd to do. `is-active` answers
|
||||
# "inactive" (the state a cutover finds the unit in) and MainPID answers
|
||||
# $STUB_MAIN_PID — 0 is the stray-port-holder shape, the false-green gate
|
||||
# every LLM handler reads after a health answer.
|
||||
printf '%s\n' "$*" >> "${STUB_SYSTEMCTL_LOG:-/dev/null}"
|
||||
case "$*" in
|
||||
"is-active "*) echo inactive ;;
|
||||
*"MainPID --value "*) echo "${STUB_MAIN_PID:-4242}" ;;
|
||||
esac
|
||||
exit 0
|
||||
Reference in new issue
Block a user