From c5a816e6a8cd2ca0542ba47bd00cb990a5550c38 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 13 Oct 2026 21:58:13 -0700 Subject: [PATCH] refact(deployer): the LLM handlers gate through the shared library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deploy_llm_dragoon/zealot/arkon now call require_image / require_bytes / require_min_mib / require_file / require_dir_files, so the deployer and the serve scripts refuse the same missing payload with the same words — the 2026-10-01 crash-loop law, enforced by construction instead of by duplicated shell. require_exec joins the library for the native-runtime gate (a compiled tree that lost its +x refuses like a missing one). relink_unit takes SYSTEMD_SYSTEM_DIR so a handler can be driven end to end in a sandbox. The library's own refusals now carry their status: a bash function that ends inside an if returns the CONDITION's status, so a gate whose last statement is its refusal printed the message and returned SUCCESS — the new seam caught it, and the gate tests pin it. New seam tests/llm_deploy_gates.bash + tests/stubs/: deploy-remote.sh's LLM handlers EXECUTED against stub systemctl/podman/sudo/ss/curl, asserting exit status, refusal text, no restart, and the pending state marker. --- scripts/deploy-remote.sh | 146 ++++++++++++++------------------- scripts/lib/llm-gates.sh | 45 ++++++++-- scripts/serve-frank-dragoon.sh | 4 +- tests/deploy_sandbox.bash | 126 ++++++++++++++++++++++++++++ tests/frank_dragoon_unit.bats | 4 +- tests/llm_deploy_gates.bats | 133 ++++++++++++++++++++++++++++++ tests/stubs/curl | 2 + tests/stubs/podman | 11 +++ tests/stubs/sleep | 2 + tests/stubs/ss | 2 + tests/stubs/sudo | 13 +++ tests/stubs/systemctl | 12 +++ 12 files changed, 403 insertions(+), 97 deletions(-) create mode 100755 tests/deploy_sandbox.bash create mode 100755 tests/llm_deploy_gates.bats create mode 100755 tests/stubs/curl create mode 100755 tests/stubs/podman create mode 100755 tests/stubs/sleep create mode 100755 tests/stubs/ss create mode 100755 tests/stubs/sudo create mode 100755 tests/stubs/systemctl diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 4444b51..0d64784 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -9,6 +9,13 @@ # pending until its service is explicitly deployed. set -euo pipefail +# The LLM handlers' artifact gates are the serve scripts' own library, so +# the deployer and the unit refuse the same missing payload with the same +# words (the fleet law from the 2026-10-01 crash-loop). GATE_PREFIX is set +# per handler. +# shellcheck disable=SC1091 +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/llm-gates.sh" + TARGET="${1:-}" REVISION="${2:-}" shift 2 || true @@ -721,9 +728,15 @@ deploy_gitea_unit() { systemctl --user restart "$service.service" } +# The root units are symlinks from the systemd system directory into the +# checkout. SYSTEMD_SYSTEM_DIR is production (/etc/systemd/system); the knob +# exists so the deploy seam can run a whole handler — relink included — in a +# sandbox and read the unit links as evidence, unprivileged. +SYSTEMD_SYSTEM_DIR="${SYSTEMD_SYSTEM_DIR:-/etc/systemd/system}" + relink_unit() { local service="$1" - run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service" + run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "$SYSTEMD_SYSTEM_DIR/$service.service" } # Authentik runs as three boot-persistent Quadlet units (postgres -> server -> @@ -1155,51 +1168,24 @@ wait_port_free() { } deploy_llm_dragoon() { - # Refuse rather than restart-loop; every gate below has a twin in - # serve-frank-dragoon.sh and they must agree (2026-10-01 lesson). - # Capture podman output before grepping: `podman images | grep -q` - # races under pipefail (SIGPIPE 141 becomes "no match", 2026-09-30). - if ! grep -qF "${LLM_DRAGOON_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then - echo "ERROR: halogen image absent: $LLM_DRAGOON_IMAGE" >&2 - echo "frank-dragoon: pull it: podman pull ${LLM_DRAGOON_IMAGE%@*}" >&2 - return 1 - fi - if [[ ! -f "$LLM_DRAGOON_CKPT" ]]; then - echo "ERROR: halogen checkpoint absent at $LLM_DRAGOON_CKPT" >&2 - echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2 - return 1 - fi - if [[ "$(stat -c %s "$LLM_DRAGOON_CKPT")" != "$LLM_DRAGOON_CKPT_BYTES" ]]; then - echo "ERROR: halogen checkpoint is not $LLM_DRAGOON_CKPT_BYTES bytes (partial download?)" >&2 - echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2 - return 1 - fi - for gate in "$LLM_DRAGOON_NGRAM:$LLM_DRAGOON_NGRAM_BYTES:n-gram table" \ - "$LLM_DRAGOON_VISION:$LLM_DRAGOON_VISION_BYTES:vision tower"; do - IFS=: read -r gpath gbytes gname <<<"$gate" - if [[ ! -f "$gpath" ]]; then - echo "ERROR: halogen $gname absent at $gpath" >&2 - echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2 - return 1 - fi - if [[ "$(stat -c %s "$gpath")" != "$gbytes" ]]; then - echo "ERROR: halogen $gname is not $gbytes bytes (partial download?)" >&2 - echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2 - return 1 - fi - done - if [[ -L "$LLM_DRAGOON_TOK_DIR" || ! -d "$LLM_DRAGOON_TOK_DIR" ]]; then - echo "ERROR: tokenizer dir missing or a symlink at $LLM_DRAGOON_TOK_DIR" >&2 - echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2 - return 1 - fi - for tf in $LLM_DRAGOON_TOK_FILES; do - if [[ ! -f "$LLM_DRAGOON_TOK_DIR/$tf" || -L "$LLM_DRAGOON_TOK_DIR/$tf" ]]; then - echo "ERROR: tokenizer file missing or a symlink: $LLM_DRAGOON_TOK_DIR/$tf" >&2 - echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2 - return 1 - fi - done + # Refuse rather than restart-loop; every gate here is the SAME library + # call the serve script makes, so the two twins cannot drift (the + # 2026-10-01 crash-loop lesson). + # Capture podman output before grepping is moot: require_image is one + # command, no pipe (the 2026-09-30 SIGPIPE lesson). + GATE_PREFIX=frank-dragoon + require_image "$LLM_DRAGOON_IMAGE" \ + "pull it: podman pull docker://${LLM_DRAGOON_IMAGE%@*}" || return 1 + require_bytes "$LLM_DRAGOON_CKPT" checkpoint "$LLM_DRAGOON_CKPT_BYTES" \ + "fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1 + require_bytes "$LLM_DRAGOON_NGRAM" n-gram "$LLM_DRAGOON_NGRAM_BYTES" \ + "fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1 + require_bytes "$LLM_DRAGOON_VISION" vision "$LLM_DRAGOON_VISION_BYTES" \ + "fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1 + # shellcheck disable=SC2086 + require_dir_files "$LLM_DRAGOON_TOK_DIR" tokenizer \ + "fetch it: bash scripts/download-frank-dragoon-flashnext.sh" \ + $LLM_DRAGOON_TOK_FILES || return 1 retire_halogen_dragoon_quadlet # The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops @@ -1243,22 +1229,17 @@ LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K deploy_llm_zealot() { # Refuse rather than restart-loop: a unit whose runtime or payload is # missing never becomes healthy, and a green deploy would leave the - # endpoint dark with nothing in the log to say why. - if [[ ! -x "$LLM_ZEALOT_RUNTIME" ]]; then - echo "ERROR: llama.cpp runtime absent at $LLM_ZEALOT_RUNTIME" >&2 - echo "frank-zealot: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2 - echo "frank-zealot: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2 - echo "frank-zealot: && cmake --build build-cuda --target llama-server) first." >&2 - return 1 - fi - for f in "$LLM_ZEALOT_MODEL_FILE" "$LLM_ZEALOT_MMPROJ"; do - if [[ ! -f "$f" ]]; then - echo "ERROR: frank-zealot payload file absent at $f" >&2 - echo "frank-zealot: run scripts/download-frank-zealot.sh first;" >&2 - echo "frank-zealot: refusing to start a unit that cannot load." >&2 - return 1 - fi - done + # endpoint dark with nothing in the log to say why. Same library calls + # as serve-frank-zealot.sh (the 2026-10-01 law). + GATE_PREFIX=frank-zealot + require_exec "$LLM_ZEALOT_RUNTIME" runtime \ + "build it: cmake -B build-cuda -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86" \ + || return 1 + require_file "$LLM_ZEALOT_MODEL_FILE" gguf \ + "fetch it: bash scripts/download-frank-zealot.sh" || return 1 + require_file "$LLM_ZEALOT_MMPROJ" mmproj \ + "fetch it: bash scripts/download-frank-zealot.sh" || return 1 + relink_unit frank-zealot run_privileged systemctl daemon-reload run_privileged systemctl enable frank-zealot.service @@ -1300,11 +1281,16 @@ LLM_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-pr deploy_llm_archon() { # Refuse rather than restart-loop: a unit whose image or payload is # missing never becomes healthy, and a green deploy would leave the - # endpoint dark with nothing in the log to say why. - local env_file="/home/dank/serverAppConfigs/scripts/frank-archon.env" - local image model_dir model_file rad_min mib + # endpoint dark with nothing in the log to say why. The gates are the + # shared library's, called with the values the unit's own env file + # declares — so serve-frank-archon.sh and this handler cannot disagree, + # and the payload path lives in exactly one file. + GATE_PREFIX=frank-archon + local env_file="${LLM_ARCHON_ENV_FILE:-$REMOTE_DIR/scripts/frank-archon.env}" + local image model_dir model_file rad_min if [[ ! -f "$env_file" ]]; then - echo "ERROR: frank-archon env file absent: $env_file" >&2 + echo "frank-archon: env: missing at $env_file" >&2 + echo " hint: deploy the repo checkout first ($REMOTE_DIR)" >&2 return 1 fi image="$(sed -n 's/^FRANK_ARCHON_IMAGE=//p' "$env_file" | tail -1)" @@ -1312,28 +1298,16 @@ deploy_llm_archon() { model_file="$(sed -n 's/^FRANK_ARCHON_MODEL_FILE=//p' "$env_file" | tail -1)" rad_min="$(sed -n 's/^FRANK_ARCHON_RAD_MIN_MIB=//p' "$env_file" | tail -1)" if [[ -z "$image" || -z "$model_dir" || -z "$model_file" || -z "$rad_min" ]]; then - echo "ERROR: frank-archon.env missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2 - return 1 - fi - # The image gate is the digest ref itself: a name@sha256 ref exists in - # storage only if THAT manifest was pulled. One command — no - # pipefail/grep race (the 2026-09-30 lesson). - if ! podman image exists "$image"; then - echo "ERROR: runtime image absent from storage: $image" >&2 - echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest" >&2 - return 1 - fi - if [[ ! -f "$model_dir/$model_file" ]]; then - echo "ERROR: .rad container missing: $model_dir/$model_file" >&2 - echo "frank-archon: run scripts/build-frank-archon-rad.sh" >&2 - return 1 - fi - mib=$(( $(stat -c %s "$model_dir/$model_file") / 1048576 )) - if [[ "$mib" -lt "$rad_min" ]]; then - echo "ERROR: .rad container too small: $model_dir/$model_file is ${mib} MiB, want >= $rad_min" >&2 - echo "frank-archon: rebuild it (scripts/build-frank-archon-rad.sh --force)" >&2 + echo "frank-archon: env: missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2 + echo " hint: scripts/frank-archon.env" >&2 return 1 fi + require_image "$image" \ + "pull it: podman pull docker://stilldeadcode/radiance:latest (as dank)" \ + || return 1 + require_min_mib "$model_dir/$model_file" ".rad" "$rad_min" \ + "convert it: scripts/convert-swift15-w4dn8.sh (rad-info is the record)" \ + || return 1 # A stale container with the same name makes `podman run --name` fail at # start; remove any leftover from a failed previous attempt. diff --git a/scripts/lib/llm-gates.sh b/scripts/lib/llm-gates.sh index 5b0421f..5f540ac 100755 --- a/scripts/lib/llm-gates.sh +++ b/scripts/lib/llm-gates.sh @@ -8,6 +8,7 @@ # Both sides source this library, so the refusal is one text and one rule. # # Contract (public interface): +# require_exec PATH LABEL HINT # require_file PATH LABEL HINT # require_bytes PATH LABEL WANT_BYTES HINT # require_min_mib PATH LABEL MIN_MIB HINT @@ -20,7 +21,12 @@ # scripts run under `set -e` (so a refusal is terminal), while the deployer # wants `|| return 1` so a failed gate reports a red deploy for that one # service and the dispatcher's state marker stays pending. -# +# Lines are labelled by ARTIFACT CLASS, and the class is the whole label +# (the path rides in the same line): the class + path is what an operator +# acts on, so no wording order and no payload rename can make the two twins +# disagree. The deployer's old "halogen vision tower" and the serve script's +# "vision-tower" were one gate in two vocabularies that no grep could join. + # The size law is equality, not a floor, for `require_bytes`: a hub fetch # that truncates is the fleet's standing hazard (curl exits 0 on 401/403 and # writes the error body as the artifact), and the declared byte count is a @@ -29,6 +35,14 @@ # GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon). : "${GATE_PREFIX:=gate}" +# Two bash laws make the `return 1` at every call site load-bearing: +# * a function whose body ends inside an `if` returns the CONDITION's +# status (true = 0), so a gate that ends on its own refusal prints the +# refusal and reports SUCCESS; +# * `set -e` is suspended for any command in a `… || …` chain, so a gate +# that runs inside one must not be trusted to abort the caller. +# Every gate's refusal therefore carries an explicit `|| return 1`, and the +# caller consumes the status (`|| exit 1`, `|| return 1`). _gate_fail() { printf '%s: %s\n' "$GATE_PREFIX" "$1" >&2 printf ' hint: %s\n' "$2" >&2 @@ -41,8 +55,21 @@ _gate_fail() { require_file() { local path="$1" label="$2" hint="$3" if [[ -L "$path" || ! -f "$path" ]]; then - _gate_fail "$label: missing at $path" "$hint" + _gate_fail "$label: missing at $path" "$hint" || return 1 fi + return 0 +} + +# require_exec — PATH must be an executable regular file. The native runtimes +# (llama.cpp's llama-server) are binaries the operator builds; a tree that +# compiled but lost its +x, or a directory wearing the binary's name, must +# refuse the same way a missing one does. +require_exec() { + local path="$1" label="$2" hint="$3" + if [[ -L "$path" || ! -x "$path" ]]; then + _gate_fail "$label: missing at $path" "$hint" || return 1 + fi + return 0 } # require_bytes — PATH must exist AND be exactly WANT_BYTES bytes. @@ -51,8 +78,9 @@ require_bytes() { require_file "$path" "$label" "$hint" || return 1 got="$(stat -c %s "$path")" if [[ "$got" != "$want" ]]; then - _gate_fail "$label: is $got bytes, expected $want" "$hint" + _gate_fail "$label: is $got bytes, expected $want" "$hint" || return 1 fi + return 0 } # require_min_mib — PATH must be at least MIN_MIB mebibytes. The floor form @@ -63,8 +91,9 @@ require_min_mib() { require_file "$path" "$label" "$hint" || return 1 mib=$(( $(stat -c %s "$path") / 1048576 )) if (( mib < min )); then - _gate_fail "$label: is $mib MiB, expected >= $min MiB" "$hint" + _gate_fail "$label: is $mib MiB, expected >= $min MiB" "$hint" || return 1 fi + return 0 } # require_dir_files — DIR must be a directory and every named member a @@ -77,13 +106,14 @@ require_dir_files() { shift 3 local f if [[ -L "$dir" || ! -d "$dir" ]]; then - _gate_fail "$label: missing at $dir" "$hint" + _gate_fail "$label: missing at $dir" "$hint" || return 1 fi for f in "$@"; do if [[ -L "$dir/$f" || ! -f "$dir/$f" ]]; then - _gate_fail "$label: $dir/$f is missing or a symlink" "$hint" + _gate_fail "$label: $dir/$f is missing or a symlink" "$hint" || return 1 fi done + return 0 } # require_image — the image ref must be in podman storage. A name@sha256 ref @@ -92,6 +122,7 @@ require_dir_files() { require_image() { local ref="$1" hint="$2" if ! podman image exists "$ref"; then - _gate_fail "image: $ref is not in podman storage" "$hint" + _gate_fail "image: $ref is not in podman storage" "$hint" || return 1 fi + return 0 } diff --git a/scripts/serve-frank-dragoon.sh b/scripts/serve-frank-dragoon.sh index 332e8e2..9dabc27 100755 --- a/scripts/serve-frank-dragoon.sh +++ b/scripts/serve-frank-dragoon.sh @@ -79,10 +79,10 @@ require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_CHECKPOINT" \ checkpoint "$FRANK_DRAGOON_CHECKPOINT_BYTES" \ "fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1 require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_NGRAM_TABLE" \ - n-gram-table "$FRANK_DRAGOON_NGRAM_BYTES" \ + n-gram "$FRANK_DRAGOON_NGRAM_BYTES" \ "fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1 require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_VISION_TOWER" \ - vision-tower "$FRANK_DRAGOON_VISION_BYTES" \ + vision "$FRANK_DRAGOON_VISION_BYTES" \ "fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1 # The payload ships the tokenizer as a FLAT DIRECTORY. A HuggingFace cache diff --git a/tests/deploy_sandbox.bash b/tests/deploy_sandbox.bash new file mode 100755 index 0000000..e83a79d --- /dev/null +++ b/tests/deploy_sandbox.bash @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# deploy_sandbox.bash — the deploy-handler seam's fixture builder. +# +# The seam (tests/llm_deploy_gates.bats) EXECUTES scripts/deploy-remote.sh's +# LLM handlers with every host touchpoint stubbed (tests/stubs/) and every +# artifact pointed at a fixture, so the assertions are the handler's exit +# status, its refusal text, and what it ASKED systemd/podman to do. +# +# Usage: +# source tests/deploy_sandbox.bash +# sandbox_setup +# run_handler dankserver frank-archon +# sandbox_teardown +# +# After sandbox_setup the caller owns: +# $STAGE the temp root (payload/, bin/, systemd/, state/) +# $REMOTE_DIR the sandbox checkout (a git clone; the handler +# cds into it, so relative paths resolve there) +# $STAGE/systemctl.log every systemctl invocation +# $STAGE/podman.log every podman invocation +# $STAGE/systemd/.service the unit link a green deploy leaves behind +# Env the caller sets per case: STUB_IMAGE ("" = empty storage), +# STUB_MAIN_PID (0 = a stray process owns the port), the LLM_* overrides. + +sandbox_setup() { + STAGE="$(mktemp -d)" + export HOME="$STAGE/home" + mkdir -p "$HOME" "$STAGE/state" "$STAGE/systemd" "$STAGE/bin" + + # Host touchpoints: the seam's stubs, on PATH under their real names. + local stub repo_root + repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + for stub in "$repo_root"/tests/stubs/*; do + ln -s "$stub" "$STAGE/bin/$(basename "$stub")" + done + export PATH="$STAGE/bin:$PATH" + export STUB_SYSTEMCTL_LOG="$STAGE/systemctl.log" + export STUB_PODMAN_LOG="$STAGE/podman.log" + # A deploy marks state; a sandbox with no git identity would make the + # identity, not the handler, the thing under test. + export GIT_AUTHOR_NAME=seam GIT_AUTHOR_EMAIL=seam@localhost + export GIT_COMMITTER_NAME=seam GIT_COMMITTER_EMAIL=seam@localhost + export SYSTEMD_SYSTEM_DIR="$STAGE/systemd" + export DEPLOY_STATE_DIR="$STAGE/state" + # run_privileged pipes the password from FRANK_SUDO_PW into `sudo -S --`; + # the stub drops it and execs, so no privilege gate hides a handler bug. + export FRANK_SUDO_PW=test + + # Fixture payload: 3 MiB where a size floor applies, 10 bytes elsewhere. + mkdir -p "$STAGE/models/tokenizer" + truncate -s 3145728 "$STAGE/models/payload.rad" + truncate -s 10 "$STAGE/models/ck.hgn" + truncate -s 10 "$STAGE/models/ngram.hgn" + truncate -s 10 "$STAGE/models/vision.hgn" + printf 'x\n' > "$STAGE/models/llama-server" + chmod +x "$STAGE/models/llama-server" + truncate -s 10 "$STAGE/models/model.gguf" + truncate -s 10 "$STAGE/models/mmproj.gguf" + local tf + for tf in tokenizer.json chat_template.jinja tokenizer_config.json \ + merges.txt vocab.json generation_config.json; do + printf 'x\n' > "$STAGE/models/tokenizer/$tf" + done + + # $HOME/serverAppConfigs is the checkout the handler works in: a git + # WORKTREE of the caller's checkout, so the handler runs the EXACT files + # under test (the suite lives in worktrees, where uncommitted state is + # the norm), and it is a real git repo: the deployer refuses a revision + # it cannot resolve. The handler cds into REMOTE_DIR, so every relative + # path it uses resolves there. + git -C "$repo_root" worktree add -q --detach "$HOME/serverAppConfigs" HEAD + export REMOTE_DIR="$HOME/serverAppConfigs" + + # Every endpoint variable is env-overridable, so the handlers see the + # fixtures and never the real /home/dank paths. + export STUB_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:deadbeef + export LLM_DRAGOON_IMAGE="$STUB_IMAGE" + export LLM_DRAGOON_CKPT="$STAGE/models/ck.hgn" LLM_DRAGOON_CKPT_BYTES=10 + export LLM_DRAGOON_NGRAM="$STAGE/models/ngram.hgn" LLM_DRAGOON_NGRAM_BYTES=10 + export LLM_DRAGOON_VISION="$STAGE/models/vision.hgn" LLM_DRAGOON_VISION_BYTES=10 + export LLM_DRAGOON_TOK_DIR="$STAGE/models/tokenizer" + export LLM_ZEALOT_RUNTIME="$STAGE/models/llama-server" + export LLM_ZEALOT_MODEL_FILE="$STAGE/models/model.gguf" + export LLM_ZEALOT_MMPROJ="$STAGE/models/mmproj.gguf" + export LLM_ARCHON_ENV_FILE="$REMOTE_DIR/scripts/frank-archon.env" + # The archon handler reads the served image out of the unit's own env + # file, so the stub's storage must hold exactly that ref. + ARCHON_IMAGE="$(sed -nE 's/^FRANK_ARCHON_IMAGE=(.+)$/\1/p' "$LLM_ARCHON_ENV_FILE")" +} + +sandbox_teardown() { + [[ -z "${STAGE:-}" ]] && return 0 + # A worktree leaves a registration in the caller's .git/worktrees, so + # remove it through git, not just the directory. + [[ -d "$STAGE/home/serverAppConfigs" ]] && + git -C "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" \ + worktree remove --force "$STAGE/home/serverAppConfigs" + rm -rf "$STAGE" +} + +run_handler() { + local target="$1" service="$2" + run bash "$REMOTE_DIR/scripts/deploy-remote.sh" \ + "$target" "$(git -C "$REMOTE_DIR" rev-parse HEAD)" "$service" +} + +# archon_payload_env — the unit's env file with the payload coordinates +# redirected into the sandbox (and the size floor optionally lowered, which +# is how a test builds the truncated-.rad case). +archon_payload_env() { + local re="s#^FRANK_ARCHON_MODEL_DIR=.*#FRANK_ARCHON_MODEL_DIR=$STAGE/models#;" + re+=" s#^FRANK_ARCHON_MODEL_FILE=.*#FRANK_ARCHON_MODEL_FILE=payload.rad#" + if [[ -n "${1:-}" ]]; then + re+="; s#^FRANK_ARCHON_RAD_MIN_MIB=.*#FRANK_ARCHON_RAD_MIN_MIB=$1#" + fi + sed -E "$re" "$LLM_ARCHON_ENV_FILE" > "$STAGE/archon.env" + export LLM_ARCHON_ENV_FILE="$STAGE/archon.env" +} + +restarts_requested() { + # grep -c prints 0 and exits 1 on no match; the count, not the status, + # is what the assertions read. + local n=0 + [[ -f "$STAGE/systemctl.log" ]] && n="$(grep -c 'restart ' "$STAGE/systemctl.log")" + printf '%s\n' "$n" +} diff --git a/tests/frank_dragoon_unit.bats b/tests/frank_dragoon_unit.bats index cbe76cf..79daae4 100755 --- a/tests/frank_dragoon_unit.bats +++ b/tests/frank_dragoon_unit.bats @@ -204,7 +204,7 @@ good_env() { run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ FRANK_DRAGOON_NGRAM_TABLE=absent.hgn "$SERVE" [ "$status" -ne 0 ] - [[ "$output" == *"n-gram-table"*"missing"* ]] + [[ "$output" == *"frank-dragoon: n-gram: missing at $TEST_TMP/absent.hgn"* ]] run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ FRANK_DRAGOON_NGRAM_BYTES=42 "$SERVE" [ "$status" -ne 0 ] @@ -220,7 +220,7 @@ good_env() { run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ FRANK_DRAGOON_VISION_TOWER=absent.hgn "$SERVE" [ "$status" -ne 0 ] - [[ "$output" == *"vision-tower"*"missing"* ]] + [[ "$output" == *"frank-dragoon: vision: missing at $TEST_TMP/absent.hgn"* ]] run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ FRANK_DRAGOON_VISION_BYTES=42 "$SERVE" [ "$status" -ne 0 ] diff --git a/tests/llm_deploy_gates.bats b/tests/llm_deploy_gates.bats new file mode 100755 index 0000000..7d0a5b2 --- /dev/null +++ b/tests/llm_deploy_gates.bats @@ -0,0 +1,133 @@ +#!/usr/bin/env bats +# Seam: the deployer's LLM handlers are the twin of the serve scripts' +# artifact gates (the 2026-10-01 crash-loop: a green deploy against a +# missing payload left a restart-looping unit and a dark endpoint). +# +# These tests EXECUTE scripts/deploy-remote.sh's LLM handlers in a sandbox +# (tests/deploy_sandbox.bash): every host touchpoint is a stub, the artifacts +# are fixtures, and the assertions are exit status, the refusal on stderr, +# and what the handler asked systemd to do. A handler that started the unit +# before checking its artifacts would pass a text-grep seam and fail here. + +load helpers +load deploy_sandbox + +setup() { sandbox_setup; } +teardown() { sandbox_teardown; } + +@test "the dragoon handler refuses when the engine image is not in storage" { + export STUB_IMAGE= + run_handler dankclaw frank-dragoon + [ "$status" -ne 0 ] + [[ "$output" == *"frank-dragoon: image:"*"not in podman storage"* ]] + [ "$(restarts_requested)" = "0" ] +} + +@test "the dragoon handler refuses a truncated checkpoint, naming both sizes" { + truncate -s 9 "$STAGE/models/ck.hgn" + run_handler dankclaw frank-dragoon + [ "$status" -ne 0 ] + [[ "$output" == *"checkpoint: is 9 bytes, expected 10"* ]] + [ "$(restarts_requested)" = "0" ] +} + +@test "the dragoon handler refuses a symlinked tokenizer file" { + # The flat-tokenizer law: a HuggingFace cache snapshot links its + # members, and the link dangles inside the container mount. + rm "$STAGE/models/tokenizer/vocab.json" + ln -s /nowhere/vocab.json "$STAGE/models/tokenizer/vocab.json" + run_handler dankclaw frank-dragoon + [ "$status" -ne 0 ] + [[ "$output" == *"frank-dragoon: tokenizer: $STAGE/models/tokenizer/vocab.json is missing or a symlink"* ]] + [ "$(restarts_requested)" = "0" ] +} + +@test "the dragoon handler starts the unit once every artifact gate passes" { + run_handler dankclaw frank-dragoon + [ "$status" -eq 0 ] + grep -q 'restart frank-dragoon.service' "$STAGE/systemctl.log" + grep -qx 'rm -f frank-dragoon frank-dragoon-halogen' "$STAGE/podman.log" +} + +@test "the zealot handler refuses when the llama.cpp runtime is not executable" { + # The runtime is an operator-built binary; a tree that compiled but lost + # its exec bit must refuse exactly like a missing one. + rm "$STAGE/models/llama-server" + run_handler dankclaw frank-zealot + [ "$status" -ne 0 ] + [[ "$output" == *"frank-zealot: runtime: missing at $STAGE/models/llama-server"* ]] + [[ "$output" == *"cmake"* ]] + [ "$(restarts_requested)" = "0" ] +} + +@test "the zealot handler refuses a missing projector — a text-only endpoint is not deployable" { + # The projector IS the vision capability: deploying without it silently + # ships a text-only endpoint. + rm "$STAGE/models/mmproj.gguf" + run_handler dankclaw frank-zealot + [ "$status" -ne 0 ] + [[ "$output" == *"frank-zealot: mmproj: missing at $STAGE/models/mmproj.gguf"* ]] + [ "$(restarts_requested)" = "0" ] +} + +@test "the zealot handler starts the unit once every artifact gate passes" { + run_handler dankclaw frank-zealot + [ "$status" -eq 0 ] + grep -q 'restart frank-zealot.service' "$STAGE/systemctl.log" +} + +@test "the archon handler refuses when the radiance image is not in storage" { + export STUB_IMAGE= + run_handler dankserver frank-archon + [ "$status" -ne 0 ] + [[ "$output" == *"frank-archon: image:"*"not in podman storage"* ]] + [[ "$output" == *"podman pull"* ]] + [ "$(restarts_requested)" = "0" ] +} + +@test "the archon handler refuses a .rad below the env's size floor" { + truncate -s 1048576 "$STAGE/models/payload.rad" # 1 MiB, far under the floor + archon_payload_env + export STUB_IMAGE="$ARCHON_IMAGE" + run_handler dankserver frank-archon + [ "$status" -ne 0 ] + [[ "$output" == *"frank-archon: .rad: is 1 MiB, expected >= "* ]] + [[ "$output" == *"convert-swift15-w4dn8.sh"* ]] + [ "$(restarts_requested)" = "0" ] +} + +@test "the archon handler clears the legacy container identities and relinks the unit" { + # The retired identities (the R9V container, the ad hoc probe, the glm + # experiment) hold the same GPUs, so a deploy that leaves one running + # leaves the new engine unable to bind. The unit link is the root-unit + # shape's whole install. + archon_payload_env 2 + export STUB_IMAGE="$ARCHON_IMAGE" + run_handler dankserver frank-archon + [ "$status" -eq 0 ] + grep -q 'restart frank-archon.service' "$STAGE/systemctl.log" + [ -L "$STAGE/systemd/frank-archon.service" ] + local ctr + for ctr in frank-archon frank-archon-qwen frank-archon-r9v qwen-fp8 glm-probe; do + grep -qx "rm -f $ctr" "$STAGE/podman.log" + done +} + +@test "no LLM handler reports green when its unit has no MainPID behind the port" { + # The 2026-09-26 false green: a leftover probe held :7770 and satisfied + # the health curl while the real unit was stopped. + archon_payload_env 2 + export STUB_IMAGE="$ARCHON_IMAGE" + export STUB_MAIN_PID=0 + run_handler dankserver frank-archon + [ "$status" -ne 0 ] + [[ "$output" == *"frank-archon.service has no MainPID"* ]] +} + +@test "a refused LLM handler leaves its service pending in the deploy state" { + export STUB_IMAGE= + run_handler dankserver frank-archon + [ "$status" -ne 0 ] + grep -q 'status=pending' "$STAGE/state/frank-archon" + [ ! -e "$STAGE/state/frank-zealot" ] +} diff --git a/tests/stubs/curl b/tests/stubs/curl new file mode 100755 index 0000000..742e13d --- /dev/null +++ b/tests/stubs/curl @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exit 0 diff --git a/tests/stubs/podman b/tests/stubs/podman new file mode 100755 index 0000000..1764811 --- /dev/null +++ b/tests/stubs/podman @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Stub podman for the deploy-handler seam: `image exists` succeeds only for +# $STUB_IMAGE (empty = nothing in storage); every other verb is a no-op. +# The invocations are logged so a test can read what the handler executed. +printf '%s\n' "$*" >> "${STUB_PODMAN_LOG:-/dev/null}" +case "$1 $2" in + "image exists") + [[ -n "${STUB_IMAGE:-}" && "$3" == "$STUB_IMAGE" ]] || exit 1 + ;; +esac +exit 0 diff --git a/tests/stubs/sleep b/tests/stubs/sleep new file mode 100755 index 0000000..742e13d --- /dev/null +++ b/tests/stubs/sleep @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exit 0 diff --git a/tests/stubs/ss b/tests/stubs/ss new file mode 100755 index 0000000..742e13d --- /dev/null +++ b/tests/stubs/ss @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exit 0 diff --git a/tests/stubs/sudo b/tests/stubs/sudo new file mode 100755 index 0000000..d31031b --- /dev/null +++ b/tests/stubs/sudo @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Stub sudo for the deploy-handler seam: run_privileged pipes the password +# on stdin and calls `sudo -S -- `. The stub drops the password and the +# flags and execs the command, so no privilege gate hides a handler bug. +cat >/dev/null 2>&1 || true +while [ "$#" -gt 0 ]; do + case "$1" in + -S|--prompt) shift ;; + --) shift; break ;; + *) break ;; + esac +done +exec "$@" diff --git a/tests/stubs/systemctl b/tests/stubs/systemctl new file mode 100755 index 0000000..12c74b8 --- /dev/null +++ b/tests/stubs/systemctl @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Stub systemctl for the deploy-handler seam: every invocation is logged, so +# a test asserts on what the handler ASKED systemd to do. `is-active` answers +# "inactive" (the state a cutover finds the unit in) and MainPID answers +# $STUB_MAIN_PID — 0 is the stray-port-holder shape, the false-green gate +# every LLM handler reads after a health answer. +printf '%s\n' "$*" >> "${STUB_SYSTEMCTL_LOG:-/dev/null}" +case "$*" in + "is-active "*) echo inactive ;; + *"MainPID --value "*) echo "${STUB_MAIN_PID:-4242}" ;; +esac +exit 0