refact(deployer): the LLM handlers gate through the shared library

deploy_llm_dragoon/zealot/arkon now call require_image / require_bytes / require_min_mib / require_file / require_dir_files, so the deployer and the serve scripts refuse the same missing payload with the same words — the 2026-10-01 crash-loop law, enforced by construction instead of by duplicated shell. require_exec joins the library for the native-runtime gate (a compiled tree that lost its +x refuses like a missing one). relink_unit takes SYSTEMD_SYSTEM_DIR so a handler can be driven end to end in a sandbox. The library's own refusals now carry their status: a bash function that ends inside an if returns the CONDITION's status, so a gate whose last statement is its refusal printed the message and returned SUCCESS — the new seam caught it, and the gate tests pin it. New seam tests/llm_deploy_gates.bash + tests/stubs/: deploy-remote.sh's LLM handlers EXECUTED against stub systemctl/podman/sudo/ss/curl, asserting exit status, refusal text, no restart, and the pending state marker.
2026-10-13 21:58:13 -07:00
parent cb25796a6b
commit c5a816e6a8
12 changed files with 403 additions and 97 deletions
@@ -9,6 +9,13 @@
 # pending until its service is explicitly deployed.
 set -euo pipefail

+# The LLM handlers' artifact gates are the serve scripts' own library, so
+# the deployer and the unit refuse the same missing payload with the same
+# words (the fleet law from the 2026-10-01 crash-loop). GATE_PREFIX is set
+# per handler.
+# shellcheck disable=SC1091
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/llm-gates.sh"
+
 TARGET="${1:-}"
 REVISION="${2:-}"
 shift 2 || true
@@ -721,9 +728,15 @@ deploy_gitea_unit() {
    systemctl --user restart "$service.service"
 }

+# The root units are symlinks from the systemd system directory into the
+# checkout. SYSTEMD_SYSTEM_DIR is production (/etc/systemd/system); the knob
+# exists so the deploy seam can run a whole handler — relink included — in a
+# sandbox and read the unit links as evidence, unprivileged.
+SYSTEMD_SYSTEM_DIR="${SYSTEMD_SYSTEM_DIR:-/etc/systemd/system}"
+
 relink_unit() {
    local service="$1"
-    run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service"
+    run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "$SYSTEMD_SYSTEM_DIR/$service.service"
 }

 # Authentik runs as three boot-persistent Quadlet units (postgres -> server ->
@@ -1155,51 +1168,24 @@ wait_port_free() {
 }

 deploy_llm_dragoon() {
-    # Refuse rather than restart-loop; every gate below has a twin in
-    # serve-frank-dragoon.sh and they must agree (2026-10-01 lesson).
-    # Capture podman output before grepping: `podman images | grep -q`
-    # races under pipefail (SIGPIPE 141 becomes "no match", 2026-09-30).
-    if ! grep -qF "${LLM_DRAGOON_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then
-        echo "ERROR: halogen image absent: $LLM_DRAGOON_IMAGE" >&2
-        echo "frank-dragoon: pull it: podman pull ${LLM_DRAGOON_IMAGE%@*}" >&2
-        return 1
-    fi
-    if [[ ! -f "$LLM_DRAGOON_CKPT" ]]; then
-        echo "ERROR: halogen checkpoint absent at $LLM_DRAGOON_CKPT" >&2
-        echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
-        return 1
-    fi
-    if [[ "$(stat -c %s "$LLM_DRAGOON_CKPT")" != "$LLM_DRAGOON_CKPT_BYTES" ]]; then
-        echo "ERROR: halogen checkpoint is not $LLM_DRAGOON_CKPT_BYTES bytes (partial download?)" >&2
-        echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
-        return 1
-    fi
-    for gate in "$LLM_DRAGOON_NGRAM:$LLM_DRAGOON_NGRAM_BYTES:n-gram table" \
-                "$LLM_DRAGOON_VISION:$LLM_DRAGOON_VISION_BYTES:vision tower"; do
-        IFS=: read -r gpath gbytes gname <<<"$gate"
-        if [[ ! -f "$gpath" ]]; then
-            echo "ERROR: halogen $gname absent at $gpath" >&2
-            echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
-            return 1
-        fi
-        if [[ "$(stat -c %s "$gpath")" != "$gbytes" ]]; then
-            echo "ERROR: halogen $gname is not $gbytes bytes (partial download?)" >&2
-            echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
-            return 1
-        fi
-    done
-    if [[ -L "$LLM_DRAGOON_TOK_DIR" || ! -d "$LLM_DRAGOON_TOK_DIR" ]]; then
-        echo "ERROR: tokenizer dir missing or a symlink at $LLM_DRAGOON_TOK_DIR" >&2
-        echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
-        return 1
-    fi
-    for tf in $LLM_DRAGOON_TOK_FILES; do
-        if [[ ! -f "$LLM_DRAGOON_TOK_DIR/$tf" || -L "$LLM_DRAGOON_TOK_DIR/$tf" ]]; then
-            echo "ERROR: tokenizer file missing or a symlink: $LLM_DRAGOON_TOK_DIR/$tf" >&2
-            echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
-            return 1
-        fi
-    done
+    # Refuse rather than restart-loop; every gate here is the SAME library
+    # call the serve script makes, so the two twins cannot drift (the
+    # 2026-10-01 crash-loop lesson).
+    # Capture podman output before grepping is moot: require_image is one
+    # command, no pipe (the 2026-09-30 SIGPIPE lesson).
+    GATE_PREFIX=frank-dragoon
+    require_image "$LLM_DRAGOON_IMAGE" \
+        "pull it: podman pull docker://${LLM_DRAGOON_IMAGE%@*}" || return 1
+    require_bytes "$LLM_DRAGOON_CKPT" checkpoint "$LLM_DRAGOON_CKPT_BYTES" \
+        "fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
+    require_bytes "$LLM_DRAGOON_NGRAM" n-gram "$LLM_DRAGOON_NGRAM_BYTES" \
+        "fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
+    require_bytes "$LLM_DRAGOON_VISION" vision "$LLM_DRAGOON_VISION_BYTES" \
+        "fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
+    # shellcheck disable=SC2086
+    require_dir_files "$LLM_DRAGOON_TOK_DIR" tokenizer \
+        "fetch it: bash scripts/download-frank-dragoon-flashnext.sh" \
+        $LLM_DRAGOON_TOK_FILES || return 1

    retire_halogen_dragoon_quadlet
    # The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
@@ -1243,22 +1229,17 @@ LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K
 deploy_llm_zealot() {
    # Refuse rather than restart-loop: a unit whose runtime or payload is
    # missing never becomes healthy, and a green deploy would leave the
-    # endpoint dark with nothing in the log to say why.
-    if [[ ! -x "$LLM_ZEALOT_RUNTIME" ]]; then
-        echo "ERROR: llama.cpp runtime absent at $LLM_ZEALOT_RUNTIME" >&2
-        echo "frank-zealot: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
-        echo "frank-zealot: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
-        echo "frank-zealot: && cmake --build build-cuda --target llama-server) first." >&2
-        return 1
-    fi
-    for f in "$LLM_ZEALOT_MODEL_FILE" "$LLM_ZEALOT_MMPROJ"; do
-        if [[ ! -f "$f" ]]; then
-            echo "ERROR: frank-zealot payload file absent at $f" >&2
-            echo "frank-zealot: run scripts/download-frank-zealot.sh first;" >&2
-            echo "frank-zealot: refusing to start a unit that cannot load." >&2
-            return 1
-        fi
-    done
+    # endpoint dark with nothing in the log to say why. Same library calls
+    # as serve-frank-zealot.sh (the 2026-10-01 law).
+    GATE_PREFIX=frank-zealot
+    require_exec "$LLM_ZEALOT_RUNTIME" runtime \
+        "build it: cmake -B build-cuda -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86" \
+        || return 1
+    require_file "$LLM_ZEALOT_MODEL_FILE" gguf \
+        "fetch it: bash scripts/download-frank-zealot.sh" || return 1
+    require_file "$LLM_ZEALOT_MMPROJ" mmproj \
+        "fetch it: bash scripts/download-frank-zealot.sh" || return 1
+
    relink_unit frank-zealot
    run_privileged systemctl daemon-reload
    run_privileged systemctl enable frank-zealot.service
@@ -1300,11 +1281,16 @@ LLM_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-pr
 deploy_llm_archon() {
    # Refuse rather than restart-loop: a unit whose image or payload is
    # missing never becomes healthy, and a green deploy would leave the
-    # endpoint dark with nothing in the log to say why.
-    local env_file="/home/dank/serverAppConfigs/scripts/frank-archon.env"
-    local image model_dir model_file rad_min mib
+    # endpoint dark with nothing in the log to say why. The gates are the
+    # shared library's, called with the values the unit's own env file
+    # declares — so serve-frank-archon.sh and this handler cannot disagree,
+    # and the payload path lives in exactly one file.
+    GATE_PREFIX=frank-archon
+    local env_file="${LLM_ARCHON_ENV_FILE:-$REMOTE_DIR/scripts/frank-archon.env}"
+    local image model_dir model_file rad_min
    if [[ ! -f "$env_file" ]]; then
-        echo "ERROR: frank-archon env file absent: $env_file" >&2
+        echo "frank-archon: env: missing at $env_file" >&2
+        echo "  hint: deploy the repo checkout first ($REMOTE_DIR)" >&2
        return 1
    fi
    image="$(sed -n 's/^FRANK_ARCHON_IMAGE=//p' "$env_file" | tail -1)"
@@ -1312,28 +1298,16 @@ deploy_llm_archon() {
    model_file="$(sed -n 's/^FRANK_ARCHON_MODEL_FILE=//p' "$env_file" | tail -1)"
    rad_min="$(sed -n 's/^FRANK_ARCHON_RAD_MIN_MIB=//p' "$env_file" | tail -1)"
    if [[ -z "$image" || -z "$model_dir" || -z "$model_file" || -z "$rad_min" ]]; then
-        echo "ERROR: frank-archon.env missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
-        return 1
-    fi
-    # The image gate is the digest ref itself: a name@sha256 ref exists in
-    # storage only if THAT manifest was pulled. One command — no
-    # pipefail/grep race (the 2026-09-30 lesson).
-    if ! podman image exists "$image"; then
-        echo "ERROR: runtime image absent from storage: $image" >&2
-        echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest" >&2
-        return 1
-    fi
-    if [[ ! -f "$model_dir/$model_file" ]]; then
-        echo "ERROR: .rad container missing: $model_dir/$model_file" >&2
-        echo "frank-archon: run scripts/build-frank-archon-rad.sh" >&2
-        return 1
-    fi
-    mib=$(( $(stat -c %s "$model_dir/$model_file") / 1048576 ))
-    if [[ "$mib" -lt "$rad_min" ]]; then
-        echo "ERROR: .rad container too small: $model_dir/$model_file is ${mib} MiB, want >= $rad_min" >&2
-        echo "frank-archon: rebuild it (scripts/build-frank-archon-rad.sh --force)" >&2
+        echo "frank-archon: env: missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
+        echo "  hint: scripts/frank-archon.env" >&2
        return 1
    fi
+    require_image "$image" \
+        "pull it: podman pull docker://stilldeadcode/radiance:latest (as dank)" \
+        || return 1
+    require_min_mib "$model_dir/$model_file" ".rad" "$rad_min" \
+        "convert it: scripts/convert-swift15-w4dn8.sh (rad-info is the record)" \
+        || return 1

    # A stale container with the same name makes `podman run --name` fail at
    # start; remove any leftover from a failed previous attempt.
@@ -8,6 +8,7 @@
 # Both sides source this library, so the refusal is one text and one rule.
 #
 # Contract (public interface):
+#   require_exec      PATH LABEL HINT
 #   require_file      PATH LABEL HINT
 #   require_bytes     PATH LABEL WANT_BYTES HINT
 #   require_min_mib   PATH LABEL MIN_MIB HINT
@@ -20,7 +21,12 @@
 # scripts run under `set -e` (so a refusal is terminal), while the deployer
 # wants `|| return 1` so a failed gate reports a red deploy for that one
 # service and the dispatcher's state marker stays pending.
-#
+# Lines are labelled by ARTIFACT CLASS, and the class is the whole label
+# (the path rides in the same line): the class + path is what an operator
+# acts on, so no wording order and no payload rename can make the two twins
+# disagree. The deployer's old "halogen vision tower" and the serve script's
+# "vision-tower" were one gate in two vocabularies that no grep could join.
+
 # The size law is equality, not a floor, for `require_bytes`: a hub fetch
 # that truncates is the fleet's standing hazard (curl exits 0 on 401/403 and
 # writes the error body as the artifact), and the declared byte count is a
@@ -29,6 +35,14 @@
 # GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon).
 : "${GATE_PREFIX:=gate}"

+# Two bash laws make the `return 1` at every call site load-bearing:
+#   * a function whose body ends inside an `if` returns the CONDITION's
+#     status (true = 0), so a gate that ends on its own refusal prints the
+#     refusal and reports SUCCESS;
+#   * `set -e` is suspended for any command in a `… || …` chain, so a gate
+#     that runs inside one must not be trusted to abort the caller.
+# Every gate's refusal therefore carries an explicit `|| return 1`, and the
+# caller consumes the status (`|| exit 1`, `|| return 1`).
 _gate_fail() {
    printf '%s: %s\n' "$GATE_PREFIX" "$1" >&2
    printf '  hint: %s\n' "$2" >&2
@@ -41,8 +55,21 @@ _gate_fail() {
 require_file() {
    local path="$1" label="$2" hint="$3"
    if [[ -L "$path" || ! -f "$path" ]]; then
-        _gate_fail "$label: missing at $path" "$hint"
+        _gate_fail "$label: missing at $path" "$hint" || return 1
    fi
+    return 0
+}
+
+# require_exec — PATH must be an executable regular file. The native runtimes
+# (llama.cpp's llama-server) are binaries the operator builds; a tree that
+# compiled but lost its +x, or a directory wearing the binary's name, must
+# refuse the same way a missing one does.
+require_exec() {
+    local path="$1" label="$2" hint="$3"
+    if [[ -L "$path" || ! -x "$path" ]]; then
+        _gate_fail "$label: missing at $path" "$hint" || return 1
+    fi
+    return 0
 }

 # require_bytes — PATH must exist AND be exactly WANT_BYTES bytes.
@@ -51,8 +78,9 @@ require_bytes() {
    require_file "$path" "$label" "$hint" || return 1
    got="$(stat -c %s "$path")"
    if [[ "$got" != "$want" ]]; then
-        _gate_fail "$label: is $got bytes, expected $want" "$hint"
+        _gate_fail "$label: is $got bytes, expected $want" "$hint" || return 1
    fi
+    return 0
 }

 # require_min_mib — PATH must be at least MIN_MIB mebibytes. The floor form
@@ -63,8 +91,9 @@ require_min_mib() {
    require_file "$path" "$label" "$hint" || return 1
    mib=$(( $(stat -c %s "$path") / 1048576 ))
    if (( mib < min )); then
-        _gate_fail "$label: is $mib MiB, expected >= $min MiB" "$hint"
+        _gate_fail "$label: is $mib MiB, expected >= $min MiB" "$hint" || return 1
    fi
+    return 0
 }

 # require_dir_files — DIR must be a directory and every named member a
@@ -77,13 +106,14 @@ require_dir_files() {
    shift 3
    local f
    if [[ -L "$dir" || ! -d "$dir" ]]; then
-        _gate_fail "$label: missing at $dir" "$hint"
+        _gate_fail "$label: missing at $dir" "$hint" || return 1
    fi
    for f in "$@"; do
        if [[ -L "$dir/$f" || ! -f "$dir/$f" ]]; then
-            _gate_fail "$label: $dir/$f is missing or a symlink" "$hint"
+            _gate_fail "$label: $dir/$f is missing or a symlink" "$hint" || return 1
        fi
    done
+    return 0
 }

 # require_image — the image ref must be in podman storage. A name@sha256 ref
@@ -92,6 +122,7 @@ require_dir_files() {
 require_image() {
    local ref="$1" hint="$2"
    if ! podman image exists "$ref"; then
-        _gate_fail "image: $ref is not in podman storage" "$hint"
+        _gate_fail "image: $ref is not in podman storage" "$hint" || return 1
    fi
+    return 0
 }
@@ -79,10 +79,10 @@ require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_CHECKPOINT" \
    checkpoint "$FRANK_DRAGOON_CHECKPOINT_BYTES" \
    "fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1
 require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_NGRAM_TABLE" \
-    n-gram-table "$FRANK_DRAGOON_NGRAM_BYTES" \
+    n-gram "$FRANK_DRAGOON_NGRAM_BYTES" \
    "fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1
 require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_VISION_TOWER" \
-    vision-tower "$FRANK_DRAGOON_VISION_BYTES" \
+    vision "$FRANK_DRAGOON_VISION_BYTES" \
    "fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1

 # The payload ships the tokenizer as a FLAT DIRECTORY. A HuggingFace cache
@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+# deploy_sandbox.bash — the deploy-handler seam's fixture builder.
+#
+# The seam (tests/llm_deploy_gates.bats) EXECUTES scripts/deploy-remote.sh's
+# LLM handlers with every host touchpoint stubbed (tests/stubs/) and every
+# artifact pointed at a fixture, so the assertions are the handler's exit
+# status, its refusal text, and what it ASKED systemd/podman to do.
+#
+# Usage:
+#   source tests/deploy_sandbox.bash
+#   sandbox_setup
+#   run_handler dankserver frank-archon
+#   sandbox_teardown
+#
+# After sandbox_setup the caller owns:
+#   $STAGE                      the temp root (payload/, bin/, systemd/, state/)
+#   $REMOTE_DIR                 the sandbox checkout (a git clone; the handler
+#                               cds into it, so relative paths resolve there)
+#   $STAGE/systemctl.log        every systemctl invocation
+#   $STAGE/podman.log           every podman invocation
+#   $STAGE/systemd/<unit>.service  the unit link a green deploy leaves behind
+# Env the caller sets per case: STUB_IMAGE ("" = empty storage),
+# STUB_MAIN_PID (0 = a stray process owns the port), the LLM_* overrides.
+
+sandbox_setup() {
+    STAGE="$(mktemp -d)"
+    export HOME="$STAGE/home"
+    mkdir -p "$HOME" "$STAGE/state" "$STAGE/systemd" "$STAGE/bin"
+
+    # Host touchpoints: the seam's stubs, on PATH under their real names.
+    local stub repo_root
+    repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+    for stub in "$repo_root"/tests/stubs/*; do
+        ln -s "$stub" "$STAGE/bin/$(basename "$stub")"
+    done
+    export PATH="$STAGE/bin:$PATH"
+    export STUB_SYSTEMCTL_LOG="$STAGE/systemctl.log"
+    export STUB_PODMAN_LOG="$STAGE/podman.log"
+    # A deploy marks state; a sandbox with no git identity would make the
+    # identity, not the handler, the thing under test.
+    export GIT_AUTHOR_NAME=seam GIT_AUTHOR_EMAIL=seam@localhost
+    export GIT_COMMITTER_NAME=seam GIT_COMMITTER_EMAIL=seam@localhost
+    export SYSTEMD_SYSTEM_DIR="$STAGE/systemd"
+    export DEPLOY_STATE_DIR="$STAGE/state"
+    # run_privileged pipes the password from FRANK_SUDO_PW into `sudo -S --`;
+    # the stub drops it and execs, so no privilege gate hides a handler bug.
+    export FRANK_SUDO_PW=test
+
+    # Fixture payload: 3 MiB where a size floor applies, 10 bytes elsewhere.
+    mkdir -p "$STAGE/models/tokenizer"
+    truncate -s 3145728 "$STAGE/models/payload.rad"
+    truncate -s 10 "$STAGE/models/ck.hgn"
+    truncate -s 10 "$STAGE/models/ngram.hgn"
+    truncate -s 10 "$STAGE/models/vision.hgn"
+    printf 'x\n' > "$STAGE/models/llama-server"
+    chmod +x "$STAGE/models/llama-server"
+    truncate -s 10 "$STAGE/models/model.gguf"
+    truncate -s 10 "$STAGE/models/mmproj.gguf"
+    local tf
+    for tf in tokenizer.json chat_template.jinja tokenizer_config.json \
+        merges.txt vocab.json generation_config.json; do
+        printf 'x\n' > "$STAGE/models/tokenizer/$tf"
+    done
+
+    # $HOME/serverAppConfigs is the checkout the handler works in: a git
+    # WORKTREE of the caller's checkout, so the handler runs the EXACT files
+    # under test (the suite lives in worktrees, where uncommitted state is
+    # the norm), and it is a real git repo: the deployer refuses a revision
+    # it cannot resolve. The handler cds into REMOTE_DIR, so every relative
+    # path it uses resolves there.
+    git -C "$repo_root" worktree add -q --detach "$HOME/serverAppConfigs" HEAD
+    export REMOTE_DIR="$HOME/serverAppConfigs"
+
+    # Every endpoint variable is env-overridable, so the handlers see the
+    # fixtures and never the real /home/dank paths.
+    export STUB_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:deadbeef
+    export LLM_DRAGOON_IMAGE="$STUB_IMAGE"
+    export LLM_DRAGOON_CKPT="$STAGE/models/ck.hgn" LLM_DRAGOON_CKPT_BYTES=10
+    export LLM_DRAGOON_NGRAM="$STAGE/models/ngram.hgn" LLM_DRAGOON_NGRAM_BYTES=10
+    export LLM_DRAGOON_VISION="$STAGE/models/vision.hgn" LLM_DRAGOON_VISION_BYTES=10
+    export LLM_DRAGOON_TOK_DIR="$STAGE/models/tokenizer"
+    export LLM_ZEALOT_RUNTIME="$STAGE/models/llama-server"
+    export LLM_ZEALOT_MODEL_FILE="$STAGE/models/model.gguf"
+    export LLM_ZEALOT_MMPROJ="$STAGE/models/mmproj.gguf"
+    export LLM_ARCHON_ENV_FILE="$REMOTE_DIR/scripts/frank-archon.env"
+    # The archon handler reads the served image out of the unit's own env
+    # file, so the stub's storage must hold exactly that ref.
+    ARCHON_IMAGE="$(sed -nE 's/^FRANK_ARCHON_IMAGE=(.+)$/\1/p' "$LLM_ARCHON_ENV_FILE")"
+}
+
+sandbox_teardown() {
+    [[ -z "${STAGE:-}" ]] && return 0
+    # A worktree leaves a registration in the caller's .git/worktrees, so
+    # remove it through git, not just the directory.
+    [[ -d "$STAGE/home/serverAppConfigs" ]] &&
+        git -C "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" \
+            worktree remove --force "$STAGE/home/serverAppConfigs"
+    rm -rf "$STAGE"
+}
+
+run_handler() {
+    local target="$1" service="$2"
+    run bash "$REMOTE_DIR/scripts/deploy-remote.sh" \
+        "$target" "$(git -C "$REMOTE_DIR" rev-parse HEAD)" "$service"
+}
+
+# archon_payload_env — the unit's env file with the payload coordinates
+# redirected into the sandbox (and the size floor optionally lowered, which
+# is how a test builds the truncated-.rad case).
+archon_payload_env() {
+    local re="s#^FRANK_ARCHON_MODEL_DIR=.*#FRANK_ARCHON_MODEL_DIR=$STAGE/models#;"
+    re+=" s#^FRANK_ARCHON_MODEL_FILE=.*#FRANK_ARCHON_MODEL_FILE=payload.rad#"
+    if [[ -n "${1:-}" ]]; then
+        re+="; s#^FRANK_ARCHON_RAD_MIN_MIB=.*#FRANK_ARCHON_RAD_MIN_MIB=$1#"
+    fi
+    sed -E "$re" "$LLM_ARCHON_ENV_FILE" > "$STAGE/archon.env"
+    export LLM_ARCHON_ENV_FILE="$STAGE/archon.env"
+}
+
+restarts_requested() {
+    # grep -c prints 0 and exits 1 on no match; the count, not the status,
+    # is what the assertions read.
+    local n=0
+    [[ -f "$STAGE/systemctl.log" ]] && n="$(grep -c 'restart ' "$STAGE/systemctl.log")"
+    printf '%s\n' "$n"
+}
@@ -204,7 +204,7 @@ good_env() {
    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
        FRANK_DRAGOON_NGRAM_TABLE=absent.hgn "$SERVE"
    [ "$status" -ne 0 ]
-    [[ "$output" == *"n-gram-table"*"missing"* ]]
+    [[ "$output" == *"frank-dragoon: n-gram: missing at $TEST_TMP/absent.hgn"* ]]
    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
        FRANK_DRAGOON_NGRAM_BYTES=42 "$SERVE"
    [ "$status" -ne 0 ]
@@ -220,7 +220,7 @@ good_env() {
    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
        FRANK_DRAGOON_VISION_TOWER=absent.hgn "$SERVE"
    [ "$status" -ne 0 ]
-    [[ "$output" == *"vision-tower"*"missing"* ]]
+    [[ "$output" == *"frank-dragoon: vision: missing at $TEST_TMP/absent.hgn"* ]]
    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
        FRANK_DRAGOON_VISION_BYTES=42 "$SERVE"
    [ "$status" -ne 0 ]
@@ -0,0 +1,133 @@
+#!/usr/bin/env bats
+# Seam: the deployer's LLM handlers are the twin of the serve scripts'
+# artifact gates (the 2026-10-01 crash-loop: a green deploy against a
+# missing payload left a restart-looping unit and a dark endpoint).
+#
+# These tests EXECUTE scripts/deploy-remote.sh's LLM handlers in a sandbox
+# (tests/deploy_sandbox.bash): every host touchpoint is a stub, the artifacts
+# are fixtures, and the assertions are exit status, the refusal on stderr,
+# and what the handler asked systemd to do. A handler that started the unit
+# before checking its artifacts would pass a text-grep seam and fail here.
+
+load helpers
+load deploy_sandbox
+
+setup() { sandbox_setup; }
+teardown() { sandbox_teardown; }
+
+@test "the dragoon handler refuses when the engine image is not in storage" {
+    export STUB_IMAGE=
+    run_handler dankclaw frank-dragoon
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"frank-dragoon: image:"*"not in podman storage"* ]]
+    [ "$(restarts_requested)" = "0" ]
+}
+
+@test "the dragoon handler refuses a truncated checkpoint, naming both sizes" {
+    truncate -s 9 "$STAGE/models/ck.hgn"
+    run_handler dankclaw frank-dragoon
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"checkpoint: is 9 bytes, expected 10"* ]]
+    [ "$(restarts_requested)" = "0" ]
+}
+
+@test "the dragoon handler refuses a symlinked tokenizer file" {
+    # The flat-tokenizer law: a HuggingFace cache snapshot links its
+    # members, and the link dangles inside the container mount.
+    rm "$STAGE/models/tokenizer/vocab.json"
+    ln -s /nowhere/vocab.json "$STAGE/models/tokenizer/vocab.json"
+    run_handler dankclaw frank-dragoon
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"frank-dragoon: tokenizer: $STAGE/models/tokenizer/vocab.json is missing or a symlink"* ]]
+    [ "$(restarts_requested)" = "0" ]
+}
+
+@test "the dragoon handler starts the unit once every artifact gate passes" {
+    run_handler dankclaw frank-dragoon
+    [ "$status" -eq 0 ]
+    grep -q 'restart frank-dragoon.service' "$STAGE/systemctl.log"
+    grep -qx 'rm -f frank-dragoon frank-dragoon-halogen' "$STAGE/podman.log"
+}
+
+@test "the zealot handler refuses when the llama.cpp runtime is not executable" {
+    # The runtime is an operator-built binary; a tree that compiled but lost
+    # its exec bit must refuse exactly like a missing one.
+    rm "$STAGE/models/llama-server"
+    run_handler dankclaw frank-zealot
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"frank-zealot: runtime: missing at $STAGE/models/llama-server"* ]]
+    [[ "$output" == *"cmake"* ]]
+    [ "$(restarts_requested)" = "0" ]
+}
+
+@test "the zealot handler refuses a missing projector — a text-only endpoint is not deployable" {
+    # The projector IS the vision capability: deploying without it silently
+    # ships a text-only endpoint.
+    rm "$STAGE/models/mmproj.gguf"
+    run_handler dankclaw frank-zealot
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"frank-zealot: mmproj: missing at $STAGE/models/mmproj.gguf"* ]]
+    [ "$(restarts_requested)" = "0" ]
+}
+
+@test "the zealot handler starts the unit once every artifact gate passes" {
+    run_handler dankclaw frank-zealot
+    [ "$status" -eq 0 ]
+    grep -q 'restart frank-zealot.service' "$STAGE/systemctl.log"
+}
+
+@test "the archon handler refuses when the radiance image is not in storage" {
+    export STUB_IMAGE=
+    run_handler dankserver frank-archon
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"frank-archon: image:"*"not in podman storage"* ]]
+    [[ "$output" == *"podman pull"* ]]
+    [ "$(restarts_requested)" = "0" ]
+}
+
+@test "the archon handler refuses a .rad below the env's size floor" {
+    truncate -s 1048576 "$STAGE/models/payload.rad"   # 1 MiB, far under the floor
+    archon_payload_env
+    export STUB_IMAGE="$ARCHON_IMAGE"
+    run_handler dankserver frank-archon
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"frank-archon: .rad: is 1 MiB, expected >= "* ]]
+    [[ "$output" == *"convert-swift15-w4dn8.sh"* ]]
+    [ "$(restarts_requested)" = "0" ]
+}
+
+@test "the archon handler clears the legacy container identities and relinks the unit" {
+    # The retired identities (the R9V container, the ad hoc probe, the glm
+    # experiment) hold the same GPUs, so a deploy that leaves one running
+    # leaves the new engine unable to bind. The unit link is the root-unit
+    # shape's whole install.
+    archon_payload_env 2
+    export STUB_IMAGE="$ARCHON_IMAGE"
+    run_handler dankserver frank-archon
+    [ "$status" -eq 0 ]
+    grep -q 'restart frank-archon.service' "$STAGE/systemctl.log"
+    [ -L "$STAGE/systemd/frank-archon.service" ]
+    local ctr
+    for ctr in frank-archon frank-archon-qwen frank-archon-r9v qwen-fp8 glm-probe; do
+        grep -qx "rm -f $ctr" "$STAGE/podman.log"
+    done
+}
+
+@test "no LLM handler reports green when its unit has no MainPID behind the port" {
+    # The 2026-09-26 false green: a leftover probe held :7770 and satisfied
+    # the health curl while the real unit was stopped.
+    archon_payload_env 2
+    export STUB_IMAGE="$ARCHON_IMAGE"
+    export STUB_MAIN_PID=0
+    run_handler dankserver frank-archon
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"frank-archon.service has no MainPID"* ]]
+}
+
+@test "a refused LLM handler leaves its service pending in the deploy state" {
+    export STUB_IMAGE=
+    run_handler dankserver frank-archon
+    [ "$status" -ne 0 ]
+    grep -q 'status=pending' "$STAGE/state/frank-archon"
+    [ ! -e "$STAGE/state/frank-zealot" ]
+}
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exit 0
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Stub podman for the deploy-handler seam: `image exists` succeeds only for
+# $STUB_IMAGE (empty = nothing in storage); every other verb is a no-op.
+# The invocations are logged so a test can read what the handler executed.
+printf '%s\n' "$*" >> "${STUB_PODMAN_LOG:-/dev/null}"
+case "$1 $2" in
+    "image exists")
+        [[ -n "${STUB_IMAGE:-}" && "$3" == "$STUB_IMAGE" ]] || exit 1
+        ;;
+esac
+exit 0
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exit 0
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exit 0
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Stub sudo for the deploy-handler seam: run_privileged pipes the password
+# on stdin and calls `sudo -S -- <cmd>`. The stub drops the password and the
+# flags and execs the command, so no privilege gate hides a handler bug.
+cat >/dev/null 2>&1 || true
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+        -S|--prompt) shift ;;
+        --) shift; break ;;
+        *) break ;;
+    esac
+done
+exec "$@"
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Stub systemctl for the deploy-handler seam: every invocation is logged, so
+# a test asserts on what the handler ASKED systemd to do. `is-active` answers
+# "inactive" (the state a cutover finds the unit in) and MainPID answers
+# $STUB_MAIN_PID — 0 is the stray-port-holder shape, the false-green gate
+# every LLM handler reads after a health answer.
+printf '%s\n' "$*" >> "${STUB_SYSTEMCTL_LOG:-/dev/null}"
+case "$*" in
+    "is-active "*) echo inactive ;;
+    *"MainPID --value "*) echo "${STUB_MAIN_PID:-4242}" ;;
+esac
+exit 0