refact(deployer): the LLM handlers gate through the shared library

deploy_llm_dragoon/zealot/arkon now call require_image / require_bytes /
require_min_mib / require_file / require_dir_files, so the deployer and the
serve scripts refuse the same missing payload with the same words — the
2026-10-01 crash-loop law, enforced by construction instead of by duplicated
shell. require_exec joins the library for the native-runtime gate (a
compiled tree that lost its +x refuses like a missing one). relink_unit
takes SYSTEMD_SYSTEM_DIR so a handler can be driven end to end in a sandbox.

The library's own refusals now carry their status: a bash function that
ends inside an if returns the CONDITION's status, so a gate whose last
statement is its refusal printed the message and returned SUCCESS — the
new seam caught it, and the gate tests pin it.

New seam tests/llm_deploy_gates.bash + tests/stubs/: deploy-remote.sh's LLM
handlers EXECUTED against stub systemctl/podman/sudo/ss/curl, asserting
exit status, refusal text, no restart, and the pending state marker.
This commit is contained in:
Frank
2026-10-13 21:58:13 -07:00
parent cb25796a6b
commit c5a816e6a8
12 changed files with 403 additions and 97 deletions
+60 -86
View File
@@ -9,6 +9,13 @@
# pending until its service is explicitly deployed.
set -euo pipefail
# The LLM handlers' artifact gates are the serve scripts' own library, so
# the deployer and the unit refuse the same missing payload with the same
# words (the fleet law from the 2026-10-01 crash-loop). GATE_PREFIX is set
# per handler.
# shellcheck disable=SC1091
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/llm-gates.sh"
TARGET="${1:-}"
REVISION="${2:-}"
shift 2 || true
@@ -721,9 +728,15 @@ deploy_gitea_unit() {
systemctl --user restart "$service.service"
}
# The root units are symlinks from the systemd system directory into the
# checkout. SYSTEMD_SYSTEM_DIR is production (/etc/systemd/system); the knob
# exists so the deploy seam can run a whole handler — relink included — in a
# sandbox and read the unit links as evidence, unprivileged.
SYSTEMD_SYSTEM_DIR="${SYSTEMD_SYSTEM_DIR:-/etc/systemd/system}"
relink_unit() {
local service="$1"
run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service"
run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "$SYSTEMD_SYSTEM_DIR/$service.service"
}
# Authentik runs as three boot-persistent Quadlet units (postgres -> server ->
@@ -1155,51 +1168,24 @@ wait_port_free() {
}
deploy_llm_dragoon() {
# Refuse rather than restart-loop; every gate below has a twin in
# serve-frank-dragoon.sh and they must agree (2026-10-01 lesson).
# Capture podman output before grepping: `podman images | grep -q`
# races under pipefail (SIGPIPE 141 becomes "no match", 2026-09-30).
if ! grep -qF "${LLM_DRAGOON_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then
echo "ERROR: halogen image absent: $LLM_DRAGOON_IMAGE" >&2
echo "frank-dragoon: pull it: podman pull ${LLM_DRAGOON_IMAGE%@*}" >&2
return 1
fi
if [[ ! -f "$LLM_DRAGOON_CKPT" ]]; then
echo "ERROR: halogen checkpoint absent at $LLM_DRAGOON_CKPT" >&2
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
return 1
fi
if [[ "$(stat -c %s "$LLM_DRAGOON_CKPT")" != "$LLM_DRAGOON_CKPT_BYTES" ]]; then
echo "ERROR: halogen checkpoint is not $LLM_DRAGOON_CKPT_BYTES bytes (partial download?)" >&2
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
return 1
fi
for gate in "$LLM_DRAGOON_NGRAM:$LLM_DRAGOON_NGRAM_BYTES:n-gram table" \
"$LLM_DRAGOON_VISION:$LLM_DRAGOON_VISION_BYTES:vision tower"; do
IFS=: read -r gpath gbytes gname <<<"$gate"
if [[ ! -f "$gpath" ]]; then
echo "ERROR: halogen $gname absent at $gpath" >&2
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
return 1
fi
if [[ "$(stat -c %s "$gpath")" != "$gbytes" ]]; then
echo "ERROR: halogen $gname is not $gbytes bytes (partial download?)" >&2
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
return 1
fi
done
if [[ -L "$LLM_DRAGOON_TOK_DIR" || ! -d "$LLM_DRAGOON_TOK_DIR" ]]; then
echo "ERROR: tokenizer dir missing or a symlink at $LLM_DRAGOON_TOK_DIR" >&2
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
return 1
fi
for tf in $LLM_DRAGOON_TOK_FILES; do
if [[ ! -f "$LLM_DRAGOON_TOK_DIR/$tf" || -L "$LLM_DRAGOON_TOK_DIR/$tf" ]]; then
echo "ERROR: tokenizer file missing or a symlink: $LLM_DRAGOON_TOK_DIR/$tf" >&2
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
return 1
fi
done
# Refuse rather than restart-loop; every gate here is the SAME library
# call the serve script makes, so the two twins cannot drift (the
# 2026-10-01 crash-loop lesson).
# Capture podman output before grepping is moot: require_image is one
# command, no pipe (the 2026-09-30 SIGPIPE lesson).
GATE_PREFIX=frank-dragoon
require_image "$LLM_DRAGOON_IMAGE" \
"pull it: podman pull docker://${LLM_DRAGOON_IMAGE%@*}" || return 1
require_bytes "$LLM_DRAGOON_CKPT" checkpoint "$LLM_DRAGOON_CKPT_BYTES" \
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
require_bytes "$LLM_DRAGOON_NGRAM" n-gram "$LLM_DRAGOON_NGRAM_BYTES" \
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
require_bytes "$LLM_DRAGOON_VISION" vision "$LLM_DRAGOON_VISION_BYTES" \
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
# shellcheck disable=SC2086
require_dir_files "$LLM_DRAGOON_TOK_DIR" tokenizer \
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh" \
$LLM_DRAGOON_TOK_FILES || return 1
retire_halogen_dragoon_quadlet
# The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
@@ -1243,22 +1229,17 @@ LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K
deploy_llm_zealot() {
# Refuse rather than restart-loop: a unit whose runtime or payload is
# missing never becomes healthy, and a green deploy would leave the
# endpoint dark with nothing in the log to say why.
if [[ ! -x "$LLM_ZEALOT_RUNTIME" ]]; then
echo "ERROR: llama.cpp runtime absent at $LLM_ZEALOT_RUNTIME" >&2
echo "frank-zealot: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
echo "frank-zealot: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
echo "frank-zealot: && cmake --build build-cuda --target llama-server) first." >&2
return 1
fi
for f in "$LLM_ZEALOT_MODEL_FILE" "$LLM_ZEALOT_MMPROJ"; do
if [[ ! -f "$f" ]]; then
echo "ERROR: frank-zealot payload file absent at $f" >&2
echo "frank-zealot: run scripts/download-frank-zealot.sh first;" >&2
echo "frank-zealot: refusing to start a unit that cannot load." >&2
return 1
fi
done
# endpoint dark with nothing in the log to say why. Same library calls
# as serve-frank-zealot.sh (the 2026-10-01 law).
GATE_PREFIX=frank-zealot
require_exec "$LLM_ZEALOT_RUNTIME" runtime \
"build it: cmake -B build-cuda -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86" \
|| return 1
require_file "$LLM_ZEALOT_MODEL_FILE" gguf \
"fetch it: bash scripts/download-frank-zealot.sh" || return 1
require_file "$LLM_ZEALOT_MMPROJ" mmproj \
"fetch it: bash scripts/download-frank-zealot.sh" || return 1
relink_unit frank-zealot
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-zealot.service
@@ -1300,11 +1281,16 @@ LLM_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-pr
deploy_llm_archon() {
# Refuse rather than restart-loop: a unit whose image or payload is
# missing never becomes healthy, and a green deploy would leave the
# endpoint dark with nothing in the log to say why.
local env_file="/home/dank/serverAppConfigs/scripts/frank-archon.env"
local image model_dir model_file rad_min mib
# endpoint dark with nothing in the log to say why. The gates are the
# shared library's, called with the values the unit's own env file
# declares — so serve-frank-archon.sh and this handler cannot disagree,
# and the payload path lives in exactly one file.
GATE_PREFIX=frank-archon
local env_file="${LLM_ARCHON_ENV_FILE:-$REMOTE_DIR/scripts/frank-archon.env}"
local image model_dir model_file rad_min
if [[ ! -f "$env_file" ]]; then
echo "ERROR: frank-archon env file absent: $env_file" >&2
echo "frank-archon: env: missing at $env_file" >&2
echo " hint: deploy the repo checkout first ($REMOTE_DIR)" >&2
return 1
fi
image="$(sed -n 's/^FRANK_ARCHON_IMAGE=//p' "$env_file" | tail -1)"
@@ -1312,28 +1298,16 @@ deploy_llm_archon() {
model_file="$(sed -n 's/^FRANK_ARCHON_MODEL_FILE=//p' "$env_file" | tail -1)"
rad_min="$(sed -n 's/^FRANK_ARCHON_RAD_MIN_MIB=//p' "$env_file" | tail -1)"
if [[ -z "$image" || -z "$model_dir" || -z "$model_file" || -z "$rad_min" ]]; then
echo "ERROR: frank-archon.env missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
return 1
fi
# The image gate is the digest ref itself: a name@sha256 ref exists in
# storage only if THAT manifest was pulled. One command — no
# pipefail/grep race (the 2026-09-30 lesson).
if ! podman image exists "$image"; then
echo "ERROR: runtime image absent from storage: $image" >&2
echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest" >&2
return 1
fi
if [[ ! -f "$model_dir/$model_file" ]]; then
echo "ERROR: .rad container missing: $model_dir/$model_file" >&2
echo "frank-archon: run scripts/build-frank-archon-rad.sh" >&2
return 1
fi
mib=$(( $(stat -c %s "$model_dir/$model_file") / 1048576 ))
if [[ "$mib" -lt "$rad_min" ]]; then
echo "ERROR: .rad container too small: $model_dir/$model_file is ${mib} MiB, want >= $rad_min" >&2
echo "frank-archon: rebuild it (scripts/build-frank-archon-rad.sh --force)" >&2
echo "frank-archon: env: missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
echo " hint: scripts/frank-archon.env" >&2
return 1
fi
require_image "$image" \
"pull it: podman pull docker://stilldeadcode/radiance:latest (as dank)" \
|| return 1
require_min_mib "$model_dir/$model_file" ".rad" "$rad_min" \
"convert it: scripts/convert-swift15-w4dn8.sh (rad-info is the record)" \
|| return 1
# A stale container with the same name makes `podman run --name` fail at
# start; remove any leftover from a failed previous attempt.
+38 -7
View File
@@ -8,6 +8,7 @@
# Both sides source this library, so the refusal is one text and one rule.
#
# Contract (public interface):
# require_exec PATH LABEL HINT
# require_file PATH LABEL HINT
# require_bytes PATH LABEL WANT_BYTES HINT
# require_min_mib PATH LABEL MIN_MIB HINT
@@ -20,7 +21,12 @@
# scripts run under `set -e` (so a refusal is terminal), while the deployer
# wants `|| return 1` so a failed gate reports a red deploy for that one
# service and the dispatcher's state marker stays pending.
#
# Lines are labelled by ARTIFACT CLASS, and the class is the whole label
# (the path rides in the same line): the class + path is what an operator
# acts on, so no wording order and no payload rename can make the two twins
# disagree. The deployer's old "halogen vision tower" and the serve script's
# "vision-tower" were one gate in two vocabularies that no grep could join.
# The size law is equality, not a floor, for `require_bytes`: a hub fetch
# that truncates is the fleet's standing hazard (curl exits 0 on 401/403 and
# writes the error body as the artifact), and the declared byte count is a
@@ -29,6 +35,14 @@
# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon).
: "${GATE_PREFIX:=gate}"
# Two bash laws make the `return 1` at every call site load-bearing:
# * a function whose body ends inside an `if` returns the CONDITION's
# status (true = 0), so a gate that ends on its own refusal prints the
# refusal and reports SUCCESS;
# * `set -e` is suspended for any command in a `… || …` chain, so a gate
# that runs inside one must not be trusted to abort the caller.
# Every gate's refusal therefore carries an explicit `|| return 1`, and the
# caller consumes the status (`|| exit 1`, `|| return 1`).
_gate_fail() {
printf '%s: %s\n' "$GATE_PREFIX" "$1" >&2
printf ' hint: %s\n' "$2" >&2
@@ -41,8 +55,21 @@ _gate_fail() {
require_file() {
local path="$1" label="$2" hint="$3"
if [[ -L "$path" || ! -f "$path" ]]; then
_gate_fail "$label: missing at $path" "$hint"
_gate_fail "$label: missing at $path" "$hint" || return 1
fi
return 0
}
# require_exec — PATH must be an executable regular file. The native runtimes
# (llama.cpp's llama-server) are binaries the operator builds; a tree that
# compiled but lost its +x, or a directory wearing the binary's name, must
# refuse the same way a missing one does.
require_exec() {
local path="$1" label="$2" hint="$3"
if [[ -L "$path" || ! -x "$path" ]]; then
_gate_fail "$label: missing at $path" "$hint" || return 1
fi
return 0
}
# require_bytes — PATH must exist AND be exactly WANT_BYTES bytes.
@@ -51,8 +78,9 @@ require_bytes() {
require_file "$path" "$label" "$hint" || return 1
got="$(stat -c %s "$path")"
if [[ "$got" != "$want" ]]; then
_gate_fail "$label: is $got bytes, expected $want" "$hint"
_gate_fail "$label: is $got bytes, expected $want" "$hint" || return 1
fi
return 0
}
# require_min_mib — PATH must be at least MIN_MIB mebibytes. The floor form
@@ -63,8 +91,9 @@ require_min_mib() {
require_file "$path" "$label" "$hint" || return 1
mib=$(( $(stat -c %s "$path") / 1048576 ))
if (( mib < min )); then
_gate_fail "$label: is $mib MiB, expected >= $min MiB" "$hint"
_gate_fail "$label: is $mib MiB, expected >= $min MiB" "$hint" || return 1
fi
return 0
}
# require_dir_files — DIR must be a directory and every named member a
@@ -77,13 +106,14 @@ require_dir_files() {
shift 3
local f
if [[ -L "$dir" || ! -d "$dir" ]]; then
_gate_fail "$label: missing at $dir" "$hint"
_gate_fail "$label: missing at $dir" "$hint" || return 1
fi
for f in "$@"; do
if [[ -L "$dir/$f" || ! -f "$dir/$f" ]]; then
_gate_fail "$label: $dir/$f is missing or a symlink" "$hint"
_gate_fail "$label: $dir/$f is missing or a symlink" "$hint" || return 1
fi
done
return 0
}
# require_image — the image ref must be in podman storage. A name@sha256 ref
@@ -92,6 +122,7 @@ require_dir_files() {
require_image() {
local ref="$1" hint="$2"
if ! podman image exists "$ref"; then
_gate_fail "image: $ref is not in podman storage" "$hint"
_gate_fail "image: $ref is not in podman storage" "$hint" || return 1
fi
return 0
}
+2 -2
View File
@@ -79,10 +79,10 @@ require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_CHECKPOINT" \
checkpoint "$FRANK_DRAGOON_CHECKPOINT_BYTES" \
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1
require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_NGRAM_TABLE" \
n-gram-table "$FRANK_DRAGOON_NGRAM_BYTES" \
n-gram "$FRANK_DRAGOON_NGRAM_BYTES" \
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1
require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_VISION_TOWER" \
vision-tower "$FRANK_DRAGOON_VISION_BYTES" \
vision "$FRANK_DRAGOON_VISION_BYTES" \
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1
# The payload ships the tokenizer as a FLAT DIRECTORY. A HuggingFace cache