refact(deployer): the LLM handlers gate through the shared library
deploy_llm_dragoon/zealot/arkon now call require_image / require_bytes / require_min_mib / require_file / require_dir_files, so the deployer and the serve scripts refuse the same missing payload with the same words — the 2026-10-01 crash-loop law, enforced by construction instead of by duplicated shell. require_exec joins the library for the native-runtime gate (a compiled tree that lost its +x refuses like a missing one). relink_unit takes SYSTEMD_SYSTEM_DIR so a handler can be driven end to end in a sandbox. The library's own refusals now carry their status: a bash function that ends inside an if returns the CONDITION's status, so a gate whose last statement is its refusal printed the message and returned SUCCESS — the new seam caught it, and the gate tests pin it. New seam tests/llm_deploy_gates.bash + tests/stubs/: deploy-remote.sh's LLM handlers EXECUTED against stub systemctl/podman/sudo/ss/curl, asserting exit status, refusal text, no restart, and the pending state marker.
This commit is contained in:
+60
-86
@@ -9,6 +9,13 @@
|
||||
# pending until its service is explicitly deployed.
|
||||
set -euo pipefail
|
||||
|
||||
# The LLM handlers' artifact gates are the serve scripts' own library, so
|
||||
# the deployer and the unit refuse the same missing payload with the same
|
||||
# words (the fleet law from the 2026-10-01 crash-loop). GATE_PREFIX is set
|
||||
# per handler.
|
||||
# shellcheck disable=SC1091
|
||||
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/llm-gates.sh"
|
||||
|
||||
TARGET="${1:-}"
|
||||
REVISION="${2:-}"
|
||||
shift 2 || true
|
||||
@@ -721,9 +728,15 @@ deploy_gitea_unit() {
|
||||
systemctl --user restart "$service.service"
|
||||
}
|
||||
|
||||
# The root units are symlinks from the systemd system directory into the
|
||||
# checkout. SYSTEMD_SYSTEM_DIR is production (/etc/systemd/system); the knob
|
||||
# exists so the deploy seam can run a whole handler — relink included — in a
|
||||
# sandbox and read the unit links as evidence, unprivileged.
|
||||
SYSTEMD_SYSTEM_DIR="${SYSTEMD_SYSTEM_DIR:-/etc/systemd/system}"
|
||||
|
||||
relink_unit() {
|
||||
local service="$1"
|
||||
run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service"
|
||||
run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "$SYSTEMD_SYSTEM_DIR/$service.service"
|
||||
}
|
||||
|
||||
# Authentik runs as three boot-persistent Quadlet units (postgres -> server ->
|
||||
@@ -1155,51 +1168,24 @@ wait_port_free() {
|
||||
}
|
||||
|
||||
deploy_llm_dragoon() {
|
||||
# Refuse rather than restart-loop; every gate below has a twin in
|
||||
# serve-frank-dragoon.sh and they must agree (2026-10-01 lesson).
|
||||
# Capture podman output before grepping: `podman images | grep -q`
|
||||
# races under pipefail (SIGPIPE 141 becomes "no match", 2026-09-30).
|
||||
if ! grep -qF "${LLM_DRAGOON_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then
|
||||
echo "ERROR: halogen image absent: $LLM_DRAGOON_IMAGE" >&2
|
||||
echo "frank-dragoon: pull it: podman pull ${LLM_DRAGOON_IMAGE%@*}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$LLM_DRAGOON_CKPT" ]]; then
|
||||
echo "ERROR: halogen checkpoint absent at $LLM_DRAGOON_CKPT" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(stat -c %s "$LLM_DRAGOON_CKPT")" != "$LLM_DRAGOON_CKPT_BYTES" ]]; then
|
||||
echo "ERROR: halogen checkpoint is not $LLM_DRAGOON_CKPT_BYTES bytes (partial download?)" >&2
|
||||
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
|
||||
return 1
|
||||
fi
|
||||
for gate in "$LLM_DRAGOON_NGRAM:$LLM_DRAGOON_NGRAM_BYTES:n-gram table" \
|
||||
"$LLM_DRAGOON_VISION:$LLM_DRAGOON_VISION_BYTES:vision tower"; do
|
||||
IFS=: read -r gpath gbytes gname <<<"$gate"
|
||||
if [[ ! -f "$gpath" ]]; then
|
||||
echo "ERROR: halogen $gname absent at $gpath" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(stat -c %s "$gpath")" != "$gbytes" ]]; then
|
||||
echo "ERROR: halogen $gname is not $gbytes bytes (partial download?)" >&2
|
||||
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
if [[ -L "$LLM_DRAGOON_TOK_DIR" || ! -d "$LLM_DRAGOON_TOK_DIR" ]]; then
|
||||
echo "ERROR: tokenizer dir missing or a symlink at $LLM_DRAGOON_TOK_DIR" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
for tf in $LLM_DRAGOON_TOK_FILES; do
|
||||
if [[ ! -f "$LLM_DRAGOON_TOK_DIR/$tf" || -L "$LLM_DRAGOON_TOK_DIR/$tf" ]]; then
|
||||
echo "ERROR: tokenizer file missing or a symlink: $LLM_DRAGOON_TOK_DIR/$tf" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
# Refuse rather than restart-loop; every gate here is the SAME library
|
||||
# call the serve script makes, so the two twins cannot drift (the
|
||||
# 2026-10-01 crash-loop lesson).
|
||||
# Capture podman output before grepping is moot: require_image is one
|
||||
# command, no pipe (the 2026-09-30 SIGPIPE lesson).
|
||||
GATE_PREFIX=frank-dragoon
|
||||
require_image "$LLM_DRAGOON_IMAGE" \
|
||||
"pull it: podman pull docker://${LLM_DRAGOON_IMAGE%@*}" || return 1
|
||||
require_bytes "$LLM_DRAGOON_CKPT" checkpoint "$LLM_DRAGOON_CKPT_BYTES" \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
|
||||
require_bytes "$LLM_DRAGOON_NGRAM" n-gram "$LLM_DRAGOON_NGRAM_BYTES" \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
|
||||
require_bytes "$LLM_DRAGOON_VISION" vision "$LLM_DRAGOON_VISION_BYTES" \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
|
||||
# shellcheck disable=SC2086
|
||||
require_dir_files "$LLM_DRAGOON_TOK_DIR" tokenizer \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh" \
|
||||
$LLM_DRAGOON_TOK_FILES || return 1
|
||||
|
||||
retire_halogen_dragoon_quadlet
|
||||
# The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
|
||||
@@ -1243,22 +1229,17 @@ LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K
|
||||
deploy_llm_zealot() {
|
||||
# Refuse rather than restart-loop: a unit whose runtime or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
if [[ ! -x "$LLM_ZEALOT_RUNTIME" ]]; then
|
||||
echo "ERROR: llama.cpp runtime absent at $LLM_ZEALOT_RUNTIME" >&2
|
||||
echo "frank-zealot: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
|
||||
echo "frank-zealot: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
|
||||
echo "frank-zealot: && cmake --build build-cuda --target llama-server) first." >&2
|
||||
return 1
|
||||
fi
|
||||
for f in "$LLM_ZEALOT_MODEL_FILE" "$LLM_ZEALOT_MMPROJ"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: frank-zealot payload file absent at $f" >&2
|
||||
echo "frank-zealot: run scripts/download-frank-zealot.sh first;" >&2
|
||||
echo "frank-zealot: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
# endpoint dark with nothing in the log to say why. Same library calls
|
||||
# as serve-frank-zealot.sh (the 2026-10-01 law).
|
||||
GATE_PREFIX=frank-zealot
|
||||
require_exec "$LLM_ZEALOT_RUNTIME" runtime \
|
||||
"build it: cmake -B build-cuda -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86" \
|
||||
|| return 1
|
||||
require_file "$LLM_ZEALOT_MODEL_FILE" gguf \
|
||||
"fetch it: bash scripts/download-frank-zealot.sh" || return 1
|
||||
require_file "$LLM_ZEALOT_MMPROJ" mmproj \
|
||||
"fetch it: bash scripts/download-frank-zealot.sh" || return 1
|
||||
|
||||
relink_unit frank-zealot
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-zealot.service
|
||||
@@ -1300,11 +1281,16 @@ LLM_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-pr
|
||||
deploy_llm_archon() {
|
||||
# Refuse rather than restart-loop: a unit whose image or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
local env_file="/home/dank/serverAppConfigs/scripts/frank-archon.env"
|
||||
local image model_dir model_file rad_min mib
|
||||
# endpoint dark with nothing in the log to say why. The gates are the
|
||||
# shared library's, called with the values the unit's own env file
|
||||
# declares — so serve-frank-archon.sh and this handler cannot disagree,
|
||||
# and the payload path lives in exactly one file.
|
||||
GATE_PREFIX=frank-archon
|
||||
local env_file="${LLM_ARCHON_ENV_FILE:-$REMOTE_DIR/scripts/frank-archon.env}"
|
||||
local image model_dir model_file rad_min
|
||||
if [[ ! -f "$env_file" ]]; then
|
||||
echo "ERROR: frank-archon env file absent: $env_file" >&2
|
||||
echo "frank-archon: env: missing at $env_file" >&2
|
||||
echo " hint: deploy the repo checkout first ($REMOTE_DIR)" >&2
|
||||
return 1
|
||||
fi
|
||||
image="$(sed -n 's/^FRANK_ARCHON_IMAGE=//p' "$env_file" | tail -1)"
|
||||
@@ -1312,28 +1298,16 @@ deploy_llm_archon() {
|
||||
model_file="$(sed -n 's/^FRANK_ARCHON_MODEL_FILE=//p' "$env_file" | tail -1)"
|
||||
rad_min="$(sed -n 's/^FRANK_ARCHON_RAD_MIN_MIB=//p' "$env_file" | tail -1)"
|
||||
if [[ -z "$image" || -z "$model_dir" || -z "$model_file" || -z "$rad_min" ]]; then
|
||||
echo "ERROR: frank-archon.env missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
|
||||
return 1
|
||||
fi
|
||||
# The image gate is the digest ref itself: a name@sha256 ref exists in
|
||||
# storage only if THAT manifest was pulled. One command — no
|
||||
# pipefail/grep race (the 2026-09-30 lesson).
|
||||
if ! podman image exists "$image"; then
|
||||
echo "ERROR: runtime image absent from storage: $image" >&2
|
||||
echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$model_dir/$model_file" ]]; then
|
||||
echo "ERROR: .rad container missing: $model_dir/$model_file" >&2
|
||||
echo "frank-archon: run scripts/build-frank-archon-rad.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
mib=$(( $(stat -c %s "$model_dir/$model_file") / 1048576 ))
|
||||
if [[ "$mib" -lt "$rad_min" ]]; then
|
||||
echo "ERROR: .rad container too small: $model_dir/$model_file is ${mib} MiB, want >= $rad_min" >&2
|
||||
echo "frank-archon: rebuild it (scripts/build-frank-archon-rad.sh --force)" >&2
|
||||
echo "frank-archon: env: missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
|
||||
echo " hint: scripts/frank-archon.env" >&2
|
||||
return 1
|
||||
fi
|
||||
require_image "$image" \
|
||||
"pull it: podman pull docker://stilldeadcode/radiance:latest (as dank)" \
|
||||
|| return 1
|
||||
require_min_mib "$model_dir/$model_file" ".rad" "$rad_min" \
|
||||
"convert it: scripts/convert-swift15-w4dn8.sh (rad-info is the record)" \
|
||||
|| return 1
|
||||
|
||||
# A stale container with the same name makes `podman run --name` fail at
|
||||
# start; remove any leftover from a failed previous attempt.
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
# Both sides source this library, so the refusal is one text and one rule.
|
||||
#
|
||||
# Contract (public interface):
|
||||
# require_exec PATH LABEL HINT
|
||||
# require_file PATH LABEL HINT
|
||||
# require_bytes PATH LABEL WANT_BYTES HINT
|
||||
# require_min_mib PATH LABEL MIN_MIB HINT
|
||||
@@ -20,7 +21,12 @@
|
||||
# scripts run under `set -e` (so a refusal is terminal), while the deployer
|
||||
# wants `|| return 1` so a failed gate reports a red deploy for that one
|
||||
# service and the dispatcher's state marker stays pending.
|
||||
#
|
||||
# Lines are labelled by ARTIFACT CLASS, and the class is the whole label
|
||||
# (the path rides in the same line): the class + path is what an operator
|
||||
# acts on, so no wording order and no payload rename can make the two twins
|
||||
# disagree. The deployer's old "halogen vision tower" and the serve script's
|
||||
# "vision-tower" were one gate in two vocabularies that no grep could join.
|
||||
|
||||
# The size law is equality, not a floor, for `require_bytes`: a hub fetch
|
||||
# that truncates is the fleet's standing hazard (curl exits 0 on 401/403 and
|
||||
# writes the error body as the artifact), and the declared byte count is a
|
||||
@@ -29,6 +35,14 @@
|
||||
# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon).
|
||||
: "${GATE_PREFIX:=gate}"
|
||||
|
||||
# Two bash laws make the `return 1` at every call site load-bearing:
|
||||
# * a function whose body ends inside an `if` returns the CONDITION's
|
||||
# status (true = 0), so a gate that ends on its own refusal prints the
|
||||
# refusal and reports SUCCESS;
|
||||
# * `set -e` is suspended for any command in a `… || …` chain, so a gate
|
||||
# that runs inside one must not be trusted to abort the caller.
|
||||
# Every gate's refusal therefore carries an explicit `|| return 1`, and the
|
||||
# caller consumes the status (`|| exit 1`, `|| return 1`).
|
||||
_gate_fail() {
|
||||
printf '%s: %s\n' "$GATE_PREFIX" "$1" >&2
|
||||
printf ' hint: %s\n' "$2" >&2
|
||||
@@ -41,8 +55,21 @@ _gate_fail() {
|
||||
require_file() {
|
||||
local path="$1" label="$2" hint="$3"
|
||||
if [[ -L "$path" || ! -f "$path" ]]; then
|
||||
_gate_fail "$label: missing at $path" "$hint"
|
||||
_gate_fail "$label: missing at $path" "$hint" || return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# require_exec — PATH must be an executable regular file. The native runtimes
|
||||
# (llama.cpp's llama-server) are binaries the operator builds; a tree that
|
||||
# compiled but lost its +x, or a directory wearing the binary's name, must
|
||||
# refuse the same way a missing one does.
|
||||
require_exec() {
|
||||
local path="$1" label="$2" hint="$3"
|
||||
if [[ -L "$path" || ! -x "$path" ]]; then
|
||||
_gate_fail "$label: missing at $path" "$hint" || return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# require_bytes — PATH must exist AND be exactly WANT_BYTES bytes.
|
||||
@@ -51,8 +78,9 @@ require_bytes() {
|
||||
require_file "$path" "$label" "$hint" || return 1
|
||||
got="$(stat -c %s "$path")"
|
||||
if [[ "$got" != "$want" ]]; then
|
||||
_gate_fail "$label: is $got bytes, expected $want" "$hint"
|
||||
_gate_fail "$label: is $got bytes, expected $want" "$hint" || return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# require_min_mib — PATH must be at least MIN_MIB mebibytes. The floor form
|
||||
@@ -63,8 +91,9 @@ require_min_mib() {
|
||||
require_file "$path" "$label" "$hint" || return 1
|
||||
mib=$(( $(stat -c %s "$path") / 1048576 ))
|
||||
if (( mib < min )); then
|
||||
_gate_fail "$label: is $mib MiB, expected >= $min MiB" "$hint"
|
||||
_gate_fail "$label: is $mib MiB, expected >= $min MiB" "$hint" || return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# require_dir_files — DIR must be a directory and every named member a
|
||||
@@ -77,13 +106,14 @@ require_dir_files() {
|
||||
shift 3
|
||||
local f
|
||||
if [[ -L "$dir" || ! -d "$dir" ]]; then
|
||||
_gate_fail "$label: missing at $dir" "$hint"
|
||||
_gate_fail "$label: missing at $dir" "$hint" || return 1
|
||||
fi
|
||||
for f in "$@"; do
|
||||
if [[ -L "$dir/$f" || ! -f "$dir/$f" ]]; then
|
||||
_gate_fail "$label: $dir/$f is missing or a symlink" "$hint"
|
||||
_gate_fail "$label: $dir/$f is missing or a symlink" "$hint" || return 1
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# require_image — the image ref must be in podman storage. A name@sha256 ref
|
||||
@@ -92,6 +122,7 @@ require_dir_files() {
|
||||
require_image() {
|
||||
local ref="$1" hint="$2"
|
||||
if ! podman image exists "$ref"; then
|
||||
_gate_fail "image: $ref is not in podman storage" "$hint"
|
||||
_gate_fail "image: $ref is not in podman storage" "$hint" || return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -79,10 +79,10 @@ require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_CHECKPOINT" \
|
||||
checkpoint "$FRANK_DRAGOON_CHECKPOINT_BYTES" \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1
|
||||
require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_NGRAM_TABLE" \
|
||||
n-gram-table "$FRANK_DRAGOON_NGRAM_BYTES" \
|
||||
n-gram "$FRANK_DRAGOON_NGRAM_BYTES" \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1
|
||||
require_bytes "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_VISION_TOWER" \
|
||||
vision-tower "$FRANK_DRAGOON_VISION_BYTES" \
|
||||
vision "$FRANK_DRAGOON_VISION_BYTES" \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh" || exit 1
|
||||
|
||||
# The payload ships the tokenizer as a FLAT DIRECTORY. A HuggingFace cache
|
||||
|
||||
Reference in New Issue
Block a user