refact(deployer): the LLM handlers gate through the shared library
deploy_llm_dragoon/zealot/arkon now call require_image / require_bytes / require_min_mib / require_file / require_dir_files, so the deployer and the serve scripts refuse the same missing payload with the same words — the 2026-10-01 crash-loop law, enforced by construction instead of by duplicated shell. require_exec joins the library for the native-runtime gate (a compiled tree that lost its +x refuses like a missing one). relink_unit takes SYSTEMD_SYSTEM_DIR so a handler can be driven end to end in a sandbox. The library's own refusals now carry their status: a bash function that ends inside an if returns the CONDITION's status, so a gate whose last statement is its refusal printed the message and returned SUCCESS — the new seam caught it, and the gate tests pin it. New seam tests/llm_deploy_gates.bash + tests/stubs/: deploy-remote.sh's LLM handlers EXECUTED against stub systemctl/podman/sudo/ss/curl, asserting exit status, refusal text, no restart, and the pending state marker.
This commit is contained in:
Frank committed
1 parent
cb25796a6b
commit
c5a816e6a8
12 files changed
+403
-97
No files matched your search
+60
-86
@@ -9,6 +9,13 @@
|
||||
# pending until its service is explicitly deployed.
|
||||
set -euo pipefail
|
||||
|
||||
# The LLM handlers' artifact gates are the serve scripts' own library, so
|
||||
# the deployer and the unit refuse the same missing payload with the same
|
||||
# words (the fleet law from the 2026-10-01 crash-loop). GATE_PREFIX is set
|
||||
# per handler.
|
||||
# shellcheck disable=SC1091
|
||||
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/llm-gates.sh"
|
||||
|
||||
TARGET="${1:-}"
|
||||
REVISION="${2:-}"
|
||||
shift 2 || true
|
||||
@@ -721,9 +728,15 @@ deploy_gitea_unit() {
|
||||
systemctl --user restart "$service.service"
|
||||
}
|
||||
|
||||
# The root units are symlinks from the systemd system directory into the
|
||||
# checkout. SYSTEMD_SYSTEM_DIR is production (/etc/systemd/system); the knob
|
||||
# exists so the deploy seam can run a whole handler — relink included — in a
|
||||
# sandbox and read the unit links as evidence, unprivileged.
|
||||
SYSTEMD_SYSTEM_DIR="${SYSTEMD_SYSTEM_DIR:-/etc/systemd/system}"
|
||||
|
||||
relink_unit() {
|
||||
local service="$1"
|
||||
run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service"
|
||||
run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "$SYSTEMD_SYSTEM_DIR/$service.service"
|
||||
}
|
||||
|
||||
# Authentik runs as three boot-persistent Quadlet units (postgres -> server ->
|
||||
@@ -1155,51 +1168,24 @@ wait_port_free() {
|
||||
}
|
||||
|
||||
deploy_llm_dragoon() {
|
||||
# Refuse rather than restart-loop; every gate below has a twin in
|
||||
# serve-frank-dragoon.sh and they must agree (2026-10-01 lesson).
|
||||
# Capture podman output before grepping: `podman images | grep -q`
|
||||
# races under pipefail (SIGPIPE 141 becomes "no match", 2026-09-30).
|
||||
if ! grep -qF "${LLM_DRAGOON_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then
|
||||
echo "ERROR: halogen image absent: $LLM_DRAGOON_IMAGE" >&2
|
||||
echo "frank-dragoon: pull it: podman pull ${LLM_DRAGOON_IMAGE%@*}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$LLM_DRAGOON_CKPT" ]]; then
|
||||
echo "ERROR: halogen checkpoint absent at $LLM_DRAGOON_CKPT" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(stat -c %s "$LLM_DRAGOON_CKPT")" != "$LLM_DRAGOON_CKPT_BYTES" ]]; then
|
||||
echo "ERROR: halogen checkpoint is not $LLM_DRAGOON_CKPT_BYTES bytes (partial download?)" >&2
|
||||
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
|
||||
return 1
|
||||
fi
|
||||
for gate in "$LLM_DRAGOON_NGRAM:$LLM_DRAGOON_NGRAM_BYTES:n-gram table" \
|
||||
"$LLM_DRAGOON_VISION:$LLM_DRAGOON_VISION_BYTES:vision tower"; do
|
||||
IFS=: read -r gpath gbytes gname <<<"$gate"
|
||||
if [[ ! -f "$gpath" ]]; then
|
||||
echo "ERROR: halogen $gname absent at $gpath" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(stat -c %s "$gpath")" != "$gbytes" ]]; then
|
||||
echo "ERROR: halogen $gname is not $gbytes bytes (partial download?)" >&2
|
||||
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
if [[ -L "$LLM_DRAGOON_TOK_DIR" || ! -d "$LLM_DRAGOON_TOK_DIR" ]]; then
|
||||
echo "ERROR: tokenizer dir missing or a symlink at $LLM_DRAGOON_TOK_DIR" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
for tf in $LLM_DRAGOON_TOK_FILES; do
|
||||
if [[ ! -f "$LLM_DRAGOON_TOK_DIR/$tf" || -L "$LLM_DRAGOON_TOK_DIR/$tf" ]]; then
|
||||
echo "ERROR: tokenizer file missing or a symlink: $LLM_DRAGOON_TOK_DIR/$tf" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
# Refuse rather than restart-loop; every gate here is the SAME library
|
||||
# call the serve script makes, so the two twins cannot drift (the
|
||||
# 2026-10-01 crash-loop lesson).
|
||||
# Capture podman output before grepping is moot: require_image is one
|
||||
# command, no pipe (the 2026-09-30 SIGPIPE lesson).
|
||||
GATE_PREFIX=frank-dragoon
|
||||
require_image "$LLM_DRAGOON_IMAGE" \
|
||||
"pull it: podman pull docker://${LLM_DRAGOON_IMAGE%@*}" || return 1
|
||||
require_bytes "$LLM_DRAGOON_CKPT" checkpoint "$LLM_DRAGOON_CKPT_BYTES" \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
|
||||
require_bytes "$LLM_DRAGOON_NGRAM" n-gram "$LLM_DRAGOON_NGRAM_BYTES" \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
|
||||
require_bytes "$LLM_DRAGOON_VISION" vision "$LLM_DRAGOON_VISION_BYTES" \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh; re-check: --verify-only" || return 1
|
||||
# shellcheck disable=SC2086
|
||||
require_dir_files "$LLM_DRAGOON_TOK_DIR" tokenizer \
|
||||
"fetch it: bash scripts/download-frank-dragoon-flashnext.sh" \
|
||||
$LLM_DRAGOON_TOK_FILES || return 1
|
||||
|
||||
retire_halogen_dragoon_quadlet
|
||||
# The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
|
||||
@@ -1243,22 +1229,17 @@ LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K
|
||||
deploy_llm_zealot() {
|
||||
# Refuse rather than restart-loop: a unit whose runtime or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
if [[ ! -x "$LLM_ZEALOT_RUNTIME" ]]; then
|
||||
echo "ERROR: llama.cpp runtime absent at $LLM_ZEALOT_RUNTIME" >&2
|
||||
echo "frank-zealot: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
|
||||
echo "frank-zealot: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
|
||||
echo "frank-zealot: && cmake --build build-cuda --target llama-server) first." >&2
|
||||
return 1
|
||||
fi
|
||||
for f in "$LLM_ZEALOT_MODEL_FILE" "$LLM_ZEALOT_MMPROJ"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: frank-zealot payload file absent at $f" >&2
|
||||
echo "frank-zealot: run scripts/download-frank-zealot.sh first;" >&2
|
||||
echo "frank-zealot: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
# endpoint dark with nothing in the log to say why. Same library calls
|
||||
# as serve-frank-zealot.sh (the 2026-10-01 law).
|
||||
GATE_PREFIX=frank-zealot
|
||||
require_exec "$LLM_ZEALOT_RUNTIME" runtime \
|
||||
"build it: cmake -B build-cuda -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86" \
|
||||
|| return 1
|
||||
require_file "$LLM_ZEALOT_MODEL_FILE" gguf \
|
||||
"fetch it: bash scripts/download-frank-zealot.sh" || return 1
|
||||
require_file "$LLM_ZEALOT_MMPROJ" mmproj \
|
||||
"fetch it: bash scripts/download-frank-zealot.sh" || return 1
|
||||
|
||||
relink_unit frank-zealot
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-zealot.service
|
||||
@@ -1300,11 +1281,16 @@ LLM_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-pr
|
||||
deploy_llm_archon() {
|
||||
# Refuse rather than restart-loop: a unit whose image or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
local env_file="/home/dank/serverAppConfigs/scripts/frank-archon.env"
|
||||
local image model_dir model_file rad_min mib
|
||||
# endpoint dark with nothing in the log to say why. The gates are the
|
||||
# shared library's, called with the values the unit's own env file
|
||||
# declares — so serve-frank-archon.sh and this handler cannot disagree,
|
||||
# and the payload path lives in exactly one file.
|
||||
GATE_PREFIX=frank-archon
|
||||
local env_file="${LLM_ARCHON_ENV_FILE:-$REMOTE_DIR/scripts/frank-archon.env}"
|
||||
local image model_dir model_file rad_min
|
||||
if [[ ! -f "$env_file" ]]; then
|
||||
echo "ERROR: frank-archon env file absent: $env_file" >&2
|
||||
echo "frank-archon: env: missing at $env_file" >&2
|
||||
echo " hint: deploy the repo checkout first ($REMOTE_DIR)" >&2
|
||||
return 1
|
||||
fi
|
||||
image="$(sed -n 's/^FRANK_ARCHON_IMAGE=//p' "$env_file" | tail -1)"
|
||||
@@ -1312,28 +1298,16 @@ deploy_llm_archon() {
|
||||
model_file="$(sed -n 's/^FRANK_ARCHON_MODEL_FILE=//p' "$env_file" | tail -1)"
|
||||
rad_min="$(sed -n 's/^FRANK_ARCHON_RAD_MIN_MIB=//p' "$env_file" | tail -1)"
|
||||
if [[ -z "$image" || -z "$model_dir" || -z "$model_file" || -z "$rad_min" ]]; then
|
||||
echo "ERROR: frank-archon.env missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
|
||||
return 1
|
||||
fi
|
||||
# The image gate is the digest ref itself: a name@sha256 ref exists in
|
||||
# storage only if THAT manifest was pulled. One command — no
|
||||
# pipefail/grep race (the 2026-09-30 lesson).
|
||||
if ! podman image exists "$image"; then
|
||||
echo "ERROR: runtime image absent from storage: $image" >&2
|
||||
echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$model_dir/$model_file" ]]; then
|
||||
echo "ERROR: .rad container missing: $model_dir/$model_file" >&2
|
||||
echo "frank-archon: run scripts/build-frank-archon-rad.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
mib=$(( $(stat -c %s "$model_dir/$model_file") / 1048576 ))
|
||||
if [[ "$mib" -lt "$rad_min" ]]; then
|
||||
echo "ERROR: .rad container too small: $model_dir/$model_file is ${mib} MiB, want >= $rad_min" >&2
|
||||
echo "frank-archon: rebuild it (scripts/build-frank-archon-rad.sh --force)" >&2
|
||||
echo "frank-archon: env: missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
|
||||
echo " hint: scripts/frank-archon.env" >&2
|
||||
return 1
|
||||
fi
|
||||
require_image "$image" \
|
||||
"pull it: podman pull docker://stilldeadcode/radiance:latest (as dank)" \
|
||||
|| return 1
|
||||
require_min_mib "$model_dir/$model_file" ".rad" "$rad_min" \
|
||||
"convert it: scripts/convert-swift15-w4dn8.sh (rad-info is the record)" \
|
||||
|| return 1
|
||||
|
||||
# A stale container with the same name makes `podman run --name` fail at
|
||||
# start; remove any leftover from a failed previous attempt.
|
||||
|
||||
Reference in new issue
Block a user