feat(scripts): one artifact-gate library for the LLM endpoints
The serve scripts and the deployer each hand-wrote the same missing/truncated artifact gates; the 2026-10-01 crash-loop was those twins disagreeing mid-cutover. scripts/lib/llm-gates.sh is the single place the refusal lives: require_file / require_bytes / require_min_mib / require_dir_files / require_image, each returning 1 and naming the endpoint, the artifact and the remedy. New seam tests/llm_gates.bats pins the contract at the arguments-in / status-and-stderr-out boundary.
This commit is contained in:
Frank
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# llm-gates.sh — the artifact gates every LLM endpoint runs before it starts.
|
||||
#
|
||||
# Why this file exists: the fleet law from the 2026-10-01 crash-loop is that
|
||||
# a serve script and the deployer must refuse the SAME missing/truncated
|
||||
# artifact, the SAME way. When the two gates were hand-written twins they
|
||||
# drifted mid-cutover — green deploy, red unit, every time the payload moved.
|
||||
# Both sides source this library, so the refusal is one text and one rule.
|
||||
#
|
||||
# Contract (public interface):
|
||||
# require_file PATH LABEL HINT
|
||||
# require_bytes PATH LABEL WANT_BYTES HINT
|
||||
# require_min_mib PATH LABEL MIN_MIB HINT
|
||||
# require_dir_files DIR LABEL HINT FILE...
|
||||
# require_image REF HINT
|
||||
#
|
||||
# Each returns 0 and prints nothing on success. On failure it prints two
|
||||
# lines to stderr — the first naming the endpoint ($GATE_PREFIX) and the
|
||||
# artifact, the second the remedy — and returns 1. It NEVER exits: the serve
|
||||
# scripts run under `set -e` (so a refusal is terminal), while the deployer
|
||||
# wants `|| return 1` so a failed gate reports a red deploy for that one
|
||||
# service and the dispatcher's state marker stays pending.
|
||||
#
|
||||
# The size law is equality, not a floor, for `require_bytes`: a hub fetch
|
||||
# that truncates is the fleet's standing hazard (curl exits 0 on 401/403 and
|
||||
# writes the error body as the artifact), and the declared byte count is a
|
||||
# contract in both directions — a bigger file is a DIFFERENT artifact.
|
||||
|
||||
# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon).
|
||||
: "${GATE_PREFIX:=gate}"
|
||||
|
||||
_gate_fail() {
|
||||
printf '%s: %s\n' "$GATE_PREFIX" "$1" >&2
|
||||
printf ' hint: %s\n' "$2" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# require_file — PATH must be a regular file. A directory (an unpacked
|
||||
# staging dir wearing the payload's name) and a symlink (a HuggingFace cache
|
||||
# snapshot's entry, which dangles inside the container mount) both refuse.
|
||||
require_file() {
|
||||
local path="$1" label="$2" hint="$3"
|
||||
if [[ -L "$path" || ! -f "$path" ]]; then
|
||||
_gate_fail "$label: missing at $path" "$hint"
|
||||
fi
|
||||
}
|
||||
|
||||
# require_bytes — PATH must exist AND be exactly WANT_BYTES bytes.
|
||||
require_bytes() {
|
||||
local path="$1" label="$2" want="$3" hint="$4" got
|
||||
require_file "$path" "$label" "$hint" || return 1
|
||||
got="$(stat -c %s "$path")"
|
||||
if [[ "$got" != "$want" ]]; then
|
||||
_gate_fail "$label: is $got bytes, expected $want" "$hint"
|
||||
fi
|
||||
}
|
||||
|
||||
# require_min_mib — PATH must be at least MIN_MIB mebibytes. The floor form
|
||||
# is for build products (a .rad container is a range of legal sizes; a
|
||||
# truncated convert is not).
|
||||
require_min_mib() {
|
||||
local path="$1" label="$2" min="$3" hint="$4" mib
|
||||
require_file "$path" "$label" "$hint" || return 1
|
||||
mib=$(( $(stat -c %s "$path") / 1048576 ))
|
||||
if (( mib < min )); then
|
||||
_gate_fail "$label: is $mib MiB, expected >= $min MiB" "$hint"
|
||||
fi
|
||||
}
|
||||
|
||||
# require_dir_files — DIR must be a directory and every named member a
|
||||
# regular file (the flat-tokenizer law: snapshot symlinks dangle in the
|
||||
# container mount even when the directory itself is real).
|
||||
require_dir_files() {
|
||||
local dir="$1" label="$2" hint="$3"
|
||||
shift 3
|
||||
local f
|
||||
if [[ ! -d "$dir" ]]; then
|
||||
_gate_fail "$label: missing at $dir" "$hint"
|
||||
fi
|
||||
for f in "$@"; do
|
||||
if [[ -L "$dir/$f" || ! -f "$dir/$f" ]]; then
|
||||
_gate_fail "$label: $dir/$f is missing or a symlink" "$hint"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# require_image — the image ref must be in podman storage. A name@sha256 ref
|
||||
# exists only if THAT manifest was pulled, so the gate is the ref itself, in
|
||||
# one command: no pipefail/grep race (the 2026-09-30 lesson).
|
||||
require_image() {
|
||||
local ref="$1" hint="$2"
|
||||
if ! podman image exists "$ref"; then
|
||||
_gate_fail "image: $ref is not in podman storage" "$hint"
|
||||
fi
|
||||
}
|
||||
@@ -4,6 +4,12 @@
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
# skip_if_no_podman — skip the test when podman is absent (the image gate
|
||||
# asks real storage a real question)
|
||||
skip_if_no_podman() {
|
||||
command -v podman >/dev/null 2>&1 || skip "podman is not installed"
|
||||
}
|
||||
|
||||
# yaml_parse_ok <file> — assert a compose file parses as valid YAML
|
||||
yaml_parse_ok() {
|
||||
python3 - "$1" <<'PYEOF'
|
||||
|
||||
Executable
+163
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the shared artifact gates every LLM endpoint runs before it starts.
|
||||
#
|
||||
# The fleet law these encode: a serve script and the deployer must refuse the
|
||||
# SAME missing/truncated artifact, and they must refuse it the same way,
|
||||
# because the 2026-10-01 crash-loop was exactly those two gates disagreeing
|
||||
# mid-cutover (green deploy, red unit). The library is the one place the
|
||||
# refusal lives; both callers source it.
|
||||
#
|
||||
# Everything here is observable: arguments in, exit status and stderr out.
|
||||
# Nothing inspects the library's internals.
|
||||
|
||||
load helpers
|
||||
|
||||
LIB="$REPO_ROOT/scripts/lib/llm-gates.sh"
|
||||
|
||||
setup() {
|
||||
TMP="$(mktemp -d)"
|
||||
# A fixture with a known, independent size: 12 bytes.
|
||||
printf '0123456789ab' > "$TMP/artifact.bin"
|
||||
mkdir -p "$TMP/flat"
|
||||
touch "$TMP/flat/tokenizer.json" "$TMP/flat/config.json"
|
||||
}
|
||||
|
||||
teardown() {
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
|
||||
# call <invocation> — run one gate in a fresh shell under `set -e`, exactly
|
||||
# as a serve script sources the library, and capture status + both streams.
|
||||
call() {
|
||||
run env GATE_PREFIX="${GATE_PREFIX:-testgate}" PATH="$PATH" \
|
||||
bash -c "set -euo pipefail; source '$LIB'; $1"
|
||||
}
|
||||
|
||||
@test "a present artifact passes the file gate with no output" {
|
||||
GATE_PREFIX=testgate call "require_file '$TMP/artifact.bin' checkpoint 'fetch it'"
|
||||
[ "$status" -eq 0 ]
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "an absent artifact fails the file gate naming the path and the remedy" {
|
||||
call "require_file '$TMP/absent.bin' checkpoint 'run scripts/download-frank-archon-swift15.sh'"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"checkpoint: missing"* ]]
|
||||
[[ "$output" == *"$TMP/absent.bin"* ]]
|
||||
[[ "$output" == *"run scripts/download-frank-archon-swift15.sh"* ]]
|
||||
}
|
||||
|
||||
@test "the file gate refuses a directory where an artifact is required" {
|
||||
# The serve gate is `-f`, not `-e`: a directory named like the payload
|
||||
# (an unpacked staging dir) must not boot the engine.
|
||||
call "require_file '$TMP/flat' checkpoint 'fetch it'"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"checkpoint"* ]]
|
||||
}
|
||||
|
||||
@test "a symlinked artifact is refused: a HF snapshot link dangles in the container mount" {
|
||||
ln -s "$TMP/artifact.bin" "$TMP/linked.bin"
|
||||
call "require_file '$TMP/linked.bin' checkpoint 'fetch it'"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"checkpoint"*"missing"* ]]
|
||||
}
|
||||
|
||||
@test "the byte gate accepts the exact contracted size" {
|
||||
call "require_bytes '$TMP/artifact.bin' checkpoint 12 'fetch it'"
|
||||
[ "$status" -eq 0 ]
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "the byte gate refuses a size mismatch quoting both the real and the expected size" {
|
||||
# The Hub's declared byte count is the contract; curl writes the error
|
||||
# body and exits 0, so a truncation is invisible without this gate.
|
||||
call "require_bytes '$TMP/artifact.bin' checkpoint 42 'run the fetch script'"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"12 bytes"* ]]
|
||||
[[ "$output" == *"expected 42"* ]]
|
||||
[[ "$output" == *"run the fetch script"* ]]
|
||||
}
|
||||
|
||||
@test "the byte gate is an equality contract, not a floor" {
|
||||
# A LARGER file is not a better artifact: it is a different artifact
|
||||
# (the compacted r2 of a checkpoint is byte-exact, not "at least").
|
||||
call "require_bytes '$TMP/artifact.bin' checkpoint 6 'fetch it'"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"expected 6"* ]]
|
||||
}
|
||||
|
||||
@test "the minimum-size gate accepts the threshold itself" {
|
||||
dd if=/dev/zero of="$TMP/two-mib.rad" bs=1048576 count=2 status=none
|
||||
call "require_min_mib '$TMP/two-mib.rad' container 2 'rebuild it'"
|
||||
[ "$status" -eq 0 ]
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "the minimum-size gate refuses below the threshold naming both sizes" {
|
||||
dd if=/dev/zero of="$TMP/one-mib.rad" bs=1048576 count=1 status=none
|
||||
call "require_min_mib '$TMP/one-mib.rad' container 2 'rebuild it'"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"1 MiB"* ]]
|
||||
[[ "$output" == *"2 MiB"* ]]
|
||||
[[ "$output" == *"rebuild it"* ]]
|
||||
}
|
||||
|
||||
@test "the flat-directory gate accepts a directory of regular files" {
|
||||
call "require_dir_files '$TMP/flat' tokenizer 'fetch the tokenizer' tokenizer.json config.json"
|
||||
[ "$status" -eq 0 ]
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "the flat-directory gate names the member that is absent" {
|
||||
call "require_dir_files '$TMP/flat' tokenizer 'fetch the tokenizer' tokenizer.json vocab.json"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"tokenizer"*"missing"* ]]
|
||||
[[ "$output" == *"vocab.json"* ]]
|
||||
}
|
||||
|
||||
@test "the flat-directory gate refuses a symlinked member" {
|
||||
# A HuggingFace cache snapshot's entries are symlinks into blobs/ and
|
||||
# dangle inside the container mount even when the directory is real.
|
||||
ln -s /nowhere/x "$TMP/flat/vocab.json"
|
||||
call "require_dir_files '$TMP/flat' tokenizer 'fetch the tokenizer' tokenizer.json vocab.json"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"vocab.json"*"missing or a symlink"* ]]
|
||||
}
|
||||
|
||||
@test "the flat-directory gate refuses a missing directory" {
|
||||
call "require_dir_files '$TMP/nope' tokenizer 'fetch the tokenizer' tokenizer.json"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"tokenizer: missing"* ]]
|
||||
}
|
||||
|
||||
@test "the image gate refuses a ref the storage does not hold" {
|
||||
# A name@sha256 ref exists only if THAT manifest was pulled, so the
|
||||
# gate is the ref itself — one command, no grep race.
|
||||
skip_if_no_podman
|
||||
call "require_image 'example.invalid/nothing@sha256:0000000000000000000000000000000000000000000000000000000000000000' 'podman pull it'"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"not in podman storage"* ]]
|
||||
[[ "$output" == *"podman pull it"* ]]
|
||||
}
|
||||
|
||||
@test "every failure names the endpoint the operator is standing in front of" {
|
||||
GATE_PREFIX=frank-dragoon call "require_file '$TMP/absent.bin' checkpoint 'fetch it'"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == frank-dragoon:* ]]
|
||||
}
|
||||
|
||||
@test "a failing gate stops the caller from reaching the runtime" {
|
||||
# The composition property that matters: the gate is not advisory. A
|
||||
# serve script that calls it and then execs the engine must die first.
|
||||
cat > "$TMP/serve-like.sh" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
source '$LIB'
|
||||
require_file '$TMP/absent.bin' checkpoint 'fetch it'
|
||||
echo RUNTIME-EXECUTED
|
||||
EOF
|
||||
run bash "$TMP/serve-like.sh"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" != *"RUNTIME-EXECUTED"* ]]
|
||||
[[ "$output" == *"checkpoint: missing"* ]]
|
||||
}
|
||||
Reference in New Issue
Block a user