diff --git a/scripts/lib/llm-gates.sh b/scripts/lib/llm-gates.sh new file mode 100755 index 0000000..d6224ab --- /dev/null +++ b/scripts/lib/llm-gates.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# llm-gates.sh — the artifact gates every LLM endpoint runs before it starts. +# +# Why this file exists: the fleet law from the 2026-10-01 crash-loop is that +# a serve script and the deployer must refuse the SAME missing/truncated +# artifact, the SAME way. When the two gates were hand-written twins they +# drifted mid-cutover — green deploy, red unit, every time the payload moved. +# Both sides source this library, so the refusal is one text and one rule. +# +# Contract (public interface): +# require_file PATH LABEL HINT +# require_bytes PATH LABEL WANT_BYTES HINT +# require_min_mib PATH LABEL MIN_MIB HINT +# require_dir_files DIR LABEL HINT FILE... +# require_image REF HINT +# +# Each returns 0 and prints nothing on success. On failure it prints two +# lines to stderr — the first naming the endpoint ($GATE_PREFIX) and the +# artifact, the second the remedy — and returns 1. It NEVER exits: the serve +# scripts run under `set -e` (so a refusal is terminal), while the deployer +# wants `|| return 1` so a failed gate reports a red deploy for that one +# service and the dispatcher's state marker stays pending. +# +# The size law is equality, not a floor, for `require_bytes`: a hub fetch +# that truncates is the fleet's standing hazard (curl exits 0 on 401/403 and +# writes the error body as the artifact), and the declared byte count is a +# contract in both directions — a bigger file is a DIFFERENT artifact. + +# GATE_PREFIX names the endpoint in every refusal (e.g. frank-dragoon). +: "${GATE_PREFIX:=gate}" + +_gate_fail() { + printf '%s: %s\n' "$GATE_PREFIX" "$1" >&2 + printf ' hint: %s\n' "$2" >&2 + return 1 +} + +# require_file — PATH must be a regular file. A directory (an unpacked +# staging dir wearing the payload's name) and a symlink (a HuggingFace cache +# snapshot's entry, which dangles inside the container mount) both refuse. +require_file() { + local path="$1" label="$2" hint="$3" + if [[ -L "$path" || ! -f "$path" ]]; then + _gate_fail "$label: missing at $path" "$hint" + fi +} + +# require_bytes — PATH must exist AND be exactly WANT_BYTES bytes. +require_bytes() { + local path="$1" label="$2" want="$3" hint="$4" got + require_file "$path" "$label" "$hint" || return 1 + got="$(stat -c %s "$path")" + if [[ "$got" != "$want" ]]; then + _gate_fail "$label: is $got bytes, expected $want" "$hint" + fi +} + +# require_min_mib — PATH must be at least MIN_MIB mebibytes. The floor form +# is for build products (a .rad container is a range of legal sizes; a +# truncated convert is not). +require_min_mib() { + local path="$1" label="$2" min="$3" hint="$4" mib + require_file "$path" "$label" "$hint" || return 1 + mib=$(( $(stat -c %s "$path") / 1048576 )) + if (( mib < min )); then + _gate_fail "$label: is $mib MiB, expected >= $min MiB" "$hint" + fi +} + +# require_dir_files — DIR must be a directory and every named member a +# regular file (the flat-tokenizer law: snapshot symlinks dangle in the +# container mount even when the directory itself is real). +require_dir_files() { + local dir="$1" label="$2" hint="$3" + shift 3 + local f + if [[ ! -d "$dir" ]]; then + _gate_fail "$label: missing at $dir" "$hint" + fi + for f in "$@"; do + if [[ -L "$dir/$f" || ! -f "$dir/$f" ]]; then + _gate_fail "$label: $dir/$f is missing or a symlink" "$hint" + fi + done +} + +# require_image — the image ref must be in podman storage. A name@sha256 ref +# exists only if THAT manifest was pulled, so the gate is the ref itself, in +# one command: no pipefail/grep race (the 2026-09-30 lesson). +require_image() { + local ref="$1" hint="$2" + if ! podman image exists "$ref"; then + _gate_fail "image: $ref is not in podman storage" "$hint" + fi +} diff --git a/tests/helpers.bash b/tests/helpers.bash index 4fc0ad0..da0cbcd 100644 --- a/tests/helpers.bash +++ b/tests/helpers.bash @@ -4,6 +4,12 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# skip_if_no_podman — skip the test when podman is absent (the image gate +# asks real storage a real question) +skip_if_no_podman() { + command -v podman >/dev/null 2>&1 || skip "podman is not installed" +} + # yaml_parse_ok — assert a compose file parses as valid YAML yaml_parse_ok() { python3 - "$1" <<'PYEOF' diff --git a/tests/llm_gates.bats b/tests/llm_gates.bats new file mode 100755 index 0000000..dbba811 --- /dev/null +++ b/tests/llm_gates.bats @@ -0,0 +1,163 @@ +#!/usr/bin/env bats +# Seam: the shared artifact gates every LLM endpoint runs before it starts. +# +# The fleet law these encode: a serve script and the deployer must refuse the +# SAME missing/truncated artifact, and they must refuse it the same way, +# because the 2026-10-01 crash-loop was exactly those two gates disagreeing +# mid-cutover (green deploy, red unit). The library is the one place the +# refusal lives; both callers source it. +# +# Everything here is observable: arguments in, exit status and stderr out. +# Nothing inspects the library's internals. + +load helpers + +LIB="$REPO_ROOT/scripts/lib/llm-gates.sh" + +setup() { + TMP="$(mktemp -d)" + # A fixture with a known, independent size: 12 bytes. + printf '0123456789ab' > "$TMP/artifact.bin" + mkdir -p "$TMP/flat" + touch "$TMP/flat/tokenizer.json" "$TMP/flat/config.json" +} + +teardown() { + rm -rf "$TMP" +} + +# call — run one gate in a fresh shell under `set -e`, exactly +# as a serve script sources the library, and capture status + both streams. +call() { + run env GATE_PREFIX="${GATE_PREFIX:-testgate}" PATH="$PATH" \ + bash -c "set -euo pipefail; source '$LIB'; $1" +} + +@test "a present artifact passes the file gate with no output" { + GATE_PREFIX=testgate call "require_file '$TMP/artifact.bin' checkpoint 'fetch it'" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "an absent artifact fails the file gate naming the path and the remedy" { + call "require_file '$TMP/absent.bin' checkpoint 'run scripts/download-frank-archon-swift15.sh'" + [ "$status" -ne 0 ] + [[ "$output" == *"checkpoint: missing"* ]] + [[ "$output" == *"$TMP/absent.bin"* ]] + [[ "$output" == *"run scripts/download-frank-archon-swift15.sh"* ]] +} + +@test "the file gate refuses a directory where an artifact is required" { + # The serve gate is `-f`, not `-e`: a directory named like the payload + # (an unpacked staging dir) must not boot the engine. + call "require_file '$TMP/flat' checkpoint 'fetch it'" + [ "$status" -ne 0 ] + [[ "$output" == *"checkpoint"* ]] +} + +@test "a symlinked artifact is refused: a HF snapshot link dangles in the container mount" { + ln -s "$TMP/artifact.bin" "$TMP/linked.bin" + call "require_file '$TMP/linked.bin' checkpoint 'fetch it'" + [ "$status" -ne 0 ] + [[ "$output" == *"checkpoint"*"missing"* ]] +} + +@test "the byte gate accepts the exact contracted size" { + call "require_bytes '$TMP/artifact.bin' checkpoint 12 'fetch it'" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "the byte gate refuses a size mismatch quoting both the real and the expected size" { + # The Hub's declared byte count is the contract; curl writes the error + # body and exits 0, so a truncation is invisible without this gate. + call "require_bytes '$TMP/artifact.bin' checkpoint 42 'run the fetch script'" + [ "$status" -ne 0 ] + [[ "$output" == *"12 bytes"* ]] + [[ "$output" == *"expected 42"* ]] + [[ "$output" == *"run the fetch script"* ]] +} + +@test "the byte gate is an equality contract, not a floor" { + # A LARGER file is not a better artifact: it is a different artifact + # (the compacted r2 of a checkpoint is byte-exact, not "at least"). + call "require_bytes '$TMP/artifact.bin' checkpoint 6 'fetch it'" + [ "$status" -ne 0 ] + [[ "$output" == *"expected 6"* ]] +} + +@test "the minimum-size gate accepts the threshold itself" { + dd if=/dev/zero of="$TMP/two-mib.rad" bs=1048576 count=2 status=none + call "require_min_mib '$TMP/two-mib.rad' container 2 'rebuild it'" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "the minimum-size gate refuses below the threshold naming both sizes" { + dd if=/dev/zero of="$TMP/one-mib.rad" bs=1048576 count=1 status=none + call "require_min_mib '$TMP/one-mib.rad' container 2 'rebuild it'" + [ "$status" -ne 0 ] + [[ "$output" == *"1 MiB"* ]] + [[ "$output" == *"2 MiB"* ]] + [[ "$output" == *"rebuild it"* ]] +} + +@test "the flat-directory gate accepts a directory of regular files" { + call "require_dir_files '$TMP/flat' tokenizer 'fetch the tokenizer' tokenizer.json config.json" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "the flat-directory gate names the member that is absent" { + call "require_dir_files '$TMP/flat' tokenizer 'fetch the tokenizer' tokenizer.json vocab.json" + [ "$status" -ne 0 ] + [[ "$output" == *"tokenizer"*"missing"* ]] + [[ "$output" == *"vocab.json"* ]] +} + +@test "the flat-directory gate refuses a symlinked member" { + # A HuggingFace cache snapshot's entries are symlinks into blobs/ and + # dangle inside the container mount even when the directory is real. + ln -s /nowhere/x "$TMP/flat/vocab.json" + call "require_dir_files '$TMP/flat' tokenizer 'fetch the tokenizer' tokenizer.json vocab.json" + [ "$status" -ne 0 ] + [[ "$output" == *"vocab.json"*"missing or a symlink"* ]] +} + +@test "the flat-directory gate refuses a missing directory" { + call "require_dir_files '$TMP/nope' tokenizer 'fetch the tokenizer' tokenizer.json" + [ "$status" -ne 0 ] + [[ "$output" == *"tokenizer: missing"* ]] +} + +@test "the image gate refuses a ref the storage does not hold" { + # A name@sha256 ref exists only if THAT manifest was pulled, so the + # gate is the ref itself — one command, no grep race. + skip_if_no_podman + call "require_image 'example.invalid/nothing@sha256:0000000000000000000000000000000000000000000000000000000000000000' 'podman pull it'" + [ "$status" -ne 0 ] + [[ "$output" == *"not in podman storage"* ]] + [[ "$output" == *"podman pull it"* ]] +} + +@test "every failure names the endpoint the operator is standing in front of" { + GATE_PREFIX=frank-dragoon call "require_file '$TMP/absent.bin' checkpoint 'fetch it'" + [ "$status" -ne 0 ] + [[ "$output" == frank-dragoon:* ]] +} + +@test "a failing gate stops the caller from reaching the runtime" { + # The composition property that matters: the gate is not advisory. A + # serve script that calls it and then execs the engine must die first. + cat > "$TMP/serve-like.sh" <