frank-large: host-RAM pin-gate preflight in serve-frank-large.sh
The 2026-10-06 cutover crash-looped frank-large 8 boots (03:09:39-03:11:42): the engine's pin gate refused to start while host MemAvailable was ~74.7 GiB, 3.34 GiB short of the 78.11 GiB it needs (62.11 GiB checkpoint + 16 GiB floor). It self-resolved in ~2 min when the dying 27B engine freed RAM, but left the host with only ~1.8 GiB MemAvailable - a restart would crash-loop again. serve-frank-large.sh now checks MemAvailable against the pin gate BEFORE podman run and WAITS up to FRANK_LARGE_RAM_WAIT (default 600 s) for the RAM to clear, so a transient dip boots once instead of crash-looping; if it never clears it refuses loudly. The 16 GiB floor is the engine's (not HALOGEN_HOST_RESERVE_GIB=20, a separate overall-budget knob). frank_large_unit.bats: FRANK_LARGE_MEMINFO seam + a refuse test. Runbook + log: the pin-gate law, the incident, and the fix.
This commit is contained in:
@@ -59,6 +59,9 @@ good_env() {
|
||||
: > "$TEST_TMP/empty.env"
|
||||
touch "$TEST_TMP/qwen.hgn" "$TEST_TMP/ngram.hgn" "$TEST_TMP/vision.hgn"
|
||||
mkdir -p "$TEST_TMP/tokenizer" && touch "$TEST_TMP/tokenizer/tok.json"
|
||||
# A host with plenty of RAM: the pin-gate preflight passes by default so
|
||||
# the artifact gates below are the thing under test.
|
||||
printf 'MemTotal: 127108112 kB\nMemAvailable: 100000000 kB\n' > "$TEST_TMP/meminfo"
|
||||
GOOD_ENV=(
|
||||
"FRANK_LARGE_ENV_FILE=$TEST_TMP/empty.env"
|
||||
"FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"
|
||||
@@ -82,6 +85,7 @@ good_env() {
|
||||
"HALOGEN_KV_SLOTS=2"
|
||||
"HALOGEN_SPEC_ADAPT=0"
|
||||
"HALOGEN_MAX_TOK=8192"
|
||||
"FRANK_LARGE_MEMINFO=$TEST_TMP/meminfo"
|
||||
"STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"
|
||||
)
|
||||
}
|
||||
@@ -229,6 +233,23 @@ good_env() {
|
||||
[[ "$output" == *"missing or a symlink"* ]]
|
||||
}
|
||||
|
||||
@test "the serve script refuses to start when host RAM is below the pin gate" {
|
||||
# The engine's pin gate: MemAvailable must clear the checkpoint plus a
|
||||
# 16 GiB floor, or the unit crash-loops (each boot dying in ~5 s) until
|
||||
# the RAM clears. The 2026-10-06 cutover hit this — 8 boots, 74.77 GiB
|
||||
# vs 78.11 needed. RAM_WAIT=0 so the preflight refuses instead of
|
||||
# waiting; the checkpoint is zero bytes (good_env) so the floor is the
|
||||
# whole gate.
|
||||
podman_stub
|
||||
good_env
|
||||
printf 'MemTotal: 127108112 kB\nMemAvailable: 10000000 kB\n' > "$TEST_TMP/meminfo-short"
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||||
FRANK_LARGE_MEMINFO="$TEST_TMP/meminfo-short" FRANK_LARGE_RAM_WAIT=0 "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"pin gate"* ]]
|
||||
[[ "$output" == *"refusing to start"* ]]
|
||||
}
|
||||
|
||||
@test "the render names the container, the port map and the engine shape" {
|
||||
podman_stub
|
||||
good_env
|
||||
|
||||
Reference in New Issue
Block a user