From c71a3066c0315e21fa1bb18d93c93adebb71ced8 Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Tue, 6 Oct 2026 12:58:34 -0700
Subject: [PATCH] frank-large: host-RAM pin-gate preflight in
 serve-frank-large.sh

The 2026-10-06 cutover crash-looped frank-large 8 boots (03:09:39-03:11:42):
the engine's pin gate refused to start while host MemAvailable was ~74.7 GiB,
3.34 GiB short of the 78.11 GiB it needs (62.11 GiB checkpoint + 16 GiB floor).
It self-resolved in ~2 min when the dying 27B engine freed RAM, but left the
host with only ~1.8 GiB MemAvailable - a restart would crash-loop again.

serve-frank-large.sh now checks MemAvailable against the pin gate BEFORE
podman run and WAITS up to FRANK_LARGE_RAM_WAIT (default 600 s) for the RAM
to clear, so a transient dip boots once instead of crash-looping; if it never
clears it refuses loudly. The 16 GiB floor is the engine's (not
HALOGEN_HOST_RESERVE_GIB=20, a separate overall-budget knob).

frank_large_unit.bats: FRANK_LARGE_MEMINFO seam + a refuse test. Runbook +
log: the pin-gate law, the incident, and the fix.
---
 scripts/serve-frank-large.sh                  | 33 +++++++++++++++++++
 .../frank-large-flash-server-v2-swift15.md    | 13 ++++++++
 serverAppConfigs-docs/log.md                  | 19 +++++++++++
 tests/frank_large_unit.bats                   | 21 ++++++++++++
 4 files changed, 86 insertions(+)

diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh
index 3a342f9..e92323e 100755
--- a/scripts/serve-frank-large.sh
+++ b/scripts/serve-frank-large.sh
@@ -115,6 +115,39 @@ if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then
     echo "frank-large: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
     exit 1
 fi
+# Host-RAM preflight — the engine's pin gate, checked BEFORE podman run.
+# The engine refuses to pin when MemAvailable is below the checkpoint plus
+# a 16 GiB floor: pinned pages cannot be swapped, and under the floor the
+# n-gram lookup pages in from disk on every long prompt. A RAM-short host
+# (a co-tenant spike, a sibling engine dying) crash-loops the unit — each
+# boot dies in ~5 s at the pin gate — until the RAM clears. The 2026-10-06
+# cutover hit this: 8 boots, MemAvailable 74.77 GiB vs the 78.11 GiB needed,
+# self-resolved in ~2 min when the dying 27B engine freed RAM. Check here and
+# WAIT for the RAM to clear, so a transient dip boots once instead of
+# crash-looping; if it never clears, refuse loudly.
+MEMINFO="${FRANK_LARGE_MEMINFO:-/proc/meminfo}"
+PIN_KB=$(( FRANK_LARGE_CHECKPOINT_BYTES / 1024 ))
+NEED_KB=$(( PIN_KB + 16 * 1048576 ))
+ram_avail_kb() { awk '/^MemAvailable:/{print $2; exit}' "$MEMINFO"; }
+MEMAVAIL_KB="$(ram_avail_kb)" || MEMAVAIL_KB=""
+if [ -n "$MEMAVAIL_KB" ] && [ "$MEMAVAIL_KB" -lt "$NEED_KB" ]; then
+    WAIT_S="${FRANK_LARGE_RAM_WAIT:-600}"
+    GIB() { awk -v kb="$1" 'BEGIN{printf "%.2f", kb/1048576}'; }
+    echo "frank-large: host RAM below the pin gate — MemAvailable $(GIB "$MEMAVAIL_KB") GiB < needed $(GIB "$NEED_KB") GiB ($(GIB "$PIN_KB") GiB checkpoint + 16.00 GiB floor); waiting up to ${WAIT_S}s for host RAM to clear" >&2
+    deadline=$(( $(date +%s) + WAIT_S ))
+    while [ "$(date +%s)" -lt "$deadline" ]; do
+        sleep 5
+        MEMAVAIL_KB="$(ram_avail_kb)" || MEMAVAIL_KB=""
+        if [ -n "$MEMAVAIL_KB" ] && [ "$MEMAVAIL_KB" -ge "$NEED_KB" ]; then
+            break
+        fi
+    done
+    if [ -z "$MEMAVAIL_KB" ] || [ "$MEMAVAIL_KB" -lt "$NEED_KB" ]; then
+        echo "frank-large: MemAvailable $(GIB "${MEMAVAIL_KB:-0}") GiB after ${WAIT_S}s; refusing to start (the engine would refuse the pin)" >&2
+        exit 1
+    fi
+    echo "frank-large: MemAvailable cleared to $(GIB "$MEMAVAIL_KB") GiB; starting" >&2
+fi
 for CTR in ${FRANK_LARGE_LEGACY_CONTAINERS:-} "$FRANK_LARGE_CONTAINER"; do
     podman rm -f "$CTR" >/dev/null 2>&1 || true
 done
diff --git a/serverAppConfigs-docs/LLM/frank-large-flash-server-v2-swift15.md b/serverAppConfigs-docs/LLM/frank-large-flash-server-v2-swift15.md
index 16ea61b..90e0f2a 100644
--- a/serverAppConfigs-docs/LLM/frank-large-flash-server-v2-swift15.md
+++ b/serverAppConfigs-docs/LLM/frank-large-flash-server-v2-swift15.md
@@ -143,6 +143,19 @@ the 27B engine ran on. Only revisit if the ledger shows SVM complaints.
   image digest in podman storage, byte-exact checkpoint + n-gram table +
   vision tower, flat tokenizer with all six files as regular files,
   GPU nodes present.
+  - **Host-RAM pin-gate preflight** (added 2026-10-06): the engine
+  refuses to pin when `MemAvailable` is below the checkpoint (62.11 GiB)
+  plus a **16 GiB floor** — pinned pages cannot be swapped, and under the
+  floor the n-gram lookup pages in from disk on every long prompt. A
+  RAM-short host crash-loops the unit (each boot dies in ~5 s at the pin
+  gate) until the RAM clears; the 2026-10-06 cutover hit this (8 boots,
+  MemAvailable 74.77 GiB vs the 78.11 GiB needed, self-resolved in ~2 min
+  when the dying 27B engine freed RAM). The serve script now checks
+  `MemAvailable` BEFORE `podman run` and WAITS up to `FRANK_LARGE_RAM_WAIT`
+  (default 600 s) for the RAM to clear, so a transient dip boots once
+  instead of crash-looping; if it never clears it refuses loudly. The 16
+  GiB floor is the engine's (not `HALOGEN_HOST_RESERVE_GIB=20` — a
+  separate, higher, overall-budget knob).
 - Engine stdout goes to the log, NOT the journal. The FIRST request after
   a (re)start can sit silent ~45–60 s on this engine class (health
   `engine_unresponsive`, 503s) and self-recovers — do not restart over it.
diff --git a/serverAppConfigs-docs/log.md b/serverAppConfigs-docs/log.md
index ec4fbcf..3d3d5c4 100644
--- a/serverAppConfigs-docs/log.md
+++ b/serverAppConfigs-docs/log.md
@@ -20,6 +20,25 @@
   ngram-mod still skipped (recurrent-state rollback law on the qwen35
   hybrid). Runbook renamed + rewritten:
   `serverAppConfigs-docs/LLM/frank-small-llama-swift15-unc-iq3xxs-mtp-131072.md`.
+- **frank-large (Dankclaw): halogen 27B-dense → halogen-flash-server
+  0.16.2 v2 Swift1.5 ABLITERATED (Strix Halo iGPU, port 7770).** Cutover
+  commit 31b1ec3 (03:07:40): the v2 `.hgn` payload loads ONLY in the
+  flash-server line, so the engine moved with the model. Abliterated base
+  (refusals gone), vision back (the v2 tower loads beside the checkpoint),
+  MTP head inside the checkpoint, 2×200,192 slots, `HALOGEN_SPEC_ADAPT=0`.
+  Runbook: `serverAppConfigs-docs/LLM/frank-large-flash-server-v2-swift15.md`.
+  **Incident (03:09:39–03:11:42):** the deploy started the unit while host
+  `MemAvailable` was ~74.7 GiB, 3.34 GiB short of the engine's pin-gate
+  requirement (62.11 GiB checkpoint + 16 GiB floor = 78.11 GiB); it
+  crash-looped 8 boots, each dying in ~5 s at the pin gate, and
+  self-resolved in ~2 min when the dying 27B engine freed RAM (boot #9
+  pinned clean at 88.4 GiB available). **Latent risk + fix:** the host was
+  left with only ~1.8 GiB `MemAvailable`, so a restart would crash-loop
+  again; `serve-frank-large.sh` now has a host-RAM preflight that checks
+  `MemAvailable` against the pin gate BEFORE `podman run` and WAITS up to
+  `FRANK_LARGE_RAM_WAIT` (600 s) for the RAM to clear, refusing loudly if
+  it never does (the 16 GiB floor is the engine's, not
+  `HALOGEN_HOST_RESERVE_GIB`).
 
 ## 2026-10-05
 - **frank-primary: tcclaviger vLLM fork → FreeToken (PR #587 ROCm) on
diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats
index cd503c6..75004ef 100755
--- a/tests/frank_large_unit.bats
+++ b/tests/frank_large_unit.bats
@@ -59,6 +59,9 @@ good_env() {
     : > "$TEST_TMP/empty.env"
     touch "$TEST_TMP/qwen.hgn" "$TEST_TMP/ngram.hgn" "$TEST_TMP/vision.hgn"
     mkdir -p "$TEST_TMP/tokenizer" && touch "$TEST_TMP/tokenizer/tok.json"
+    # A host with plenty of RAM: the pin-gate preflight passes by default so
+    # the artifact gates below are the thing under test.
+    printf 'MemTotal:       127108112 kB\nMemAvailable:    100000000 kB\n' > "$TEST_TMP/meminfo"
     GOOD_ENV=(
         "FRANK_LARGE_ENV_FILE=$TEST_TMP/empty.env"
         "FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"
@@ -82,6 +85,7 @@ good_env() {
         "HALOGEN_KV_SLOTS=2"
         "HALOGEN_SPEC_ADAPT=0"
         "HALOGEN_MAX_TOK=8192"
+        "FRANK_LARGE_MEMINFO=$TEST_TMP/meminfo"
         "STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen-flash-server@sha256:cafe"
     )
 }
@@ -229,6 +233,23 @@ good_env() {
     [[ "$output" == *"missing or a symlink"* ]]
 }
 
+@test "the serve script refuses to start when host RAM is below the pin gate" {
+    # The engine's pin gate: MemAvailable must clear the checkpoint plus a
+    # 16 GiB floor, or the unit crash-loops (each boot dying in ~5 s) until
+    # the RAM clears. The 2026-10-06 cutover hit this — 8 boots, 74.77 GiB
+    # vs 78.11 needed. RAM_WAIT=0 so the preflight refuses instead of
+    # waiting; the checkpoint is zero bytes (good_env) so the floor is the
+    # whole gate.
+    podman_stub
+    good_env
+    printf 'MemTotal:       127108112 kB\nMemAvailable:    10000000 kB\n' > "$TEST_TMP/meminfo-short"
+    run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
+        FRANK_LARGE_MEMINFO="$TEST_TMP/meminfo-short" FRANK_LARGE_RAM_WAIT=0 "$SERVE"
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"pin gate"* ]]
+    [[ "$output" == *"refusing to start"* ]]
+}
+
 @test "the render names the container, the port map and the engine shape" {
     podman_stub
     good_env