frank-large: mmap the CPU-overridden weights (no CUDA_Host pinning)

scripts/serve-frank-large.sh

@@ -2,15 +2,17 @@
 # Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-large
 # via ik_llama.cpp on Dankclaw.
 #
-# Topology (ikawrakow's hybrid recipe, upstream issue #2402): the RTX 3090
-# (CUDA, sm_86) takes the dense/attention/output weights and the KV cache
-# (-ngl 999); ALL MoE expert tensors stay in the Strix Halo's LPDDR5X via
-# the CPU path (--cpu-moe) — ik's fastest known configuration for this
-# hybrid shape. The ~35 GB PLE n-gram table (per_layer_token_embd) is
-# mmap-resident BY DESIGN: only a tiny slice is touched per token, so it
-# streams from the OS page cache. NEVER add --no-mmap — the STRIX-era
-# lesson transfers directly (the cgroup killer takes the process with
-# nothing in the server log).
+# Topology (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
+# the RTX 3090 (CUDA, sm_86) takes the dense/attention/output weights and
+# the KV cache (-ngl 999); ALL MoE expert tensors and the ~32 GB PLE
+# n-gram table stay in the Strix Halo's LPDDR5X via the plain CPU backend
+# buffer (-ot ...=CPU — mmap-backed page cache; only a tiny slice of PLE is
+# touched per token). NEVER use --cpu-moe on this host: it routes the
+# CPU-overridden weights through the CUDA_Host PINNED buffer (measured
+# 108.12 GiB page-locked on 2026-09-15, which evicted every co-tenant into
+# swap and starved sshd into unreachability until a power cycle). NEVER add
+# --no-mmap — the STRIX-era lesson transfers directly (the cgroup killer
+# takes the process with nothing in the server log).
 #
 # Runtime: ik_llama.cpp master d85a1d7 built with CUDA (sm_86) + Zen5
 # AVX-512 IQK kernels; the startup banner must show HAVE_FANCY_SIMD
@@ -51,12 +53,16 @@ fi
 : "${NGL:=999}"
 : "${CPU_MOE:=on}"
 : "${FLASH_ATTN:=on}"
-: "${CACHE_RAM:=8192}"
-: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}"
-: "${TEMPERATURE:=0.0}"
-: "${REASONING:=on}"
-: "${REASONING_FORMAT:=deepseek}"
-: "${REASONING_BUDGET:=-1}"
+# Hard guard: on this host CUDA pinned host memory strands the co-tenants
+# (measured 2026-09-15: a default-policy start tried to page-lock 108.12 GiB,
+# evicted every co-tenant into swap and starved sshd into unreachability
+# until a power cycle). GGML_CUDA_NO_PINNED_WEIGHTS=1 keeps the
+# CPU-overridden weights mmapped (src/llama-load-tensors.cpp); the env file
+# sets it. Refuse to run without it.
+if [[ "${GGML_CUDA_NO_PINNED_WEIGHTS:-}" != "1" ]]; then
+    echo "frank-large: refusing to start — GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host" >&2
+    exit 1
+fi
 
 mkdir -p "$SLOT_SAVE_PATH"
 
@@ -66,7 +72,10 @@ nvidia-modprobe -u -c=0 2>/dev/null || true
 
 CPU_MOE_ARGS=()
 if [[ "$CPU_MOE" == "on" ]]; then
-    CPU_MOE_ARGS=(--cpu-moe)
+    # -ot ...=CPU (plain CPU backend buffer) is the mmap-backed path. NEVER
+    # swap this for --cpu-moe: that routes the weights through the CUDA_Host
+    # PINNED buffer (108.12 GiB measured) and strands the co-tenants.
+    CPU_MOE_ARGS=(-ot "exps=CPU" -ot "per_layer_token_embd=CPU")
 fi
 
 MMPROJ_ARGS=()