frank-large: mmap the CPU-overridden weights (no CUDA_Host pinning)

scripts/frank-large.env

@@ -18,18 +18,26 @@
 # as 3 parts, concatenated on disk; each part's sha256 was verified against
 # its HF LFS oid at download time.
 #
-# Placement (ikawrakow's hybrid recipe, upstream issue #2402):
-#   - --cpu-moe: ALL expert tensors (121B elems, ~71 GB) live in the Strix
-#     Halo's LPDDR5X via the CPU path — ik's fastest known configuration
-#     for this shape (the author's 2x3090 + Threadripper reference box
-#     measures ~50 t/s on the same model family).
-#   - -ngl 999: dense/attention/output weights and the KV cache on the
-#     3090 (~3 GB of weights + ~6 GB KV at 262k f16; 24 GB card).
-#   - The ~35 GB PLE table (per_layer_token_embd) is intentionally
-#     RAM/mmap-resident by design — only a tiny slice is touched per token.
-#     NEVER add --no-mmap (the STRIX-era lesson transfers: the cgroup
-#     killer takes the process with nothing in the server log) and do not
-#     try to pin PLE to VRAM (it cannot fit a 24 GB card).
+# Placement (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
+#   - -ot "exps=CPU": ALL 512-expert tensors (121B elems, ~84 GB) live in
+#     the Strix Halo's LPDDR5X via the plain CPU backend buffer — mmap-backed
+#     page cache, ik's fastest known configuration for this shape (the
+#     author's 2x3090 + Threadripper reference box measures ~50 t/s on the
+#     same model family). The shared experts (shexp) keep their GPU-side
+#     default: they run on every token.
+#   - -ot "per_layer_token_embd=CPU": the ~32 GB PLE n-gram table follows —
+#     only a tiny slice is touched per token, so it streams from page cache.
+#   - NEVER use --cpu-moe on this host: it routes the CPU-overridden
+#     weights through the CUDA_Host PINNED buffer (measured 2026-09-15:
+#     108.12 GiB page-locked, evicted every co-tenant into swap and starved
+#     sshd into unreachability until a power cycle). -ot ...=CPU is the
+#     mmap-backed path with identical DDR5 bandwidth for resident pages,
+#     and the kernel degrades gracefully under pressure (page cache evict
+#     + re-read) instead of stranding anyone.
+#   - GGML_CUDA_NO_PINNED_WEIGHTS=1 (src/llama-load-tensors.cpp) keeps the
+#     CPU-overridden weights mmapped and leaves only the small staging
+#     buffers pinned. (GGML_CUDA_NO_PINNED would also unpin staging —
+#     unnecessary here and slower for prefill transfers.)
 #
 # Sampling: greedy-by-default to preserve the halogen-era client contract
 # (a request without temperature decodes greedy; temp 0.0 = argmax).
@@ -53,7 +61,7 @@ MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K
 
 # Qwen-Sharp chat template (tracked in the repo, staged on the host). The
 # server emits reasoning in DeepSeek format so clients receive
-# reasoning_content separately instead of literal thinking tags in content.
+# reasoning_content separately instead of literal think tags in content.
 CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
 
 # Alias + network.
@@ -75,11 +83,13 @@ THREAD_BATCH=16
 CACHE_TYPE_K=f16
 CACHE_TYPE_V=f16
 
-# GPU/MoE placement: everything on the 3090 except the experts, which all
-# live in host RAM (--cpu-moe). See the placement notes above.
+# GPU/MoE placement: everything on the 3090 except the experts and the PLE
+# table, which both live in host RAM via the CPU backend buffer (see the
+# placement notes above).
 NGL=999
-CPU_MOE=on
-FLASH_ATTN=on
+# ik_llama.cpp CUDA host-memory policy: CPU-overridden weights stay mmapped;
+# only the small staging buffers are pinned (see the placement notes above).
+GGML_CUDA_NO_PINNED_WEIGHTS=1
 
 # Prompt cache (RAM budget in MiB) and slot state dir.
 CACHE_RAM=8192