frank-large: mmap the CPU-overridden weights (no CUDA_Host pinning)

scripts/frank-large.env

@@ -18,18 +18,26 @@
 # as 3 parts, concatenated on disk; each part's sha256 was verified against
 # its HF LFS oid at download time.
 #
-# Placement (ikawrakow's hybrid recipe, upstream issue #2402):
-#   - --cpu-moe: ALL expert tensors (121B elems, ~71 GB) live in the Strix
-#     Halo's LPDDR5X via the CPU path — ik's fastest known configuration
-#     for this shape (the author's 2x3090 + Threadripper reference box
-#     measures ~50 t/s on the same model family).
-#   - -ngl 999: dense/attention/output weights and the KV cache on the
-#     3090 (~3 GB of weights + ~6 GB KV at 262k f16; 24 GB card).
-#   - The ~35 GB PLE table (per_layer_token_embd) is intentionally
-#     RAM/mmap-resident by design — only a tiny slice is touched per token.
-#     NEVER add --no-mmap (the STRIX-era lesson transfers: the cgroup
-#     killer takes the process with nothing in the server log) and do not
-#     try to pin PLE to VRAM (it cannot fit a 24 GB card).
+# Placement (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
+#   - -ot "exps=CPU": ALL 512-expert tensors (121B elems, ~84 GB) live in
+#     the Strix Halo's LPDDR5X via the plain CPU backend buffer — mmap-backed
+#     page cache, ik's fastest known configuration for this shape (the
+#     author's 2x3090 + Threadripper reference box measures ~50 t/s on the
+#     same model family). The shared experts (shexp) keep their GPU-side
+#     default: they run on every token.
+#   - -ot "per_layer_token_embd=CPU": the ~32 GB PLE n-gram table follows —
+#     only a tiny slice is touched per token, so it streams from page cache.
+#   - NEVER use --cpu-moe on this host: it routes the CPU-overridden
+#     weights through the CUDA_Host PINNED buffer (measured 2026-09-15:
+#     108.12 GiB page-locked, evicted every co-tenant into swap and starved
+#     sshd into unreachability until a power cycle). -ot ...=CPU is the
+#     mmap-backed path with identical DDR5 bandwidth for resident pages,
+#     and the kernel degrades gracefully under pressure (page cache evict
+#     + re-read) instead of stranding anyone.
+#   - GGML_CUDA_NO_PINNED_WEIGHTS=1 (src/llama-load-tensors.cpp) keeps the
+#     CPU-overridden weights mmapped and leaves only the small staging
+#     buffers pinned. (GGML_CUDA_NO_PINNED would also unpin staging —
+#     unnecessary here and slower for prefill transfers.)
 #
 # Sampling: greedy-by-default to preserve the halogen-era client contract
 # (a request without temperature decodes greedy; temp 0.0 = argmax).
@@ -53,7 +61,7 @@ MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K
 
 # Qwen-Sharp chat template (tracked in the repo, staged on the host). The
 # server emits reasoning in DeepSeek format so clients receive
-# reasoning_content separately instead of literal thinking tags in content.
+# reasoning_content separately instead of literal think tags in content.
 CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
 
 # Alias + network.
@@ -75,11 +83,13 @@ THREAD_BATCH=16
 CACHE_TYPE_K=f16
 CACHE_TYPE_V=f16
 
-# GPU/MoE placement: everything on the 3090 except the experts, which all
-# live in host RAM (--cpu-moe). See the placement notes above.
+# GPU/MoE placement: everything on the 3090 except the experts and the PLE
+# table, which both live in host RAM via the CPU backend buffer (see the
+# placement notes above).
 NGL=999
-CPU_MOE=on
-FLASH_ATTN=on
+# ik_llama.cpp CUDA host-memory policy: CPU-overridden weights stay mmapped;
+# only the small staging buffers are pinned (see the placement notes above).
+GGML_CUDA_NO_PINNED_WEIGHTS=1
 
 # Prompt cache (RAM budget in MiB) and slot state dir.
 CACHE_RAM=8192

scripts/frank-large.service

@@ -7,6 +7,11 @@ Wants=network-online.target
 Type=simple
 User=dank
 Group=dank
+# Staging buffers for the CUDA transfers are page-locked; the weights
+# themselves stay mmapped (GGML_CUDA_NO_PINNED_WEIGHTS=1), but the staging
+# budget still needs room. Parity with the halogen quadlet's
+# Ulimit=memlock=-1:-1.
+LimitMEMLOCK=infinity
 Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
 ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
 Restart=on-failure

scripts/serve-frank-large.sh

@@ -2,15 +2,17 @@
 # Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-large
 # via ik_llama.cpp on Dankclaw.
 #
-# Topology (ikawrakow's hybrid recipe, upstream issue #2402): the RTX 3090
-# (CUDA, sm_86) takes the dense/attention/output weights and the KV cache
-# (-ngl 999); ALL MoE expert tensors stay in the Strix Halo's LPDDR5X via
-# the CPU path (--cpu-moe) — ik's fastest known configuration for this
-# hybrid shape. The ~35 GB PLE n-gram table (per_layer_token_embd) is
-# mmap-resident BY DESIGN: only a tiny slice is touched per token, so it
-# streams from the OS page cache. NEVER add --no-mmap — the STRIX-era
-# lesson transfers directly (the cgroup killer takes the process with
-# nothing in the server log).
+# Topology (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
+# the RTX 3090 (CUDA, sm_86) takes the dense/attention/output weights and
+# the KV cache (-ngl 999); ALL MoE expert tensors and the ~32 GB PLE
+# n-gram table stay in the Strix Halo's LPDDR5X via the plain CPU backend
+# buffer (-ot ...=CPU — mmap-backed page cache; only a tiny slice of PLE is
+# touched per token). NEVER use --cpu-moe on this host: it routes the
+# CPU-overridden weights through the CUDA_Host PINNED buffer (measured
+# 108.12 GiB page-locked on 2026-09-15, which evicted every co-tenant into
+# swap and starved sshd into unreachability until a power cycle). NEVER add
+# --no-mmap — the STRIX-era lesson transfers directly (the cgroup killer
+# takes the process with nothing in the server log).
 #
 # Runtime: ik_llama.cpp master d85a1d7 built with CUDA (sm_86) + Zen5
 # AVX-512 IQK kernels; the startup banner must show HAVE_FANCY_SIMD
@@ -51,12 +53,16 @@ fi
 : "${NGL:=999}"
 : "${CPU_MOE:=on}"
 : "${FLASH_ATTN:=on}"
-: "${CACHE_RAM:=8192}"
-: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}"
-: "${TEMPERATURE:=0.0}"
-: "${REASONING:=on}"
-: "${REASONING_FORMAT:=deepseek}"
-: "${REASONING_BUDGET:=-1}"
+# Hard guard: on this host CUDA pinned host memory strands the co-tenants
+# (measured 2026-09-15: a default-policy start tried to page-lock 108.12 GiB,
+# evicted every co-tenant into swap and starved sshd into unreachability
+# until a power cycle). GGML_CUDA_NO_PINNED_WEIGHTS=1 keeps the
+# CPU-overridden weights mmapped (src/llama-load-tensors.cpp); the env file
+# sets it. Refuse to run without it.
+if [[ "${GGML_CUDA_NO_PINNED_WEIGHTS:-}" != "1" ]]; then
+    echo "frank-large: refusing to start — GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host" >&2
+    exit 1
+fi
 
 mkdir -p "$SLOT_SAVE_PATH"
 
@@ -66,7 +72,10 @@ nvidia-modprobe -u -c=0 2>/dev/null || true
 
 CPU_MOE_ARGS=()
 if [[ "$CPU_MOE" == "on" ]]; then
-    CPU_MOE_ARGS=(--cpu-moe)
+    # -ot ...=CPU (plain CPU backend buffer) is the mmap-backed path. NEVER
+    # swap this for --cpu-moe: that routes the weights through the CUDA_Host
+    # PINNED buffer (108.12 GiB measured) and strands the co-tenants.
+    CPU_MOE_ARGS=(-ot "exps=CPU" -ot "per_layer_token_embd=CPU")
 fi
 
 MMPROJ_ARGS=()