frank-large: mmap the CPU-overridden weights (no CUDA_Host pinning)

scripts/frank-large.env

@@ -18,18 +18,26 @@
 # as 3 parts, concatenated on disk; each part's sha256 was verified against
 # its HF LFS oid at download time.
 #
-# Placement (ikawrakow's hybrid recipe, upstream issue #2402):
-#   - --cpu-moe: ALL expert tensors (121B elems, ~71 GB) live in the Strix
-#     Halo's LPDDR5X via the CPU path — ik's fastest known configuration
-#     for this shape (the author's 2x3090 + Threadripper reference box
-#     measures ~50 t/s on the same model family).
-#   - -ngl 999: dense/attention/output weights and the KV cache on the
-#     3090 (~3 GB of weights + ~6 GB KV at 262k f16; 24 GB card).
-#   - The ~35 GB PLE table (per_layer_token_embd) is intentionally
-#     RAM/mmap-resident by design — only a tiny slice is touched per token.
-#     NEVER add --no-mmap (the STRIX-era lesson transfers: the cgroup
-#     killer takes the process with nothing in the server log) and do not
-#     try to pin PLE to VRAM (it cannot fit a 24 GB card).
+# Placement (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
+#   - -ot "exps=CPU": ALL 512-expert tensors (121B elems, ~84 GB) live in
+#     the Strix Halo's LPDDR5X via the plain CPU backend buffer — mmap-backed
+#     page cache, ik's fastest known configuration for this shape (the
+#     author's 2x3090 + Threadripper reference box measures ~50 t/s on the
+#     same model family). The shared experts (shexp) keep their GPU-side
+#     default: they run on every token.
+#   - -ot "per_layer_token_embd=CPU": the ~32 GB PLE n-gram table follows —
+#     only a tiny slice is touched per token, so it streams from page cache.
+#   - NEVER use --cpu-moe on this host: it routes the CPU-overridden
+#     weights through the CUDA_Host PINNED buffer (measured 2026-09-15:
+#     108.12 GiB page-locked, evicted every co-tenant into swap and starved
+#     sshd into unreachability until a power cycle). -ot ...=CPU is the
+#     mmap-backed path with identical DDR5 bandwidth for resident pages,
+#     and the kernel degrades gracefully under pressure (page cache evict
+#     + re-read) instead of stranding anyone.
+#   - GGML_CUDA_NO_PINNED_WEIGHTS=1 (src/llama-load-tensors.cpp) keeps the
+#     CPU-overridden weights mmapped and leaves only the small staging
+#     buffers pinned. (GGML_CUDA_NO_PINNED would also unpin staging —
+#     unnecessary here and slower for prefill transfers.)
 #
 # Sampling: greedy-by-default to preserve the halogen-era client contract
 # (a request without temperature decodes greedy; temp 0.0 = argmax).
@@ -53,7 +61,7 @@ MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K
 
 # Qwen-Sharp chat template (tracked in the repo, staged on the host). The
 # server emits reasoning in DeepSeek format so clients receive
-# reasoning_content separately instead of literal thinking tags in content.
+# reasoning_content separately instead of literal think tags in content.
 CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
 
 # Alias + network.
@@ -75,11 +83,13 @@ THREAD_BATCH=16
 CACHE_TYPE_K=f16
 CACHE_TYPE_V=f16
 
-# GPU/MoE placement: everything on the 3090 except the experts, which all
-# live in host RAM (--cpu-moe). See the placement notes above.
+# GPU/MoE placement: everything on the 3090 except the experts and the PLE
+# table, which both live in host RAM via the CPU backend buffer (see the
+# placement notes above).
 NGL=999
-CPU_MOE=on
-FLASH_ATTN=on
+# ik_llama.cpp CUDA host-memory policy: CPU-overridden weights stay mmapped;
+# only the small staging buffers are pinned (see the placement notes above).
+GGML_CUDA_NO_PINNED_WEIGHTS=1
 
 # Prompt cache (RAM budget in MiB) and slot state dir.
 CACHE_RAM=8192

scripts/frank-large.service

@@ -7,6 +7,11 @@ Wants=network-online.target
 Type=simple
 User=dank
 Group=dank
+# Staging buffers for the CUDA transfers are page-locked; the weights
+# themselves stay mmapped (GGML_CUDA_NO_PINNED_WEIGHTS=1), but the staging
+# budget still needs room. Parity with the halogen quadlet's
+# Ulimit=memlock=-1:-1.
+LimitMEMLOCK=infinity
 Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
 ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
 Restart=on-failure

scripts/serve-frank-large.sh

@@ -2,15 +2,17 @@
 # Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-large
 # via ik_llama.cpp on Dankclaw.
 #
-# Topology (ikawrakow's hybrid recipe, upstream issue #2402): the RTX 3090
-# (CUDA, sm_86) takes the dense/attention/output weights and the KV cache
-# (-ngl 999); ALL MoE expert tensors stay in the Strix Halo's LPDDR5X via
-# the CPU path (--cpu-moe) — ik's fastest known configuration for this
-# hybrid shape. The ~35 GB PLE n-gram table (per_layer_token_embd) is
-# mmap-resident BY DESIGN: only a tiny slice is touched per token, so it
-# streams from the OS page cache. NEVER add --no-mmap — the STRIX-era
-# lesson transfers directly (the cgroup killer takes the process with
-# nothing in the server log).
+# Topology (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
+# the RTX 3090 (CUDA, sm_86) takes the dense/attention/output weights and
+# the KV cache (-ngl 999); ALL MoE expert tensors and the ~32 GB PLE
+# n-gram table stay in the Strix Halo's LPDDR5X via the plain CPU backend
+# buffer (-ot ...=CPU — mmap-backed page cache; only a tiny slice of PLE is
+# touched per token). NEVER use --cpu-moe on this host: it routes the
+# CPU-overridden weights through the CUDA_Host PINNED buffer (measured
+# 108.12 GiB page-locked on 2026-09-15, which evicted every co-tenant into
+# swap and starved sshd into unreachability until a power cycle). NEVER add
+# --no-mmap — the STRIX-era lesson transfers directly (the cgroup killer
+# takes the process with nothing in the server log).
 #
 # Runtime: ik_llama.cpp master d85a1d7 built with CUDA (sm_86) + Zen5
 # AVX-512 IQK kernels; the startup banner must show HAVE_FANCY_SIMD
@@ -51,12 +53,16 @@ fi
 : "${NGL:=999}"
 : "${CPU_MOE:=on}"
 : "${FLASH_ATTN:=on}"
-: "${CACHE_RAM:=8192}"
-: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}"
-: "${TEMPERATURE:=0.0}"
-: "${REASONING:=on}"
-: "${REASONING_FORMAT:=deepseek}"
-: "${REASONING_BUDGET:=-1}"
+# Hard guard: on this host CUDA pinned host memory strands the co-tenants
+# (measured 2026-09-15: a default-policy start tried to page-lock 108.12 GiB,
+# evicted every co-tenant into swap and starved sshd into unreachability
+# until a power cycle). GGML_CUDA_NO_PINNED_WEIGHTS=1 keeps the
+# CPU-overridden weights mmapped (src/llama-load-tensors.cpp); the env file
+# sets it. Refuse to run without it.
+if [[ "${GGML_CUDA_NO_PINNED_WEIGHTS:-}" != "1" ]]; then
+    echo "frank-large: refusing to start — GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host" >&2
+    exit 1
+fi
 
 mkdir -p "$SLOT_SAVE_PATH"
 
@@ -66,7 +72,10 @@ nvidia-modprobe -u -c=0 2>/dev/null || true
 
 CPU_MOE_ARGS=()
 if [[ "$CPU_MOE" == "on" ]]; then
-    CPU_MOE_ARGS=(--cpu-moe)
+    # -ot ...=CPU (plain CPU backend buffer) is the mmap-backed path. NEVER
+    # swap this for --cpu-moe: that routes the weights through the CUDA_Host
+    # PINNED buffer (108.12 GiB measured) and strands the co-tenants.
+    CPU_MOE_ARGS=(-ot "exps=CPU" -ot "per_layer_token_embd=CPU")
 fi
 
 MMPROJ_ARGS=()

tests/frank_large_unit.bats

@@ -0,0 +1,86 @@
+#!/usr/bin/env bats
+# Seam: the frank-large ik_llama.cpp root unit on dankclaw.
+# ik_llama.cpp replaced the halogen rootless Quadlet on 2026-09-15. This seam
+# pins the deployment-policy properties that no reviewer can see by reading
+# the flags alone: a root unit whose files all live in this repo, the hybrid
+# placement contract (experts + PLE mmap'd on the CPU path — NEVER the
+# CUDA_Host pinned buffer), no pinned CUDA host memory, the single client
+# port every consumer already targets, the fleet alias, greedy-by-default
+# sampling, vision via the mtmd mmproj, and a deploy handler that retires
+# the halogen quadlet without deleting it.
+
+load helpers
+
+UNIT="$REPO_ROOT/scripts/frank-large.service"
+SERVE="$REPO_ROOT/scripts/serve-frank-large.sh"
+ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
+REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
+
+@test "frank-large unit, serve script and env exist in scripts/" {
+    [ -f "$UNIT" ]
+    [ -f "$SERVE" ]
+    [ -f "$ENV_FILE" ]
+}
+
+@test "frank-large unit is boot-persistent, self-healing and unprivileged" {
+    grep -q '^WantedBy=multi-user.target$' "$UNIT"
+    grep -q '^Restart=on-failure$' "$UNIT"
+    grep -q '^RestartSec=' "$UNIT"
+    grep -q '^User=dank$' "$UNIT"
+    grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
+}
+
+@test "frank-large unit points at the repo serve script and env" {
+    grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh$' "$UNIT"
+    grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT"
+}
+
+@test "frank-large places experts and PLE on the mmap'd CPU path, never the pinned buffer" {
+    # --cpu-moe routes CPU-overridden weights through the CUDA_Host PINNED
+    # buffer (108.12 GiB measured on 2026-09-15; it evicted every co-tenant
+    # into swap and starved sshd into unreachability). The -ot ...=CPU plain
+    # CPU backend buffer is the mmap-backed path.
+    grep -q -- '-ot "exps=CPU" -ot "per_layer_token_embd=CPU"' "$SERVE"
+    grep -q '^GGML_CUDA_NO_PINNED_WEIGHTS=1$' "$ENV_FILE"
+    # The serve script refuses to run without the no-pinned policy.
+    grep -q 'GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host' "$SERVE"
+}
+
+@test "frank-large serves the ik runtime from the unified-memory hybrid layout" {
+    grep -q 'ik_llama.cpp/build/bin/llama-server' "$ENV_FILE"
+    grep -q 'mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M' "$ENV_FILE"
+    grep -q -- '--gpu-layers "$NGL"' "$SERVE"
+    grep -q -- '--flash-attn "$FLASH_ATTN"' "$SERVE"
+}
+
+@test "frank-large keeps the fleet identity and the client port" {
+    grep -q '^LISTEN_PORT=7770$' "$ENV_FILE"
+    grep -q '^ALIAS=frank-large$' "$ENV_FILE"
+    grep -q -- '--port "$LISTEN_PORT"' "$SERVE"
+    grep -q -- '--host "$BIND_ADDR"' "$SERVE"
+}
+
+@test "frank-large decodes greedy by default (halogen client contract)" {
+    grep -q '^TEMPERATURE=0.0$' "$ENV_FILE"
+    grep -q -- '--temp "$TEMPERATURE"' "$SERVE"
+}
+
+@test "frank-large keeps image input via the mtmd mmproj" {
+    grep -q '^MMPROJ_PATH=.*mmproj-f16.gguf$' "$ENV_FILE"
+    grep -q -- '--mmproj "$MMPROJ_PATH"' "$SERVE"
+}
+
+@test "the deploy handler retires the halogen quadlet without deleting it" {
+    grep -q 'retire_quadlet_llm_large' "$REMOTE_DEPLOY"
+    grep -q 'systemctl --user stop frank-large.service' "$REMOTE_DEPLOY"
+    grep -q 'frank-large.container' "$REMOTE_DEPLOY"
+    grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY"
+    ! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?.*frank-large' "$REMOTE_DEPLOY"
+}
+
+@test "the deploy handler installs the root unit and refuses a missing checkpoint" {
+    grep -q 'relink_unit frank-large' "$REMOTE_DEPLOY"
+    grep -q 'systemctl enable frank-large.service' "$REMOTE_DEPLOY"
+    grep -q 'checkpoint absent' "$REMOTE_DEPLOY"
+    ! grep -q 'install_quadlet frank-large' "$REMOTE_DEPLOY"
+}