From 0022ba96ad9b9363ede6805986b4e93e666e392e Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 15 Sep 2026 23:58:34 -0700 Subject: [PATCH] frank-large: mmap the CPU-overridden weights (no CUDA_Host pinning) --- scripts/frank-large.env | 44 +++++++++++------- scripts/frank-large.service | 5 +++ scripts/serve-frank-large.sh | 41 ++++++++++------- tests/frank_large_unit.bats | 86 ++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 33 deletions(-) create mode 100755 tests/frank_large_unit.bats diff --git a/scripts/frank-large.env b/scripts/frank-large.env index 083a36e..467606c 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -18,18 +18,26 @@ # as 3 parts, concatenated on disk; each part's sha256 was verified against # its HF LFS oid at download time. # -# Placement (ikawrakow's hybrid recipe, upstream issue #2402): -# - --cpu-moe: ALL expert tensors (121B elems, ~71 GB) live in the Strix -# Halo's LPDDR5X via the CPU path — ik's fastest known configuration -# for this shape (the author's 2x3090 + Threadripper reference box -# measures ~50 t/s on the same model family). -# - -ngl 999: dense/attention/output weights and the KV cache on the -# 3090 (~3 GB of weights + ~6 GB KV at 262k f16; 24 GB card). -# - The ~35 GB PLE table (per_layer_token_embd) is intentionally -# RAM/mmap-resident by design — only a tiny slice is touched per token. -# NEVER add --no-mmap (the STRIX-era lesson transfers: the cgroup -# killer takes the process with nothing in the server log) and do not -# try to pin PLE to VRAM (it cannot fit a 24 GB card). +# Placement (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE): +# - -ot "exps=CPU": ALL 512-expert tensors (121B elems, ~84 GB) live in +# the Strix Halo's LPDDR5X via the plain CPU backend buffer — mmap-backed +# page cache, ik's fastest known configuration for this shape (the +# author's 2x3090 + Threadripper reference box measures ~50 t/s on the +# same model family). The shared experts (shexp) keep their GPU-side +# default: they run on every token. +# - -ot "per_layer_token_embd=CPU": the ~32 GB PLE n-gram table follows — +# only a tiny slice is touched per token, so it streams from page cache. +# - NEVER use --cpu-moe on this host: it routes the CPU-overridden +# weights through the CUDA_Host PINNED buffer (measured 2026-09-15: +# 108.12 GiB page-locked, evicted every co-tenant into swap and starved +# sshd into unreachability until a power cycle). -ot ...=CPU is the +# mmap-backed path with identical DDR5 bandwidth for resident pages, +# and the kernel degrades gracefully under pressure (page cache evict +# + re-read) instead of stranding anyone. +# - GGML_CUDA_NO_PINNED_WEIGHTS=1 (src/llama-load-tensors.cpp) keeps the +# CPU-overridden weights mmapped and leaves only the small staging +# buffers pinned. (GGML_CUDA_NO_PINNED would also unpin staging — +# unnecessary here and slower for prefill transfers.) # # Sampling: greedy-by-default to preserve the halogen-era client contract # (a request without temperature decodes greedy; temp 0.0 = argmax). @@ -53,7 +61,7 @@ MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K # Qwen-Sharp chat template (tracked in the repo, staged on the host). The # server emits reasoning in DeepSeek format so clients receive -# reasoning_content separately instead of literal thinking tags in content. +# reasoning_content separately instead of literal think tags in content. CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja # Alias + network. @@ -75,11 +83,13 @@ THREAD_BATCH=16 CACHE_TYPE_K=f16 CACHE_TYPE_V=f16 -# GPU/MoE placement: everything on the 3090 except the experts, which all -# live in host RAM (--cpu-moe). See the placement notes above. +# GPU/MoE placement: everything on the 3090 except the experts and the PLE +# table, which both live in host RAM via the CPU backend buffer (see the +# placement notes above). NGL=999 -CPU_MOE=on -FLASH_ATTN=on +# ik_llama.cpp CUDA host-memory policy: CPU-overridden weights stay mmapped; +# only the small staging buffers are pinned (see the placement notes above). +GGML_CUDA_NO_PINNED_WEIGHTS=1 # Prompt cache (RAM budget in MiB) and slot state dir. CACHE_RAM=8192 diff --git a/scripts/frank-large.service b/scripts/frank-large.service index 8b5d677..2badb4f 100644 --- a/scripts/frank-large.service +++ b/scripts/frank-large.service @@ -7,6 +7,11 @@ Wants=network-online.target Type=simple User=dank Group=dank +# Staging buffers for the CUDA transfers are page-locked; the weights +# themselves stay mmapped (GGML_CUDA_NO_PINNED_WEIGHTS=1), but the staging +# budget still needs room. Parity with the halogen quadlet's +# Ulimit=memlock=-1:-1. +LimitMEMLOCK=infinity Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh Restart=on-failure diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index dc8439f..594772f 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -2,15 +2,17 @@ # Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-large # via ik_llama.cpp on Dankclaw. # -# Topology (ikawrakow's hybrid recipe, upstream issue #2402): the RTX 3090 -# (CUDA, sm_86) takes the dense/attention/output weights and the KV cache -# (-ngl 999); ALL MoE expert tensors stay in the Strix Halo's LPDDR5X via -# the CPU path (--cpu-moe) — ik's fastest known configuration for this -# hybrid shape. The ~35 GB PLE n-gram table (per_layer_token_embd) is -# mmap-resident BY DESIGN: only a tiny slice is touched per token, so it -# streams from the OS page cache. NEVER add --no-mmap — the STRIX-era -# lesson transfers directly (the cgroup killer takes the process with -# nothing in the server log). +# Topology (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE): +# the RTX 3090 (CUDA, sm_86) takes the dense/attention/output weights and +# the KV cache (-ngl 999); ALL MoE expert tensors and the ~32 GB PLE +# n-gram table stay in the Strix Halo's LPDDR5X via the plain CPU backend +# buffer (-ot ...=CPU — mmap-backed page cache; only a tiny slice of PLE is +# touched per token). NEVER use --cpu-moe on this host: it routes the +# CPU-overridden weights through the CUDA_Host PINNED buffer (measured +# 108.12 GiB page-locked on 2026-09-15, which evicted every co-tenant into +# swap and starved sshd into unreachability until a power cycle). NEVER add +# --no-mmap — the STRIX-era lesson transfers directly (the cgroup killer +# takes the process with nothing in the server log). # # Runtime: ik_llama.cpp master d85a1d7 built with CUDA (sm_86) + Zen5 # AVX-512 IQK kernels; the startup banner must show HAVE_FANCY_SIMD @@ -51,12 +53,16 @@ fi : "${NGL:=999}" : "${CPU_MOE:=on}" : "${FLASH_ATTN:=on}" -: "${CACHE_RAM:=8192}" -: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}" -: "${TEMPERATURE:=0.0}" -: "${REASONING:=on}" -: "${REASONING_FORMAT:=deepseek}" -: "${REASONING_BUDGET:=-1}" +# Hard guard: on this host CUDA pinned host memory strands the co-tenants +# (measured 2026-09-15: a default-policy start tried to page-lock 108.12 GiB, +# evicted every co-tenant into swap and starved sshd into unreachability +# until a power cycle). GGML_CUDA_NO_PINNED_WEIGHTS=1 keeps the +# CPU-overridden weights mmapped (src/llama-load-tensors.cpp); the env file +# sets it. Refuse to run without it. +if [[ "${GGML_CUDA_NO_PINNED_WEIGHTS:-}" != "1" ]]; then + echo "frank-large: refusing to start — GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host" >&2 + exit 1 +fi mkdir -p "$SLOT_SAVE_PATH" @@ -66,7 +72,10 @@ nvidia-modprobe -u -c=0 2>/dev/null || true CPU_MOE_ARGS=() if [[ "$CPU_MOE" == "on" ]]; then - CPU_MOE_ARGS=(--cpu-moe) + # -ot ...=CPU (plain CPU backend buffer) is the mmap-backed path. NEVER + # swap this for --cpu-moe: that routes the weights through the CUDA_Host + # PINNED buffer (108.12 GiB measured) and strands the co-tenants. + CPU_MOE_ARGS=(-ot "exps=CPU" -ot "per_layer_token_embd=CPU") fi MMPROJ_ARGS=() diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats new file mode 100755 index 0000000..3c66d83 --- /dev/null +++ b/tests/frank_large_unit.bats @@ -0,0 +1,86 @@ +#!/usr/bin/env bats +# Seam: the frank-large ik_llama.cpp root unit on dankclaw. +# ik_llama.cpp replaced the halogen rootless Quadlet on 2026-09-15. This seam +# pins the deployment-policy properties that no reviewer can see by reading +# the flags alone: a root unit whose files all live in this repo, the hybrid +# placement contract (experts + PLE mmap'd on the CPU path — NEVER the +# CUDA_Host pinned buffer), no pinned CUDA host memory, the single client +# port every consumer already targets, the fleet alias, greedy-by-default +# sampling, vision via the mtmd mmproj, and a deploy handler that retires +# the halogen quadlet without deleting it. + +load helpers + +UNIT="$REPO_ROOT/scripts/frank-large.service" +SERVE="$REPO_ROOT/scripts/serve-frank-large.sh" +ENV_FILE="$REPO_ROOT/scripts/frank-large.env" +REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" + +@test "frank-large unit, serve script and env exist in scripts/" { + [ -f "$UNIT" ] + [ -f "$SERVE" ] + [ -f "$ENV_FILE" ] +} + +@test "frank-large unit is boot-persistent, self-healing and unprivileged" { + grep -q '^WantedBy=multi-user.target$' "$UNIT" + grep -q '^Restart=on-failure$' "$UNIT" + grep -q '^RestartSec=' "$UNIT" + grep -q '^User=dank$' "$UNIT" + grep -q '^LimitMEMLOCK=infinity$' "$UNIT" +} + +@test "frank-large unit points at the repo serve script and env" { + grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh$' "$UNIT" + grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT" +} + +@test "frank-large places experts and PLE on the mmap'd CPU path, never the pinned buffer" { + # --cpu-moe routes CPU-overridden weights through the CUDA_Host PINNED + # buffer (108.12 GiB measured on 2026-09-15; it evicted every co-tenant + # into swap and starved sshd into unreachability). The -ot ...=CPU plain + # CPU backend buffer is the mmap-backed path. + grep -q -- '-ot "exps=CPU" -ot "per_layer_token_embd=CPU"' "$SERVE" + grep -q '^GGML_CUDA_NO_PINNED_WEIGHTS=1$' "$ENV_FILE" + # The serve script refuses to run without the no-pinned policy. + grep -q 'GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host' "$SERVE" +} + +@test "frank-large serves the ik runtime from the unified-memory hybrid layout" { + grep -q 'ik_llama.cpp/build/bin/llama-server' "$ENV_FILE" + grep -q 'mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M' "$ENV_FILE" + grep -q -- '--gpu-layers "$NGL"' "$SERVE" + grep -q -- '--flash-attn "$FLASH_ATTN"' "$SERVE" +} + +@test "frank-large keeps the fleet identity and the client port" { + grep -q '^LISTEN_PORT=7770$' "$ENV_FILE" + grep -q '^ALIAS=frank-large$' "$ENV_FILE" + grep -q -- '--port "$LISTEN_PORT"' "$SERVE" + grep -q -- '--host "$BIND_ADDR"' "$SERVE" +} + +@test "frank-large decodes greedy by default (halogen client contract)" { + grep -q '^TEMPERATURE=0.0$' "$ENV_FILE" + grep -q -- '--temp "$TEMPERATURE"' "$SERVE" +} + +@test "frank-large keeps image input via the mtmd mmproj" { + grep -q '^MMPROJ_PATH=.*mmproj-f16.gguf$' "$ENV_FILE" + grep -q -- '--mmproj "$MMPROJ_PATH"' "$SERVE" +} + +@test "the deploy handler retires the halogen quadlet without deleting it" { + grep -q 'retire_quadlet_llm_large' "$REMOTE_DEPLOY" + grep -q 'systemctl --user stop frank-large.service' "$REMOTE_DEPLOY" + grep -q 'frank-large.container' "$REMOTE_DEPLOY" + grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY" + ! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?.*frank-large' "$REMOTE_DEPLOY" +} + +@test "the deploy handler installs the root unit and refuses a missing checkpoint" { + grep -q 'relink_unit frank-large' "$REMOTE_DEPLOY" + grep -q 'systemctl enable frank-large.service' "$REMOTE_DEPLOY" + grep -q 'checkpoint absent' "$REMOTE_DEPLOY" + ! grep -q 'install_quadlet frank-large' "$REMOTE_DEPLOY" +}