frank-large: swap halogen Quadlet for ik_llama.cpp (3090 + Strix Halo CPU path)

scripts/serve-frank-large.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-large
+# via ik_llama.cpp on Dankclaw.
+#
+# Topology (ikawrakow's hybrid recipe, upstream issue #2402): the RTX 3090
+# (CUDA, sm_86) takes the dense/attention/output weights and the KV cache
+# (-ngl 999); ALL MoE expert tensors stay in the Strix Halo's LPDDR5X via
+# the CPU path (--cpu-moe) — ik's fastest known configuration for this
+# hybrid shape. The ~35 GB PLE n-gram table (per_layer_token_embd) is
+# mmap-resident BY DESIGN: only a tiny slice is touched per token, so it
+# streams from the OS page cache. NEVER add --no-mmap — the STRIX-era
+# lesson transfers directly (the cgroup killer takes the process with
+# nothing in the server log).
+#
+# Runtime: ik_llama.cpp master d85a1d7 built with CUDA (sm_86) + Zen5
+# AVX-512 IQK kernels; the startup banner must show HAVE_FANCY_SIMD
+# is defined. Flag set verified against this binary's --help before this
+# script was written. ik has no --cache-prompt/--cache-idle-slots (prompt
+# caching is its default behavior) and no --alias: it is a single-model
+# server and reports the GGUF's general.name at /v1/models.
+#
+# Sampling: greedy-by-default (halogen client contract) — clients that
+# send their own temperature get their own values.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -------------------
+: "${LLAMA_BIN:=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server}"
+: "${MODEL_DIR:=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M}"
+: "${MODEL_FILE:=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf}"
+: "${MMPROJ_PATH:=$MODEL_DIR/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf}"
+: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
+: "${BIND_ADDR:=0.0.0.0}"
+: "${LISTEN_PORT:=7770}"
+: "${CTX_SIZE:=262144}"
+: "${PARALLEL:=1}"
+: "${BATCH_SIZE:=4096}"
+: "${UBATCH_SIZE:=512}"
+: "${NUM_THREADS:=16}"
+: "${THREAD_BATCH:=16}"
+: "${CACHE_TYPE_K:=f16}"
+: "${CACHE_TYPE_V:=f16}"
+: "${NGL:=999}"
+: "${CPU_MOE:=on}"
+: "${FLASH_ATTN:=on}"
+: "${CACHE_RAM:=8192}"
+: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}"
+: "${TEMPERATURE:=0.0}"
+: "${REASONING:=on}"
+: "${REASONING_FORMAT:=deepseek}"
+: "${REASONING_BUDGET:=-1}"
+
+mkdir -p "$SLOT_SAVE_PATH"
+
+# The UVM device node is created lazily by nvidia-modprobe on the first
+# CUDA open after boot; make sure it exists before the server needs it.
+nvidia-modprobe -u -c=0 2>/dev/null || true
+
+CPU_MOE_ARGS=()
+if [[ "$CPU_MOE" == "on" ]]; then
+    CPU_MOE_ARGS=(--cpu-moe)
+fi
+
+MMPROJ_ARGS=()
+if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
+    MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
+fi
+
+TEMPLATE_ARGS=()
+if [[ -n "${CHAT_TEMPLATE_FILE:-}" && -f "$CHAT_TEMPLATE_FILE" ]]; then
+    TEMPLATE_ARGS=(--chat-template-file "$CHAT_TEMPLATE_FILE")
+else
+    TEMPLATE_ARGS=(--jinja)
+fi
+
+exec "$LLAMA_BIN" \
+    --model "$MODEL_DIR/$MODEL_FILE" \
+    "${TEMPLATE_ARGS[@]}" \
+    --host "$BIND_ADDR" \
+    --port "$LISTEN_PORT" \
+    --gpu-layers "$NGL" \
+    "${CPU_MOE_ARGS[@]}" \
+    --flash-attn "$FLASH_ATTN" \
+    --parallel "$PARALLEL" \
+    --ctx-size "$CTX_SIZE" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --threads "$NUM_THREADS" \
+    --threads-batch "$THREAD_BATCH" \
+    --cache-type-k "$CACHE_TYPE_K" \
+    --cache-type-v "$CACHE_TYPE_V" \
+    --cache-ram "$CACHE_RAM" \
+    --slot-save-path "$SLOT_SAVE_PATH" \
+    --ctx-checkpoints 16 \
+    --metrics \
+    --temp "$TEMPERATURE" \
+    --reasoning "$REASONING" \
+    --reasoning-format "$REASONING_FORMAT" \
+    --reasoning-budget "$REASONING_BUDGET" \
+    "${MMPROJ_ARGS[@]}"