106 lines
5.2 KiB
Bash
106 lines
5.2 KiB
Bash
# Tunables for frank-large (mradermacher i1-Q4_K_M of
|
|
# orcarouter/Qwen3.8-Flash-Next-Uncensored on ik_llama.cpp, Dankclaw,
|
|
# RTX 3090 CUDA + Strix Halo CPU path, since 2026-09-15).
|
|
#
|
|
# Runtime: ik_llama.cpp master d85a1d7 (2026-09-15) with full qwen4exp
|
|
# support (upstream PRs #2365/#2369/#2373 merged). Build recipe (CUDA for
|
|
# the 3090 at sm_86, Zen5 AVX-512 IQK kernels for the CPU expert path):
|
|
# cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON \
|
|
# -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 \
|
|
# -DGGML_AVX512=ON -DGGML_AVX512_VBMI=ON -DGGML_AVX512_VNNI=ON \
|
|
# -DGGML_AVX512_BF16=ON -DLLAMA_CURL=OFF
|
|
# The startup banner must print HAVE_FANCY_SIMD is defined — without those
|
|
# macros the quantized GEMM silently falls back to AVX2 (no warning).
|
|
#
|
|
# Model: mradermacher imatrix quant of the uncensored Flash-Next (qwen4exp
|
|
# MoE: 512 experts / 10 used, 48 layers = 12 attention + 36 GDN/ssm, QSA
|
|
# indexer, hyper-connections, PLE n-gram table ~320M rows). 111 GiB shipped
|
|
# as 3 parts, concatenated on disk; each part's sha256 was verified against
|
|
# its HF LFS oid at download time.
|
|
#
|
|
# Placement (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
|
|
# - -ot "exps=CPU": ALL 512-expert tensors (121B elems, ~84 GB) live in
|
|
# the Strix Halo's LPDDR5X via the plain CPU backend buffer — mmap-backed
|
|
# page cache, ik's fastest known configuration for this shape (the
|
|
# author's 2x3090 + Threadripper reference box measures ~50 t/s on the
|
|
# same model family). The shared experts (shexp) keep their GPU-side
|
|
# default: they run on every token.
|
|
# - -ot "per_layer_token_embd=CPU": the ~32 GB PLE n-gram table follows —
|
|
# only a tiny slice is touched per token, so it streams from page cache.
|
|
# - NEVER use --cpu-moe on this host: it routes the CPU-overridden
|
|
# weights through the CUDA_Host PINNED buffer (measured 2026-09-15:
|
|
# 108.12 GiB page-locked, evicted every co-tenant into swap and starved
|
|
# sshd into unreachability until a power cycle). -ot ...=CPU is the
|
|
# mmap-backed path with identical DDR5 bandwidth for resident pages,
|
|
# and the kernel degrades gracefully under pressure (page cache evict
|
|
# + re-read) instead of stranding anyone.
|
|
# - GGML_CUDA_NO_PINNED_WEIGHTS=1 (src/llama-load-tensors.cpp) keeps the
|
|
# CPU-overridden weights mmapped and leaves only the small staging
|
|
# buffers pinned. (GGML_CUDA_NO_PINNED would also unpin staging —
|
|
# unnecessary here and slower for prefill transfers.)
|
|
#
|
|
# Sampling: greedy-by-default to preserve the halogen-era client contract
|
|
# (a request without temperature decodes greedy; temp 0.0 = argmax).
|
|
# Clients that send their own temperature get their own values.
|
|
#
|
|
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (see
|
|
# the 2026-08-13 HIP incident in AGENTS.md). llama.cpp reads --host and
|
|
# --temp as CLI flags, never as environment variables.
|
|
|
|
# ik_llama.cpp runtime binary (master d85a1d7 build).
|
|
LLAMA_BIN=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server
|
|
|
|
# Model package from
|
|
# https://huggingface.co/mradermacher/Qwen3.8-Flash-Next-Uncensored-i1-GGUF
|
|
# (parts concatenated). The mmproj is the matching vision tower from the
|
|
# static repo mradermacher/Qwen3.8-Flash-Next-Uncensored-GGUF — ik_llama.cpp
|
|
# links examples/mtmd, so image input survives the runtime swap.
|
|
MODEL_DIR=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M
|
|
MODEL_FILE=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf
|
|
MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf
|
|
|
|
# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
|
|
# server emits reasoning in DeepSeek format so clients receive
|
|
# reasoning_content separately instead of literal think tags in content.
|
|
CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
|
|
|
# Alias + network.
|
|
ALIAS=frank-large
|
|
BIND_ADDR=0.0.0.0
|
|
LISTEN_PORT=7770
|
|
|
|
# Serving: single 262144-token slot (the model's full context window),
|
|
# batch 4096 / ubatch 512, 16+16 threads (the Halo CPU has 16 cores, no
|
|
# SMT), f16 KV. The KV cache scales with the 12 full-attention layers only
|
|
# (~6 GB f16 at 262k); the GDN/ssm layers keep constant-size state. If
|
|
# VRAM headroom is needed, --cache-type-k/v q8_0 halves the KV footprint.
|
|
CTX_SIZE=262144
|
|
PARALLEL=1
|
|
BATCH_SIZE=4096
|
|
UBATCH_SIZE=512
|
|
NUM_THREADS=16
|
|
THREAD_BATCH=16
|
|
CACHE_TYPE_K=f16
|
|
CACHE_TYPE_V=f16
|
|
|
|
# GPU/MoE placement: everything on the 3090 except the experts and the PLE
|
|
# table, which both live in host RAM via the CPU backend buffer (see the
|
|
# placement notes above).
|
|
NGL=999
|
|
# ik_llama.cpp CUDA host-memory policy: CPU-overridden weights stay mmapped;
|
|
# only the small staging buffers are pinned (see the placement notes above).
|
|
GGML_CUDA_NO_PINNED_WEIGHTS=1
|
|
|
|
# Prompt cache (RAM budget in MiB) and slot state dir.
|
|
CACHE_RAM=8192
|
|
SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state
|
|
|
|
# Greedy-by-default (halogen contract). See sampling note above.
|
|
TEMPERATURE=0.0
|
|
|
|
# Reasoning on: DeepSeek format projects thinking into the API's
|
|
# reasoning_content field instead of literal think tags in content;
|
|
# unlimited reasoning budget (the model decides).
|
|
REASONING_FORMAT=deepseek
|
|
REASONING_BUDGET=-1
|