96 lines
4.4 KiB
Bash
96 lines
4.4 KiB
Bash
# Tunables for frank-large (mradermacher i1-Q4_K_M of
|
|
# orcarouter/Qwen3.8-Flash-Next-Uncensored on ik_llama.cpp, Dankclaw,
|
|
# RTX 3090 CUDA + Strix Halo CPU path, since 2026-09-15).
|
|
#
|
|
# Runtime: ik_llama.cpp master d85a1d7 (2026-09-15) with full qwen4exp
|
|
# support (upstream PRs #2365/#2369/#2373 merged). Build recipe (CUDA for
|
|
# the 3090 at sm_86, Zen5 AVX-512 IQK kernels for the CPU expert path):
|
|
# cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON \
|
|
# -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 \
|
|
# -DGGML_AVX512=ON -DGGML_AVX512_VBMI=ON -DGGML_AVX512_VNNI=ON \
|
|
# -DGGML_AVX512_BF16=ON -DLLAMA_CURL=OFF
|
|
# The startup banner must print HAVE_FANCY_SIMD is defined — without those
|
|
# macros the quantized GEMM silently falls back to AVX2 (no warning).
|
|
#
|
|
# Model: mradermacher imatrix quant of the uncensored Flash-Next (qwen4exp
|
|
# MoE: 512 experts / 10 used, 48 layers = 12 attention + 36 GDN/ssm, QSA
|
|
# indexer, hyper-connections, PLE n-gram table ~320M rows). 111 GiB shipped
|
|
# as 3 parts, concatenated on disk; each part's sha256 was verified against
|
|
# its HF LFS oid at download time.
|
|
#
|
|
# Placement (ikawrakow's hybrid recipe, upstream issue #2402):
|
|
# - --cpu-moe: ALL expert tensors (121B elems, ~71 GB) live in the Strix
|
|
# Halo's LPDDR5X via the CPU path — ik's fastest known configuration
|
|
# for this shape (the author's 2x3090 + Threadripper reference box
|
|
# measures ~50 t/s on the same model family).
|
|
# - -ngl 999: dense/attention/output weights and the KV cache on the
|
|
# 3090 (~3 GB of weights + ~6 GB KV at 262k f16; 24 GB card).
|
|
# - The ~35 GB PLE table (per_layer_token_embd) is intentionally
|
|
# RAM/mmap-resident by design — only a tiny slice is touched per token.
|
|
# NEVER add --no-mmap (the STRIX-era lesson transfers: the cgroup
|
|
# killer takes the process with nothing in the server log) and do not
|
|
# try to pin PLE to VRAM (it cannot fit a 24 GB card).
|
|
#
|
|
# Sampling: greedy-by-default to preserve the halogen-era client contract
|
|
# (a request without temperature decodes greedy; temp 0.0 = argmax).
|
|
# Clients that send their own temperature get their own values.
|
|
#
|
|
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (see
|
|
# the 2026-08-13 HIP incident in AGENTS.md). llama.cpp reads --host and
|
|
# --temp as CLI flags, never as environment variables.
|
|
|
|
# ik_llama.cpp runtime binary (master d85a1d7 build).
|
|
LLAMA_BIN=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server
|
|
|
|
# Model package from
|
|
# https://huggingface.co/mradermacher/Qwen3.8-Flash-Next-Uncensored-i1-GGUF
|
|
# (parts concatenated). The mmproj is the matching vision tower from the
|
|
# static repo mradermacher/Qwen3.8-Flash-Next-Uncensored-GGUF — ik_llama.cpp
|
|
# links examples/mtmd, so image input survives the runtime swap.
|
|
MODEL_DIR=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M
|
|
MODEL_FILE=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf
|
|
MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf
|
|
|
|
# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
|
|
# server emits reasoning in DeepSeek format so clients receive
|
|
# reasoning_content separately instead of literal thinking tags in content.
|
|
CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
|
|
|
# Alias + network.
|
|
ALIAS=frank-large
|
|
BIND_ADDR=0.0.0.0
|
|
LISTEN_PORT=7770
|
|
|
|
# Serving: single 262144-token slot (the model's full context window),
|
|
# batch 4096 / ubatch 512, 16+16 threads (the Halo CPU has 16 cores, no
|
|
# SMT), f16 KV. The KV cache scales with the 12 full-attention layers only
|
|
# (~6 GB f16 at 262k); the GDN/ssm layers keep constant-size state. If
|
|
# VRAM headroom is needed, --cache-type-k/v q8_0 halves the KV footprint.
|
|
CTX_SIZE=262144
|
|
PARALLEL=1
|
|
BATCH_SIZE=4096
|
|
UBATCH_SIZE=512
|
|
NUM_THREADS=16
|
|
THREAD_BATCH=16
|
|
CACHE_TYPE_K=f16
|
|
CACHE_TYPE_V=f16
|
|
|
|
# GPU/MoE placement: everything on the 3090 except the experts, which all
|
|
# live in host RAM (--cpu-moe). See the placement notes above.
|
|
NGL=999
|
|
CPU_MOE=on
|
|
FLASH_ATTN=on
|
|
|
|
# Prompt cache (RAM budget in MiB) and slot state dir.
|
|
CACHE_RAM=8192
|
|
SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state
|
|
|
|
# Greedy-by-default (halogen contract). See sampling note above.
|
|
TEMPERATURE=0.0
|
|
|
|
# Reasoning on: DeepSeek format projects thinking into the API's
|
|
# reasoning_content field instead of literal think tags in content;
|
|
# unlimited reasoning budget (the model decides).
|
|
REASONING_FORMAT=deepseek
|
|
REASONING_BUDGET=-1
|