Frank
1ca3db4ae2
- UBATCH_SIZE: 4096 -> 512 (reduce GPU memory pressure) - SPEC_TYPE: ngram-mod -> none (disable speculative decoding entirely)
117 lines
4.9 KiB
Bash
117 lines
4.9 KiB
Bash
# Tunables for frank-default-large (agentionai/Qwen3.8-Flash-Next-ROCmFP4-
|
||
# FAST-imatrix-GGUF on the charlie12345/ROCmFPX llama.cpp runtime,
|
||
# build-qwen38, Dankclaw, Strix Halo gfx1151 unified memory).
|
||
#
|
||
# Runtime: charlie12345/ROCmFPX llama.cpp at
|
||
# /home/dank/GitPackages/ROCmFPX/build-qwen38 (HIP, ROCm 7.2.4, gfx1151,
|
||
# PR #27742 qwen4exp merged). The ROCmFP4 quant types are hand-ported from
|
||
# ciru-ai/ROCmFPX; upstream llama.cpp cannot load this GGUF. Replaced the
|
||
# jcbtc CIRU v1.1 (ciru-ai runtime) and kingjones STRIX_LEAN deployments.
|
||
#
|
||
# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used,
|
||
# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE
|
||
# 1. Flash attention is ON. fa on used to abort graph build: the QSA
|
||
# indexer path called ggml_fill on the F16-cast kq mask (ggml_fill
|
||
# asserts GGML_TYPE_F32 at ggml.c:5371). Fixed in the fork source by
|
||
# the QSA F32-mask patch (scripts/patches/qwen4exp-qsa-f32-mask.patch,
|
||
# sha256 f2f4a486e21cfad80bfa42683281d18e1cba22fa43c5e0e2aa0cfdbe141e925f)
|
||
# and the rebuilt binary at LLAMA_BIN already carries the fix. A clean
|
||
# rebuild must re-apply: git apply scripts/patches/qwen4exp-qsa-f32-mask.patch
|
||
# inside /home/dank/GitPackages/ROCmFPX before make. Verified live:
|
||
# --flash-attn on + v2 main at CTX_SIZE=32768 loads, health 200, real
|
||
# completion.
|
||
# 2. The MTP draft head does NOT load on this binary. The qwen4exp loader
|
||
# requires blk.0.hc_attn_norm.weight per layer; the agentionai FAST
|
||
# head carries only blk.48 (verified 3x: EasiiX Q8_0 + FAST against
|
||
# kingjones STRIX_LEAN, FAST against this v2 main). Speculative
|
||
# decoding therefore runs ngram-mod only. The 2.28 GiB head stays on
|
||
# disk in case a compatible binary/head pair appears.
|
||
#
|
||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
|
||
# incident). llama.cpp reads --host/--port/--temp as CLI flags, never as
|
||
# env vars.
|
||
|
||
# ROCmFPX (build-qwen38) runtime binary.
|
||
LLAMA_BIN=/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server
|
||
|
||
# Model package from
|
||
# https://huggingface.co/agentionai/Qwen3.8-Flash-Next-ROCmFP4-FAST-imatrix-GGUF
|
||
# Layout: v2/ joined-layout GGUF (PLE n-gram table as a single
|
||
# per_layer_token_embd.weight tensor — the layout the charlie12345/ROCmFPX
|
||
# qwen4exp loader requires; the root -ple16 file splits the table per head
|
||
# and only loads in the LaurentZuijdwijk Vulkan fork). mmproj/ vision tower.
|
||
# The MTP draft head lives in /home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
|
||
# (already on the host).
|
||
MODEL_DIR=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/v2
|
||
MODEL_FILE=Qwen3.8-Flash-Next-ROCmFP4-FAST-imatrix-v2.gguf
|
||
MMPROJ_PATH=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf
|
||
|
||
# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
|
||
# Sharp template drives thinking on by default (medium effort); the server
|
||
# emits reasoning in DeepSeek format so clients receive reasoning_content
|
||
# separately instead of literal thinking tags in content.
|
||
CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
||
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'
|
||
|
||
# Alias + network.
|
||
ALIAS=frank-default-large
|
||
BIND_ADDR=0.0.0.0
|
||
LISTEN_PORT=7770
|
||
|
||
# Serving: 401408-token context, TWO slots (parallel 2), batch 4096 /
|
||
# ubatch 512, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
|
||
# prompt cache. --fit off: auto-fit segfaults on this model's
|
||
# hyper-connection/indexer config; -ngl 999 puts every layer on the
|
||
# SVM-backed GPU pool.
|
||
#
|
||
# CTX_SIZE=401408 / PARALLEL=2: untested experiment. At 32768×1 the
|
||
# context buffer was ~13 GiB and total ~100 GiB in the 119.3 GiB HIP
|
||
# heap. Doubling slots × 12× context could OOM — verify load.
|
||
CTX_SIZE=401408
|
||
PARALLEL=2
|
||
|
||
BATCH_SIZE=4096
|
||
UBATCH_SIZE=512
|
||
NUM_THREADS=16
|
||
THREAD_BATCH=16
|
||
CACHE_TYPE_K=f16
|
||
CACHE_TYPE_V=f16
|
||
CTX_CHECKPOINTS=16
|
||
CACHE_RAM=8192
|
||
SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state
|
||
SPLIT_MODE=none
|
||
FIT=off
|
||
NGL=999
|
||
FLASH_ATTN=on
|
||
|
||
# Sampling (card instruct table, non-thinking).
|
||
TEMPERATURE=0.7
|
||
TOP_K=20
|
||
TOP_P=0.8
|
||
MIN_P=0.0
|
||
PRESENCE_PENALTY=1.5
|
||
|
||
# Reasoning on (Sharp template drives it; preserve_thinking + preserve_reasoning
|
||
# keep the reasoning trail in the conversation). DeepSeek format projects the
|
||
# reasoning into the API's reasoning_content field instead of literal thinking
|
||
# tags.
|
||
REASONING=on
|
||
REASONING_FORMAT=deepseek
|
||
|
||
# Strix Halo unified memory + native gfx1151.
|
||
HSA_OVERRIDE_GFX_VERSION=11.5.1
|
||
|
||
VERBOSITY=4
|
||
|
||
|
||
# Speculative decoding: disabled.
|
||
DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
|
||
DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf
|
||
SPEC_TYPE=none
|
||
SPEC_DRAFT_N_MAX=3
|
||
SPEC_DRAFT_N_MIN=0
|
||
SPEC_DRAFT_P_MIN=0.75
|
||
SPEC_DRAFT_NGL=999
|
||
SPEC_DRAFT_DEVICE=ROCm0
|
||
SPEC_DRAFT_TYPE_K=f16
|
||
SPEC_DRAFT_TYPE_V=f16 |