118 lines
5.1 KiB
Bash
118 lines
5.1 KiB
Bash
# Tunables for frank-default-large (agentionai/Qwen3.8-Flash-Next-ROCmFP4-
|
|
# FAST-imatrix-GGUF on the charlie12345/ROCmFPX llama.cpp runtime,
|
|
# build-qwen38, Dankclaw, Strix Halo gfx1151 unified memory).
|
|
#
|
|
# Runtime: charlie12345/ROCmFPX llama.cpp at
|
|
# /home/dank/GitPackages/ROCmFPX/build-qwen38 (HIP, ROCm 7.2.4, gfx1151,
|
|
# PR #27742 qwen4exp merged). The ROCmFP4 quant types are hand-ported from
|
|
# ciru-ai/ROCmFPX; upstream llama.cpp cannot load this GGUF. Replaced the
|
|
# jcbtc CIRU v1.1 (ciru-ai runtime) and kingjones STRIX_LEAN deployments.
|
|
#
|
|
# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used,
|
|
# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE
|
|
# KNOWN CONSTRAINTS on the charlie12345/ROCmFPX build-qwen38 runtime.
|
|
# 1. FA must stay OFF. --flash-attn on aborts graph build: the QSA
|
|
# indexer path calls ggml_fill on the F16-cast kq mask (ggml_fill
|
|
# asserts GGML_TYPE_F32 at ggml.c:5371). The v2 main loads and serves
|
|
# fine with --flash-attn off (verified live: model loaded, health 200,
|
|
# real completion).
|
|
# 2. The MTP draft head does NOT load on this binary. The qwen4exp loader
|
|
# requires blk.0.hc_attn_norm.weight per layer; the agentionai FAST
|
|
# head carries only blk.48 (verified 3x: EasiiX Q8_0 + FAST against
|
|
# kingjones STRIX_LEAN, FAST against this v2 main). Speculative
|
|
# decoding therefore runs ngram-mod only. The 2.28 GiB head stays on
|
|
# disk in case a compatible binary/head pair appears.
|
|
#
|
|
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
|
|
# incident). llama.cpp reads --host/--port/--temp as CLI flags, never as
|
|
# env vars.
|
|
|
|
# ROCmFPX (build-qwen38) runtime binary.
|
|
LLAMA_BIN=/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server
|
|
|
|
# Model package from
|
|
# https://huggingface.co/agentionai/Qwen3.8-Flash-Next-ROCmFP4-FAST-imatrix-GGUF
|
|
# Layout: v2/ joined-layout GGUF (PLE n-gram table as a single
|
|
# per_layer_token_embd.weight tensor — the layout the charlie12345/ROCmFPX
|
|
# qwen4exp loader requires; the root -ple16 file splits the table per head
|
|
# and only loads in the LaurentZuijdwijk Vulkan fork). mmproj/ vision tower.
|
|
# The MTP draft head lives in /home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
|
|
# (already on the host).
|
|
MODEL_DIR=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/v2
|
|
MODEL_FILE=Qwen3.8-Flash-Next-ROCmFP4-FAST-imatrix-v2.gguf
|
|
MMPROJ_PATH=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf
|
|
|
|
# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
|
|
# Sharp template drives thinking on by default (medium effort); the server
|
|
# emits reasoning in DeepSeek format so clients receive reasoning_content
|
|
# separately instead of literal thinking tags in content.
|
|
CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
|
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'
|
|
|
|
# Alias + network.
|
|
ALIAS=frank-default-large
|
|
BIND_ADDR=0.0.0.0
|
|
LISTEN_PORT=7770
|
|
|
|
# Serving: 131072-token context on ONE slot (parallel 1), batch 4096 /
|
|
# ubatch 4096, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
|
|
# prompt cache. --fit off: auto-fit segfaults on this model's
|
|
# hyper-connection/indexer config; -ngl 999 puts every layer on the
|
|
# SVM-backed GPU pool.
|
|
#
|
|
# CTX_SIZE is capped at 131072: the fully GPU-resident 87 GiB model plus a
|
|
# 262144-token f16 KV cache and the worst-case compute buffer exceed the
|
|
# Strix Halo SVM pool — verified live (cudaMalloc 103176 MiB OOM at
|
|
# 262144). 131072 is the proven STRIX-family value on this box.
|
|
CTX_SIZE=131072
|
|
PARALLEL=1
|
|
BATCH_SIZE=4096
|
|
UBATCH_SIZE=4096
|
|
NUM_THREADS=16
|
|
THREAD_BATCH=16
|
|
CACHE_TYPE_K=f16
|
|
CACHE_TYPE_V=f16
|
|
CTX_CHECKPOINTS=16
|
|
CACHE_RAM=8192
|
|
SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state
|
|
SPLIT_MODE=none
|
|
FIT=off
|
|
NGL=999
|
|
FLASH_ATTN=off
|
|
|
|
# Sampling (card instruct table, non-thinking).
|
|
TEMPERATURE=0.7
|
|
TOP_K=20
|
|
TOP_P=0.8
|
|
MIN_P=0.0
|
|
PRESENCE_PENALTY=1.5
|
|
|
|
# Reasoning on (Sharp template drives it; preserve_thinking + preserve_reasoning
|
|
# keep the reasoning trail in the conversation). DeepSeek format projects the
|
|
# reasoning into the API's reasoning_content field instead of literal thinking
|
|
# tags.
|
|
REASONING=on
|
|
REASONING_FORMAT=deepseek
|
|
|
|
# Strix Halo unified memory + native gfx1151.
|
|
HSA_OVERRIDE_GFX_VERSION=11.5.1
|
|
|
|
VERBOSITY=4
|
|
|
|
# Speculative decoding: ngram-mod only. The MTP draft head is retained
|
|
# (agentionai Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST-GGUF, 2.28 GiB) but does
|
|
# NOT load on build-qwen38 (loader requires blk.0.hc_attn_norm.weight the
|
|
# head lacks — see KNOWN CONSTRAINTS). If a compatible binary appears, set
|
|
# SPEC_TYPE=draft-mtp,ngram-mod (n-max 3 measured best at 32.4 t/s / 0.612
|
|
# acceptance; adaptive p-split measured worse, stays off). SPEC_TYPE=none
|
|
# disables speculative decoding entirely.
|
|
DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
|
|
DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf
|
|
SPEC_TYPE=ngram-mod
|
|
SPEC_DRAFT_N_MAX=3
|
|
SPEC_DRAFT_N_MIN=0
|
|
SPEC_DRAFT_P_MIN=0.75
|
|
SPEC_DRAFT_NGL=999
|
|
SPEC_DRAFT_DEVICE=ROCm0
|
|
SPEC_DRAFT_TYPE_K=f16
|
|
SPEC_DRAFT_TYPE_V=f16 |