Frank
3066d810ee
- runtime: charlie build-qwen38 -> kingjones30/ROCmFPX (card recipe build, LLAMA_BUILD_WEBUI=OFF since npm/HF-bucket provisioning fails; all 35 of our serving flags verified present in the tree) - model: agentionai FAST imatrix -> kingjones777 STRIX single GGUF (on-host copy sha256 == HF LFS oid c6770d74...49fa verified) - args unchanged from our validated profile (401408x2, b4096/ub512, t16/tb16, f16 KV, fa on, fit off); mmap stays default per card (PLE streams via page cache; --no-mmap = silent cgroup OOM)
126 lines
5.4 KiB
Bash
126 lines
5.4 KiB
Bash
# Tunables for frank-default-large (kingjones777/Qwen3.8-Flash-Next-ROCmFP4-
|
|
# STRIX-GGUF on the kingjones30/ROCmFPX llama.cpp runtime, Dankclaw,
|
|
# Strix Halo gfx1151, unified memory).
|
|
#
|
|
# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the
|
|
# qwen4exp-on-rocmfpx-d3ca537 patch applied — the model card's pinned
|
|
# runtime: ROCmFP4 tensor types AND the qwen4exp architecture in one
|
|
# tree; stock llama.cpp and charlie's fork alone cannot load this GGUF).
|
|
# Built per the card recipe:
|
|
# cmake -B build -DGGML_HIP=ON -DGPU_TARGETS=gfx1151 -DGGML_NATIVE=ON -DCMAKE_BUILD_TYPE=Release
|
|
# at /home/dank/GitPackages/kingjones-ROCmFPX/build.
|
|
# Replaced the agentionai ROCmFP4-FAST imatrix deployment (charlie's
|
|
# build-qwen38 runtime) on 2026-08-30.
|
|
#
|
|
# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used,
|
|
# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE
|
|
# at layer 1), Q4_0_ROCMFP4_STRIX (file_type 105), 113.47 GiB single
|
|
# file. STRIX is the top quality tier: PLE table at Q8_0 (50.66 GiB),
|
|
# token embeddings + lm head at Q6_K. The PLE table NEVER enters GPU
|
|
# memory — it streams from the file through the OS page cache. Two hard
|
|
# card constraints:
|
|
# 1. mmap MUST stay enabled (default) — --no-mmap forces the 50.66 GiB
|
|
# table into anonymous memory and the cgroup killer takes the
|
|
# process with nothing in the server log (only dmesg).
|
|
# 2. Do NOT pin the PLE tensor to CPU with -ot — measured gen fell
|
|
# ~23 -> 13.4 tok/s; the kernel streams it better.
|
|
# NOTE: the card's own build_attn_qsa still calls ggml_fill on the
|
|
# F16-cast kq mask (the path that aborted our charlie build with
|
|
# --flash-attn on before scripts/patches/qwen4exp-qsa-f32-mask.patch).
|
|
# If a clean kingjones-tree build aborts graph build with
|
|
# GGML_ASSERT(a-type == GGML_TYPE_F32), port that fix (F32 base mask +
|
|
# cast at the attention call) into this tree and rebuild.
|
|
#
|
|
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
|
|
# incident). llama.cpp reads --host/--port/--temp as CLI flags, never as
|
|
# env vars.
|
|
|
|
# kingjones30/ROCmFPX runtime binary (card recipe build).
|
|
LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server
|
|
|
|
# Model package from
|
|
# https://huggingface.co/kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF
|
|
# Layout: single unsharded GGUF, text-only (the repo ships no mmproj;
|
|
# the vision tower below is borrowed from the agentionai FAST package —
|
|
# same base model, f16 — kept only if it loads clean, drop
|
|
# MMPROJ_PATH if it does not).
|
|
MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX
|
|
MODEL_FILE=Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf
|
|
MMPROJ_PATH=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf
|
|
|
|
# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
|
|
# Sharp template drives thinking on by default (medium effort); the server
|
|
# emits reasoning in DeepSeek format so clients receive reasoning_content
|
|
# separately instead of literal thinking tags in content.
|
|
CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
|
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'
|
|
|
|
# Alias + network.
|
|
ALIAS=frank-default-large
|
|
BIND_ADDR=0.0.0.0
|
|
LISTEN_PORT=7770
|
|
|
|
# Serving: 401408-token context, TWO slots (parallel 2), batch 4096 /
|
|
# ubatch 512, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
|
|
# prompt cache. --fit off: auto-fit segfaults on this model's
|
|
# hyper-connection/indexer config; -ngl 999 puts every layer on the
|
|
# SVM-backed GPU pool. ubatch 512 matches the card's re-measurement
|
|
# (-ub 1024/2048/4096 showed no effect; -t 32 vs 16 is noise).
|
|
#
|
|
# CTX_SIZE=401408 / PARALLEL=2: carried over from the FAST deployment
|
|
# where it loaded clean (~74 s, health 200). The card measures the full
|
|
# 262144 window at ~71.7 GiB GTT at load on the same 128 GB board; QSA
|
|
# caps KV growth so the extra window is cheap. KV footprint scales
|
|
# with the 12 attention layers only. Verify load on first STRIX boot.
|
|
CTX_SIZE=401408
|
|
PARALLEL=2
|
|
|
|
BATCH_SIZE=4096
|
|
UBATCH_SIZE=512
|
|
NUM_THREADS=16
|
|
THREAD_BATCH=16
|
|
CACHE_TYPE_K=f16
|
|
CACHE_TYPE_V=f16
|
|
CTX_CHECKPOINTS=16
|
|
CACHE_RAM=8192
|
|
SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state
|
|
SPLIT_MODE=none
|
|
FIT=off
|
|
NGL=999
|
|
FLASH_ATTN=on
|
|
|
|
# Sampling (card instruct table, non-thinking).
|
|
TEMPERATURE=0.7
|
|
TOP_K=20
|
|
TOP_P=0.8
|
|
MIN_P=0.0
|
|
PRESENCE_PENALTY=1.5
|
|
|
|
# Reasoning on (Sharp template drives it; preserve_thinking + preserve_reasoning
|
|
# keep the reasoning trail in the conversation). DeepSeek format projects the
|
|
# reasoning into the API's reasoning_content field instead of literal thinking
|
|
# tags.
|
|
REASONING=on
|
|
REASONING_FORMAT=deepseek
|
|
|
|
# Strix Halo unified memory + native gfx1151.
|
|
HSA_OVERRIDE_GFX_VERSION=11.5.1
|
|
|
|
VERBOSITY=4
|
|
|
|
|
|
# Speculative decoding: disabled. The card's Usage block ships no spec
|
|
# flags. The Flash-Next MTP heads do not load on this loader (requires
|
|
# per-layer blk.N.hc_attn_norm; every head carries only blk.48 —
|
|
# verified 3x on build-qwen38). DRAFT_MODEL_* point at the retired
|
|
# agentionai FAST head and are unused while SPEC_TYPE=none.
|
|
DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
|
|
DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf
|
|
SPEC_TYPE=none
|
|
SPEC_DRAFT_N_MAX=3
|
|
SPEC_DRAFT_N_MIN=0
|
|
SPEC_DRAFT_P_MIN=0.75
|
|
SPEC_DRAFT_NGL=999
|
|
SPEC_DRAFT_DEVICE=ROCm0
|
|
SPEC_DRAFT_TYPE_K=f16
|
|
SPEC_DRAFT_TYPE_V=f16 |