Frank
3066d810ee
- runtime: charlie build-qwen38 -> kingjones30/ROCmFPX (card recipe build, LLAMA_BUILD_WEBUI=OFF since npm/HF-bucket provisioning fails; all 35 of our serving flags verified present in the tree) - model: agentionai FAST imatrix -> kingjones777 STRIX single GGUF (on-host copy sha256 == HF LFS oid c6770d74...49fa verified) - args unchanged from our validated profile (401408x2, b4096/ub512, t16/tb16, f16 KV, fa on, fit off); mmap stays default per card (PLE streams via page cache; --no-mmap = silent cgroup OOM)
180 lines
6.5 KiB
Bash
Executable File
180 lines
6.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF as
|
|
# frank-default-large via the kingjones30/ROCmFPX llama.cpp runtime on
|
|
# Dankclaw (Strix Halo gfx1151, unified memory).
|
|
#
|
|
# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the
|
|
# model card's qwen4exp-on-rocmfpx patch) built at
|
|
# /home/dank/GitPackages/kingjones-ROCmFPX/build (HIP, gfx1151,
|
|
# ROCm 7.2.4, card recipe). The ROCmFP4 quant types plus the qwen4exp
|
|
# architecture must be in one tree; stock llama.cpp cannot load this
|
|
# GGUF.
|
|
#
|
|
# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used,
|
|
# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE
|
|
# at layer 1) quantized Q4_0_ROCMFP4_STRIX, 113.47 GiB single file. The
|
|
# 50.66 GiB PLE n-gram table stays OUT of GPU memory and streams via
|
|
# mmap from the OS page cache — never add --no-mmap or -ot for it.
|
|
#
|
|
# Profile: card sampling table (temp 0.7, top_p 0.80, top_k 20, min_p 0.0,
|
|
# presence 1.5 — non-thinking instruct) mixed with the validated serving
|
|
# profile: ctx 401408 x 2 slots, b 4096 / ub 512, t 16 / tb 16, f16 KV,
|
|
# fa on (if the graph build aborts on the QSA F16 mask, see the note in
|
|
# the env file: port scripts/patches/qwen4exp-qsa-f32-mask.patch into
|
|
# this tree and rebuild), cont-batching, prompt cache + 8 GiB cache-ram
|
|
# + idle-slot cache, 16 ctx checkpoints, --fit off (auto-fit segfaults
|
|
# on qwen4exp hyper-connection config). Speculative decoding off: the
|
|
# Flash-Next MTP heads do not load on this loader; the card ships no
|
|
# spec flags.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -------------------
|
|
: "${LLAMA_BIN:=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}"
|
|
: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX}"
|
|
: "${MODEL_FILE:=Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf}"
|
|
: "${MMPROJ_PATH:=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf}"
|
|
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
|
|
: "${CHAT_TEMPLATE_KWARGS:={}}"
|
|
: "${ALIAS:=frank-default-large}"
|
|
: "${BIND_ADDR:=0.0.0.0}"
|
|
: "${LISTEN_PORT:=7770}"
|
|
: "${CTX_SIZE:=401408}"
|
|
: "${PARALLEL:=2}"
|
|
: "${BATCH_SIZE:=4096}"
|
|
: "${UBATCH_SIZE:=512}"
|
|
: "${NUM_THREADS:=16}"
|
|
: "${THREAD_BATCH:=16}"
|
|
: "${CACHE_TYPE_K:=f16}"
|
|
: "${CACHE_TYPE_V:=f16}"
|
|
: "${CTX_CHECKPOINTS:=16}"
|
|
: "${CACHE_RAM:=8192}"
|
|
: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}"
|
|
: "${SPLIT_MODE:=none}"
|
|
: "${FIT:=off}"
|
|
: "${NGL:=999}"
|
|
: "${FLASH_ATTN:=on}"
|
|
: "${TEMPERATURE:=0.7}"
|
|
: "${TOP_K:=20}"
|
|
: "${TOP_P:=0.8}"
|
|
: "${MIN_P:=0.0}"
|
|
: "${PRESENCE_PENALTY:=1.5}"
|
|
: "${REASONING:=on}"
|
|
: "${REASONING_FORMAT:=deepseek}"
|
|
: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
|
|
: "${SPEC_TYPE:=none}"
|
|
: "${DRAFT_MODEL_DIR:=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST}"
|
|
: "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}"
|
|
: "${SPEC_DRAFT_N_MAX:=3}"
|
|
: "${SPEC_DRAFT_N_MIN:=0}"
|
|
: "${SPEC_DRAFT_P_MIN:=0.75}"
|
|
: "${SPEC_DRAFT_NGL:=999}"
|
|
: "${SPEC_DRAFT_DEVICE:=ROCm0}"
|
|
: "${SPEC_DRAFT_TYPE_K:=f16}"
|
|
: "${SPEC_DRAFT_TYPE_V:=f16}"
|
|
|
|
MODEL="$MODEL_DIR/$MODEL_FILE"
|
|
for f in "$LLAMA_BIN" "$MODEL" "$CHAT_TEMPLATE_FILE"; do
|
|
if [[ ! -r "$f" ]]; then
|
|
echo "missing or unreadable required artifact: $f" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
if [[ ! -x "$LLAMA_BIN" ]]; then
|
|
echo "missing executable runtime: $LLAMA_BIN" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# --- speculative decoding (MTP draft head) ---------------------------------
|
|
# The MTP draft head is a separate GGUF sidecar loaded via --spec-draft-model
|
|
# (alias --model-draft). ngram-only specs do not need a draft file.
|
|
# Set SPEC_TYPE=none in the env to disable speculative decoding entirely.
|
|
SPEC_ARGS=()
|
|
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
|
|
SPEC_ARGS=(
|
|
--spec-type "$SPEC_TYPE"
|
|
--spec-draft-n-max "$SPEC_DRAFT_N_MAX"
|
|
--spec-draft-n-min "$SPEC_DRAFT_N_MIN"
|
|
--spec-draft-p-min "$SPEC_DRAFT_P_MIN"
|
|
)
|
|
if [[ "$SPEC_TYPE" == *draft-mtp* ]]; then
|
|
DRAFT_MODEL="$DRAFT_MODEL_DIR/$DRAFT_MODEL_FILE"
|
|
if [[ ! -r "$DRAFT_MODEL" ]]; then
|
|
echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
|
|
exit 1
|
|
fi
|
|
SPEC_ARGS+=(
|
|
--spec-draft-model "$DRAFT_MODEL"
|
|
--spec-draft-ngl "$SPEC_DRAFT_NGL"
|
|
--spec-draft-device "$SPEC_DRAFT_DEVICE"
|
|
--spec-draft-type-k "$SPEC_DRAFT_TYPE_K"
|
|
--spec-draft-type-v "$SPEC_DRAFT_TYPE_V"
|
|
)
|
|
fi
|
|
if [[ -n "${SPEC_DRAFT_P_SPLIT:-}" ]]; then
|
|
SPEC_ARGS+=(--spec-draft-p-split "$SPEC_DRAFT_P_SPLIT")
|
|
fi
|
|
fi
|
|
|
|
# The STRIX repo ships no mmproj; this is the agentionai FAST package's
|
|
# f16 vision tower for the same base model. Passed only when readable.
|
|
MMPROJ_ARGS=()
|
|
if [[ -n "${MMPROJ_PATH:-}" && -r "${MMPROJ_PATH:-}" ]]; then
|
|
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
|
fi
|
|
|
|
SERVER_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
mkdir -p "$SLOT_SAVE_PATH"
|
|
ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
|
|
LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
|
|
|
exec /usr/bin/env \
|
|
ROCM_PATH="$ROCM_PATH" \
|
|
HIP_VISIBLE_DEVICES=0 \
|
|
LD_LIBRARY_PATH="$LIBRARY_PATH" \
|
|
HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \
|
|
"$LLAMA_BIN" \
|
|
--model "$MODEL" \
|
|
--alias "$ALIAS" \
|
|
--chat-template-file "$CHAT_TEMPLATE_FILE" \
|
|
--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS" \
|
|
--host "$BIND_ADDR" \
|
|
--port "$LISTEN_PORT" \
|
|
--jinja \
|
|
--device ROCm0 \
|
|
--gpu-layers "$NGL" \
|
|
--split-mode "$SPLIT_MODE" \
|
|
--parallel "$PARALLEL" \
|
|
--ctx-size "$CTX_SIZE" \
|
|
--batch-size "$BATCH_SIZE" \
|
|
--ubatch-size "$UBATCH_SIZE" \
|
|
--fit "$FIT" \
|
|
--flash-attn "$FLASH_ATTN" \
|
|
--cache-type-k "$CACHE_TYPE_K" \
|
|
--cache-type-v "$CACHE_TYPE_V" \
|
|
--threads "$NUM_THREADS" \
|
|
--threads-batch "$THREAD_BATCH" \
|
|
--ctx-checkpoints "$CTX_CHECKPOINTS" \
|
|
--cache-ram "$CACHE_RAM" \
|
|
--slot-save-path "$SLOT_SAVE_PATH" \
|
|
--cache-prompt \
|
|
--cache-idle-slots \
|
|
--metrics \
|
|
--temp "$TEMPERATURE" \
|
|
--top-p "$TOP_P" \
|
|
--top-k "$TOP_K" \
|
|
--min-p "$MIN_P" \
|
|
--presence-penalty "$PRESENCE_PENALTY" \
|
|
--reasoning "$REASONING" \
|
|
--reasoning-format "$REASONING_FORMAT" \
|
|
--reasoning-budget -1 \
|
|
"${MMPROJ_ARGS[@]}" \
|
|
"${SPEC_ARGS[@]}" |