Frank
9865f9e307
- Main model (v2 joined PLE layout, sha256 1e4160ca…) loads and serves on build-qwen38 only with --flash-attn off: the QSA indexer path calls ggml_fill on the F16-cast kq mask and aborts graph build (GGML_ASSERT F32 at ggml.c:5371). Verified live: model loaded, health 200, real completion. - MTP draft head (agentionai FAST, 2.28 GiB) does not load on this binary: qwen4exp loader requires full per-layer blk.<il>.hc_attn_norm.weight, head carries only blk.48. Third confirmation (EasiiX Q8_0 + agentionai FAST vs kingjones STRIX_LEAN previously, now own-matched pair). Spec decoding runs ngram-mod only; head retained for a future compatible build. - FLASH_ATTN=off and SPEC_TYPE=ngram-mod wired through env + serve script; runbook and AGENTS.md updated with the verified constraints.
175 lines
6.4 KiB
Bash
Executable File
175 lines
6.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve agentionai/Qwen3.8-Flash-Next-ROCmFP4-FAST-imatrix-GGUF as
|
|
# frank-default-large via the charlie12345/ROCmFPX llama.cpp runtime
|
|
# (build-qwen38, PR #27742 qwen4exp merged) on Dankclaw (Strix Halo
|
|
# gfx1151, unified memory).
|
|
#
|
|
# Runtime: charlie12345/ROCmFPX llama.cpp built at
|
|
# /home/dank/GitPackages/ROCmFPX/build-qwen38 (HIP, gfx1151, ROCm 7.2.4).
|
|
# The ROCmFP4 quant types (Q4_0_ROCMFP4_FAST) are hand-ported from
|
|
# ciru-ai/ROCmFPX; upstream llama.cpp cannot load this GGUF.
|
|
#
|
|
# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used,
|
|
# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE
|
|
# at layer 1) quantized Q4_0_ROCMFP4_FAST, imatrix-calibrated, 87.06 GiB.
|
|
# The n-gram PLE table is a single per_layer_token_embd.weight tensor —
|
|
# fully VRAM-resident, no sidecar, no --ngram-on-disk.
|
|
#
|
|
# Profile: card sampling table (temp 0.7, top_p 0.80, top_k 20, min_p 0.0,
|
|
# presence 1.5 — non-thinking instruct) mixed with the validated
|
|
# build-qwen38 serving profile: ctx 262144 x 1 slot, b 4096 / ub 4096,
|
|
# t 16 / tb 16, f16 KV, fa OFF (see KNOWN CONSTRAINTS in the env file:
|
|
# flash-attn on aborts graph build on the QSA indexer path), cont-batching,
|
|
# prompt cache + 8 GiB cache-ram + idle-slot cache, 16 ctx checkpoints,
|
|
# --fit off (auto-fit segfaults on qwen4exp hyper-connection config).
|
|
# Speculative decoding is ngram-mod only; the MTP head does not load on
|
|
# build-qwen38 (loader requires blk.0.hc_attn_norm.weight it lacks).
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -------------------
|
|
: "${LLAMA_BIN:=/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server}"
|
|
: "${MODEL_DIR:=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/v2}"
|
|
: "${MODEL_FILE:=Qwen3.8-Flash-Next-ROCmFP4-FAST-imatrix-v2.gguf}"
|
|
: "${MMPROJ_PATH:=$MODEL_DIR/../mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf}"
|
|
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
|
|
: "${CHAT_TEMPLATE_KWARGS:={}}"
|
|
: "${ALIAS:=frank-default-large}"
|
|
: "${BIND_ADDR:=0.0.0.0}"
|
|
: "${LISTEN_PORT:=7770}"
|
|
: "${CTX_SIZE:=262144}"
|
|
: "${PARALLEL:=1}"
|
|
: "${BATCH_SIZE:=4096}"
|
|
: "${UBATCH_SIZE:=4096}"
|
|
: "${NUM_THREADS:=16}"
|
|
: "${THREAD_BATCH:=16}"
|
|
: "${CACHE_TYPE_K:=f16}"
|
|
: "${CACHE_TYPE_V:=f16}"
|
|
: "${CTX_CHECKPOINTS:=16}"
|
|
: "${CACHE_RAM:=8192}"
|
|
: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}"
|
|
: "${SPLIT_MODE:=none}"
|
|
: "${FLASH_ATTN:=off}"
|
|
: "${TEMPERATURE:=0.7}"
|
|
: "${TOP_K:=20}"
|
|
: "${TOP_P:=0.8}"
|
|
: "${MIN_P:=0.0}"
|
|
: "${PRESENCE_PENALTY:=1.5}"
|
|
: "${REASONING:=on}"
|
|
: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
|
|
: "${SPEC_TYPE:=ngram-mod}"
|
|
: "${DRAFT_MODEL_DIR:=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST}"
|
|
: "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}"
|
|
: "${SPEC_DRAFT_N_MAX:=3}"
|
|
: "${SPEC_DRAFT_N_MIN:=0}"
|
|
: "${SPEC_DRAFT_P_MIN:=0.75}"
|
|
: "${SPEC_DRAFT_NGL:=999}"
|
|
: "${SPEC_DRAFT_DEVICE:=ROCm0}"
|
|
: "${SPEC_DRAFT_TYPE_K:=f16}"
|
|
: "${SPEC_DRAFT_TYPE_V:=f16}"
|
|
|
|
MODEL="$MODEL_DIR/$MODEL_FILE"
|
|
for f in "$LLAMA_BIN" "$MODEL" "$CHAT_TEMPLATE_FILE"; do
|
|
if [[ ! -r "$f" ]]; then
|
|
echo "missing or unreadable required artifact: $f" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
if [[ ! -x "$LLAMA_BIN" ]]; then
|
|
echo "missing executable runtime: $LLAMA_BIN" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# --- speculative decoding (MTP draft head) ---------------------------------
|
|
# The MTP draft head is a separate GGUF sidecar loaded via --spec-draft-model
|
|
# (alias --model-draft). ngram-only specs do not need a draft file.
|
|
# Set SPEC_TYPE=none in the env to disable speculative decoding entirely.
|
|
SPEC_ARGS=()
|
|
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
|
|
SPEC_ARGS=(
|
|
--spec-type "$SPEC_TYPE"
|
|
--spec-draft-n-max "$SPEC_DRAFT_N_MAX"
|
|
--spec-draft-n-min "$SPEC_DRAFT_N_MIN"
|
|
--spec-draft-p-min "$SPEC_DRAFT_P_MIN"
|
|
)
|
|
if [[ "$SPEC_TYPE" == *draft-mtp* ]]; then
|
|
DRAFT_MODEL="$DRAFT_MODEL_DIR/$DRAFT_MODEL_FILE"
|
|
if [[ ! -r "$DRAFT_MODEL" ]]; then
|
|
echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
|
|
exit 1
|
|
fi
|
|
SPEC_ARGS+=(
|
|
--spec-draft-model "$DRAFT_MODEL"
|
|
--spec-draft-ngl "$SPEC_DRAFT_NGL"
|
|
--spec-draft-device "$SPEC_DRAFT_DEVICE"
|
|
--spec-draft-type-k "$SPEC_DRAFT_TYPE_K"
|
|
--spec-draft-type-v "$SPEC_DRAFT_TYPE_V"
|
|
)
|
|
fi
|
|
if [[ -n "${SPEC_DRAFT_P_SPLIT:-}" ]]; then
|
|
SPEC_ARGS+=(--spec-draft-p-split "$SPEC_DRAFT_P_SPLIT")
|
|
fi
|
|
fi
|
|
|
|
# The agentionai FAST package includes the model's own vision tower (f16,
|
|
# lossless). Pass --mmproj only when MMPROJ_PATH points at a readable file.
|
|
MMPROJ_ARGS=()
|
|
if [[ -n "${MMPROJ_PATH:-}" && -r "${MMPROJ_PATH:-}" ]]; then
|
|
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
|
fi
|
|
|
|
SERVER_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
mkdir -p "$SLOT_SAVE_PATH"
|
|
ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
|
|
LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
|
|
|
exec /usr/bin/env \
|
|
ROCM_PATH="$ROCM_PATH" \
|
|
HIP_VISIBLE_DEVICES=0 \
|
|
LD_LIBRARY_PATH="$LIBRARY_PATH" \
|
|
HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \
|
|
"$LLAMA_BIN" \
|
|
--model "$MODEL" \
|
|
--alias "$ALIAS" \
|
|
--chat-template-file "$CHAT_TEMPLATE_FILE" \
|
|
--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS" \
|
|
--host "$BIND_ADDR" \
|
|
--port "$LISTEN_PORT" \
|
|
--jinja \
|
|
--device ROCm0 \
|
|
--gpu-layers "$NGL" \
|
|
--split-mode "$SPLIT_MODE" \
|
|
--parallel "$PARALLEL" \
|
|
--ctx-size "$CTX_SIZE" \
|
|
--batch-size "$BATCH_SIZE" \
|
|
--ubatch-size "$UBATCH_SIZE" \
|
|
--fit "$FIT" \
|
|
--flash-attn "$FLASH_ATTN" \
|
|
--cache-type-k "$CACHE_TYPE_K" \
|
|
--cache-type-v "$CACHE_TYPE_V" \
|
|
--threads "$NUM_THREADS" \
|
|
--threads-batch "$THREAD_BATCH" \
|
|
--ctx-checkpoints "$CTX_CHECKPOINTS" \
|
|
--cache-ram "$CACHE_RAM" \
|
|
--slot-save-path "$SLOT_SAVE_PATH" \
|
|
--cache-prompt \
|
|
--cache-idle-slots \
|
|
--metrics \
|
|
--temp "$TEMPERATURE" \
|
|
--top-p "$TOP_P" \
|
|
--top-k "$TOP_K" \
|
|
--min-p "$MIN_P" \
|
|
--presence-penalty "$PRESENCE_PENALTY" \
|
|
--reasoning "$REASONING" \
|
|
--reasoning-format "$REASONING_FORMAT" \
|
|
--reasoning-budget -1 \
|
|
"${MMPROJ_ARGS[@]}" \
|
|
"${SPEC_ARGS[@]}" |