serverAppConfigs/scripts/serve-frank-large.sh

#!/usr/bin/env bash
# Serve kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF as frank-large on
# Dankclaw's Strix Halo iGPU (gfx1151) with the ROCmFPX fork of llama.cpp.
#
# 2026-10-02 (Frank): swapped from gufo + Q4_K_XL to the highest-quality file
# in the repo, one session at the native 262,144 window.
#
# Why this runtime and not the previous one: the payload is ftype 115 — 311
# tensors of GGML_TYPE_Q8_0_ROCMFPX (103), a tensor type that exists only in
# the ROCmFPX fork. Stock llama.cpp, SGLang and gufo all refuse the file, and
# gufo additionally gates every "Qwen3.8" artifact on a template-hash
# allowlist. The qualified build is scripts/build-rocmfpx.sh's pin: it loads
# ftype 115, survives warmup (the 2026-08-25 builds of the older fork line
# segfaulted in ggml_compute_forward_set_rows on this exact file), and drafts
# with the model's own MTP head.
#
# Why this file: Q8_0_ROCMFPX_AGENT is the top of the repo's quality ladder
# (26.28 GiB, 8.39 bpw) and the AGENT routing keeps 187 tensors — the
# output-head set the draft scores against — at plain Q8_0 instead of the 2
# the non-AGENT build keeps, which is the variant the card names for MTP
# serving. Drafting uses the repo's Q4_0 MTP head: the head's cost is its
# projection over the 248K vocabulary, so the 4-bit head is faster end to end
# at equal-or-better acceptance than its Q8_0 sibling.
#
# The client contract lives here, not in frank-large.env:
#   * --jinja — without it llama.cpp silently IGNORES the harness's
#     chat_template_kwargs (enable_thinking / preserve_thinking), so the
#     endpoint would look like it had no thinking control at all. The GGUF
#     carries the official Qwen template, so no --chat-template-file.
#   * --reasoning-format deepseek — thinking is emitted as reasoning_content,
#     the field the harness reads for this endpoint.
#   * the template accepts reasoning_effort xhigh|medium|low only, which is
#     why models.yml folds the harness's levels onto that set.
#   * --alias frank-large — that is the model id the harness resolves.
#   * --ctx-checkpoints / --cache-ram — prompt-cache geometry for the 16
#     sliding-window-style layers; harmless where the hybrid state forces a
#     full re-process, and required for the checkpoint feature to work at all.
set -euo pipefail

# Fleet values live in frank-large.env. Anything already in the environment
# wins over the file, so an operator can override one value on a single run:
# the caller's values are captured, the file is sourced, then they are put
# back. (A plain `source` would let the file clobber the environment.)
TUNABLES=(LLAMA_BIN MODEL_FILE DRAFT_FILE MMPROJ_FILE ALIAS BIND_ADDR
          LISTEN_PORT CTX_SIZE PARALLEL BATCH_SIZE UBATCH_SIZE NGL FIT
          FLASH_ATTN CACHE_TYPE_K CACHE_TYPE_V SPEC_TYPE SPEC_DRAFT_NGL
          SPEC_DRAFT_DEVICE SPEC_DRAFT_N_MAX SPEC_DRAFT_N_MIN SPEC_DRAFT_P_MIN
          TEMPERATURE TOP_P TOP_K MIN_P PRESENCE_PENALTY NUM_THREADS
          THREAD_BATCH ROCM_PATH HIP_VISIBLE HSA_GFX_VERSION)
_OVERRIDES=()
for _var in "${TUNABLES[@]}"; do
    [[ -n "${!_var+set}" ]] && _OVERRIDES+=("$_var=${!_var}")
done
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$(dirname "$0")/frank-large.env}"
if [[ -f "$ENV_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    source "$ENV_FILE"
    set +a
fi
for _kv in "${_OVERRIDES[@]+${_OVERRIDES[@]}}"; do
    export "${_kv?}"
done
unset _var _kv

: "${LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}"
: "${MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf}"
: "${DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}"
: "${MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}"
: "${ALIAS=frank-large}"
: "${BIND_ADDR=0.0.0.0}"
: "${LISTEN_PORT=7770}"
: "${CTX_SIZE=262144}"
: "${PARALLEL=1}"
: "${BATCH_SIZE=2048}"
: "${UBATCH_SIZE=512}"
: "${NGL=999}"
: "${FIT=off}"
: "${FLASH_ATTN=on}"
: "${CACHE_TYPE_K=f16}"
: "${CACHE_TYPE_V=f16}"
: "${SPEC_TYPE=draft-mtp}"
: "${SPEC_DRAFT_NGL=99}"
: "${SPEC_DRAFT_DEVICE=ROCm0}"
: "${SPEC_DRAFT_N_MAX=4}"
: "${SPEC_DRAFT_N_MIN=0}"
: "${SPEC_DRAFT_P_MIN=0.0}"
: "${TEMPERATURE=1.0}"
: "${TOP_P=0.95}"
: "${TOP_K=20}"
: "${MIN_P=0.0}"
: "${PRESENCE_PENALTY=0.0}"
: "${NUM_THREADS=16}"
: "${THREAD_BATCH=16}"
: "${ROCM_PATH=/opt/rocm}"
: "${HIP_VISIBLE=0}"
: "${HSA_GFX_VERSION=11.5.1}"

[[ -n "$LLAMA_BIN" ]] || { echo "frank-large: LLAMA_BIN is empty" >&2; exit 1; }
[[ -x "$LLAMA_BIN" ]] || {
    echo "frank-large: llama-server '$LLAMA_BIN' is not executable;" >&2
    echo "           build it with scripts/build-rocmfpx.sh" >&2
    exit 1
}
# Refuse anything but the real payload. A path that is not a readable file
# makes llama-server hang rather than fail clean.
[[ -r "$MODEL_FILE" ]] || {
    echo "frank-large: model file '$MODEL_FILE' is not readable;" >&2
    echo "           fetch it with scripts/download-frank-large-kingjones.sh" >&2
    exit 1
}
# The MTP head is the performance feature: without it the Q8_0 weights decode
# at memory-bandwidth speed and the endpoint loses the ~1.4x the head buys.
[[ -r "$DRAFT_FILE" ]] || {
    echo "frank-large: MTP draft head '$DRAFT_FILE' is not readable;" >&2
    echo "           fetch it with scripts/download-frank-large-kingjones.sh" >&2
    exit 1
}
# The GGUF carries no vision tower; the projector file is the whole tower, so
# a missing file means a vision request would 500 rather than degrade.
# MMPROJ_FILE="" is the documented opt-out for a text-only endpoint.
MM_ARGS=()
if [[ -n "${MMPROJ_FILE:-}" ]]; then
    [[ -r "$MMPROJ_FILE" ]] || {
        echo "frank-large: mmproj '$MMPROJ_FILE' is not readable; set MMPROJ_FILE='' for text-only" >&2
        exit 1
    }
    MM_ARGS=(--mmproj "$MMPROJ_FILE")
fi

# HIP/ROCm host environment. HIP reads bare env vars (TEMP, HOST,
# GPU_MAX_HEAP_SIZE), which is why the tunables above are named TEMPERATURE
# and BIND_ADDR. HSA_OVERRIDE_GFX_VERSION is what lets a runtime built for a
# neighbouring gfx target run on this iGPU; the qualified pin targets gfx1151
# natively and the override is a no-op for it.
export ROCM_PATH
export HIP_VISIBLE_DEVICES="$HIP_VISIBLE"
export HSA_OVERRIDE_GFX_VERSION="$HSA_GFX_VERSION"
export LD_LIBRARY_PATH="$(dirname "$LLAMA_BIN"):${ROCM_PATH}/lib:${ROCM_PATH}/lib/rocm_sysdeps/lib:${ROCM_PATH}/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"

exec /usr/bin/env "$LLAMA_BIN" \
    --model "$MODEL_FILE" \
    "${MM_ARGS[@]}" \
    --alias "$ALIAS" \
    --host "$BIND_ADDR" \
    --port "$LISTEN_PORT" \
    --jinja \
    --device ROCm0 \
    --n-gpu-layers "$NGL" \
    --parallel "$PARALLEL" \
    --ctx-size "$CTX_SIZE" \
    --batch-size "$BATCH_SIZE" \
    --ubatch-size "$UBATCH_SIZE" \
    --threads "$NUM_THREADS" \
    --threads-batch "$THREAD_BATCH" \
    --fit "$FIT" \
    --flash-attn "$FLASH_ATTN" \
    --direct-io \
    --cache-type-k "$CACHE_TYPE_K" \
    --cache-type-v "$CACHE_TYPE_V" \
    --ctx-checkpoints 16 \
    --cache-ram 8192 \
    --cache-idle-slots \
    --temp "$TEMPERATURE" \
    --top-p "$TOP_P" \
    --top-k "$TOP_K" \
    --min-p "$MIN_P" \
    --presence-penalty "$PRESENCE_PENALTY" \
    --reasoning on \
    --reasoning-format deepseek \
    --reasoning-budget -1 \
    --spec-type "$SPEC_TYPE" \
    --model-draft "$DRAFT_FILE" \
    --spec-draft-ngl "$SPEC_DRAFT_NGL" \
    --spec-draft-device "$SPEC_DRAFT_DEVICE" \
    --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
    --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \
    --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \
    --metrics