serverAppConfigs/scripts/serve-frank-large.sh

#!/usr/bin/env bash
# Serve kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF as
# frank-large on Dankclaw's Strix Halo iGPU (gfx1151) with the ROCmFPX fork of
# llama.cpp.
#
# 2026-10-03 (Frank): swapped from the ALIGNED Q8_0_ROCMFPX_AGENT to this
# repo's Q6_0_ROCMFPX_AGENT — abliterated weights, one quant rung cheaper,
# everything else (window, KV, MTP depth, sampler) unchanged.
#
# Why this runtime and not another: the payload is ftype 114 — 339 tensors of
# GGML_TYPE_Q6_0_ROCMFPX (102) and 159 of GGML_TYPE_Q8_0_ROCMFPX (103), tensor
# types that exist only in the ROCmFPX fork. Stock llama.cpp, SGLang and gufo
# all refuse the file, and gufo additionally gates every "Qwen3.8" artifact on
# a template-hash allowlist. The qualified build is scripts/build-rocmfpx.sh's
# pin: it defines both types (ggml.h 102/103), it carries the device paths for
# type 102 (convert/cpy/getrows/mmq/fattn in ggml-cuda), it survives warmup —
# the 2026-08-25 builds of the older fork line segfaulted in
# ggml_compute_forward_set_rows on this family — and it drafts with the
# model's own MTP head.
#
# Why this file: Q6_0_ROCMFPX_AGENT (23.15 GiB, ftype 114) is the card's
# protected-head rung — "Take Q6 AGENT (114) if you want protected heads": the
# AGENT routing keeps the embeddings, the output head and every fourth
# attention projection at the Q8 rung (159 tensors), which is the variant the
# card names for MTP serving, and the refusal study was scored on this exact
# file (23/24 comply where the aligned Q8 AGENT refuses 23/24, quality score
# identical at 6/8). Drafting uses the repo's Q4_0 MTP head: the head's cost is
# its projection over the 248K vocabulary, so the 4-bit head is faster end to
# end at equal-or-better acceptance than its Q8_0 sibling.
#
# The client contract lives here, not in frank-large.env:
#   * --jinja — without it llama.cpp silently IGNORES the harness's
#     chat_template_kwargs (enable_thinking / preserve_thinking), so the
#     endpoint would look like it had no thinking control at all. The GGUF
#     carries the official Qwen template, so no --chat-template-file.
#   * --reasoning-format deepseek — thinking is emitted as reasoning_content,
#     the field the harness reads for this endpoint.
#   * the template accepts reasoning_effort xhigh|medium|low only, which is
#     why models.yml folds the harness's levels onto that set.
#   * --alias frank-large — that is the model id the harness resolves.
#   * --ctx-checkpoints / --cache-ram — prompt-cache geometry for the 16
#     sliding-window-style layers; harmless where the hybrid state forces a
#     full re-process, and required for the checkpoint feature to work at all.
set -euo pipefail

# Fleet values live in frank-large.env. Anything already in the environment
# wins over the file, so an operator can override one value on a single run:
# the caller's values are captured, the file is sourced, then they are put
# back. (A plain `source` would let the file clobber the environment.)
TUNABLES=(LLAMA_BIN MODEL_FILE DRAFT_FILE MMPROJ_FILE ALIAS BIND_ADDR
          LISTEN_PORT CTX_SIZE PARALLEL BATCH_SIZE UBATCH_SIZE NGL FIT
          FLASH_ATTN CACHE_TYPE_K CACHE_TYPE_V SPEC_TYPE SPEC_DRAFT_NGL
          SPEC_DRAFT_DEVICE SPEC_DRAFT_N_MAX SPEC_DRAFT_N_MIN SPEC_DRAFT_P_MIN
          TEMPERATURE TOP_P TOP_K MIN_P PRESENCE_PENALTY NUM_THREADS
          THREAD_BATCH ROCM_PATH HIP_VISIBLE HSA_GFX_VERSION)
_OVERRIDES=()
for _var in "${TUNABLES[@]}"; do
    [[ -n "${!_var+set}" ]] && _OVERRIDES+=("$_var=${!_var}")
done
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$(dirname "$0")/frank-large.env}"
if [[ -f "$ENV_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    source "$ENV_FILE"
    set +a
fi
for _kv in "${_OVERRIDES[@]+${_OVERRIDES[@]}}"; do
    export "${_kv?}"
done
unset _var _kv

: "${LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}"
: "${MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}"
: "${DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}"
: "${MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}"
: "${ALIAS=frank-large}"
: "${BIND_ADDR=0.0.0.0}"
: "${LISTEN_PORT=7770}"
: "${CTX_SIZE=262144}"
: "${PARALLEL=1}"
: "${BATCH_SIZE=2048}"
: "${UBATCH_SIZE=512}"
: "${NGL=999}"
: "${FIT=off}"
: "${FLASH_ATTN=on}"
: "${CACHE_TYPE_K=f16}"
: "${CACHE_TYPE_V=f16}"
: "${SPEC_TYPE=draft-mtp}"
: "${SPEC_DRAFT_NGL=99}"
: "${SPEC_DRAFT_DEVICE=ROCm0}"
: "${SPEC_DRAFT_N_MAX=4}"
: "${SPEC_DRAFT_N_MIN=0}"
: "${SPEC_DRAFT_P_MIN=0.0}"
: "${TEMPERATURE=1.0}"
: "${TOP_P=0.95}"
: "${TOP_K=20}"
: "${MIN_P=0.0}"
: "${PRESENCE_PENALTY=0.0}"
: "${NUM_THREADS=16}"
: "${THREAD_BATCH=16}"
: "${ROCM_PATH=/opt/rocm}"
: "${HIP_VISIBLE=0}"
: "${HSA_GFX_VERSION=11.5.1}"

[[ -n "$LLAMA_BIN" ]] || { echo "frank-large: LLAMA_BIN is empty" >&2; exit 1; }
[[ -x "$LLAMA_BIN" ]] || {
    echo "frank-large: llama-server '$LLAMA_BIN' is not executable;" >&2
    echo "           build it with scripts/build-rocmfpx.sh" >&2
    exit 1
}
# Refuse anything but the real payload. A path that is not a readable file
# makes llama-server hang rather than fail clean.
[[ -r "$MODEL_FILE" ]] || {
    echo "frank-large: model file '$MODEL_FILE' is not readable;" >&2
    echo "           fetch it with scripts/download-frank-large-kingjones-uncensored.sh" >&2
    exit 1
}
# The MTP head is the performance feature: without it these weights decode at
# memory-bandwidth speed (measured on the aligned sibling: 17.0 t/s with the
# head against 7.6 without) and the endpoint loses the ~2.2x the head buys.
[[ -r "$DRAFT_FILE" ]] || {
    echo "frank-large: MTP draft head '$DRAFT_FILE' is not readable;" >&2
    echo "           fetch it with scripts/download-frank-large-kingjones-uncensored.sh" >&2
    exit 1
}
# The GGUF carries no vision tower; the projector file is the whole tower, so
# a missing file means a vision request would 500 rather than degrade.
# MMPROJ_FILE="" is the documented opt-out for a text-only endpoint.
MM_ARGS=()
if [[ -n "${MMPROJ_FILE:-}" ]]; then
    [[ -r "$MMPROJ_FILE" ]] || {
        echo "frank-large: mmproj '$MMPROJ_FILE' is not readable; set MMPROJ_FILE='' for text-only" >&2
        exit 1
    }
    MM_ARGS=(--mmproj "$MMPROJ_FILE")
fi

# HIP/ROCm host environment. HIP reads bare env vars (TEMP, HOST,
# GPU_MAX_HEAP_SIZE), which is why the tunables above are named TEMPERATURE
# and BIND_ADDR. HSA_OVERRIDE_GFX_VERSION is what lets a runtime built for a
# neighbouring gfx target run on this iGPU; the qualified pin targets gfx1151
# natively and the override is a no-op for it.
export ROCM_PATH
export HIP_VISIBLE_DEVICES="$HIP_VISIBLE"
export HSA_OVERRIDE_GFX_VERSION="$HSA_GFX_VERSION"
export LD_LIBRARY_PATH="$(dirname "$LLAMA_BIN"):${ROCM_PATH}/lib:${ROCM_PATH}/lib/rocm_sysdeps/lib:${ROCM_PATH}/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"

exec /usr/bin/env "$LLAMA_BIN" \
    --model "$MODEL_FILE" \
    "${MM_ARGS[@]}" \
    --alias "$ALIAS" \
    --host "$BIND_ADDR" \
    --port "$LISTEN_PORT" \
    --jinja \
    --device ROCm0 \
    --n-gpu-layers "$NGL" \
    --parallel "$PARALLEL" \
    --ctx-size "$CTX_SIZE" \
    --batch-size "$BATCH_SIZE" \
    --ubatch-size "$UBATCH_SIZE" \
    --threads "$NUM_THREADS" \
    --threads-batch "$THREAD_BATCH" \
    --fit "$FIT" \
    --flash-attn "$FLASH_ATTN" \
    --direct-io \
    --cache-type-k "$CACHE_TYPE_K" \
    --cache-type-v "$CACHE_TYPE_V" \
    --ctx-checkpoints 16 \
    --cache-ram 8192 \
    --cache-idle-slots \
    --temp "$TEMPERATURE" \
    --top-p "$TOP_P" \
    --top-k "$TOP_K" \
    --min-p "$MIN_P" \
    --presence-penalty "$PRESENCE_PENALTY" \
    --reasoning on \
    --reasoning-format deepseek \
    --reasoning-budget -1 \
    --spec-type "$SPEC_TYPE" \
    --model-draft "$DRAFT_FILE" \
    --spec-draft-ngl "$SPEC_DRAFT_NGL" \
    --spec-draft-device "$SPEC_DRAFT_DEVICE" \
    --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
    --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \
    --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \
    --metrics