serverAppConfigs/scripts/serve-frank-medium.sh

#!/usr/bin/env bash
# Serve kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF as frank-medium via
# the charlie12345/ROCmFPX llama.cpp runtime on Dankserver
# (2x AMD Radeon AI PRO R9700, Navi 48, gfx1201, 32 GB each).
#
# Runtime: charlie12345/ROCmFPX main (c49ebdb, 2026-08-22), built for
# gfx1201 with the fork's self-contained ROCm toolchain:
#
#   cd /home/dank/GitPackages/charlie-ROCmFPX
#   env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh
#   -> build-rdna4-rocm714/bin/llama-server
#
# That script downloads the gfx120X-all TheRock ROCm 7.14 toolchain into
# ~/.local/share/rocm-7.14.0a20260624, builds with
# -DCMAKE_HIP_ARCHITECTURES="gfx1200;gfx1201", copies the ROCm runtime
# (hip/hsa/rocblas/hipblaslt + their kernel libraries) next to the binary and
# sets RPATH=$ORIGIN/../lib on it.  The resulting service is self-contained:
# no LD_LIBRARY_PATH, and the host's stale /opt/rocm-6.3.3 cannot leak in.
# gfx1201 is NOT interchangeable with gfx1200 on this card (the fork's
# docs/BUILD-AMD-ARCHITECTURES.md: a gfx1200 build links, loads and then
# segfaults), and the build verifies the emitted code objects afterwards.
#
# Model: kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT (Q8_0_ROCMFPX_AGENT,
# ftype 115, arch qwen35 MoE), 26.28 GiB main file, plus a separate
# MTP draft head and an mmproj vision tower.  8.39 bpw; the AGENT routing
# keeps 13 of the 17 attention-output projections at full Q8_0, which is
# what makes the MTP draft land ~0.95 of its guesses.  A stock llama.cpp
# cannot load this GGUF at all - Q8_0_ROCMFPX exists only in this fork.
#
# Parallelism: --split-mode layer across both cards (operator call
# 2026-09-15).  Layers are distributed in order and activations hop PCIe at
# the boundary; measured llama-bench: layer 1991/18.8 tok/s (pp4096/tg128)
# vs tensor 272/26.3 and single-card 1161/19.4.  Tensor wins decode +40 %
# but its prefill collapses 7x on this build (no RCCL: "AllReduce init
# failed; falling back to meta-backend butterfly"), so layer split wins the
# prefill-heavy agentic workload.
#
# Tunables come from frank-medium.env unless FRANK_MEDIUM_ENV_FILE
# overrides that path.
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_MEDIUM_ENV_FILE:-$SCRIPT_DIR/frank-medium.env}"
if [[ -f "$ENV_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    source "$ENV_FILE"
    set +a
fi

# --- defaults (used when the env file does not set them) -----------------
: "${LLAMA_BIN:=/home/dank/GitPackages/charlie-ROCmFPX/build-rdna4-rocm714/bin/llama-server}"
: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT}"
: "${MODEL_FILE:=Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf}"
: "${DRAFT_MODEL_FILE:=mtp-Qwen3.8-27B-Q4_0.gguf}"
: "${ALIAS:=frank-medium}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7770}"

# Devices: ROCm0/ROCm1 are the two R9700s; the Raphael iGPU enumerates as
# ROCm2 (gfx1036) and is hidden from HIP so nothing can drift onto it.
: "${HIP_VISIBLE_DEVICES:=0,1}"
: "${DEVICES:=ROCm0,ROCm1}"
: "${SPLIT_MODE:=tensor}"
: "${TENSOR_SPLIT:=}"
: "${NGL:=999}"
: "${FIT:=off}"
: "${FLASH_ATTN:=on}"

: "${CTX_SIZE:=131072}"
: "${PARALLEL:=1}"
: "${BATCH_SIZE:=2048}"
: "${UBATCH_SIZE:=512}"
: "${NUM_THREADS:=8}"
: "${THREAD_BATCH:=8}"
: "${CACHE_TYPE_K:=f16}"
: "${CACHE_TYPE_V:=f16}"
: "${CTX_CHECKPOINTS:=8}"
: "${CACHE_RAM:=8192}"

# Sampling: the checkpoint's instruct table.
: "${TEMPERATURE:=0.7}"
: "${TOP_P:=0.8}"
: "${TOP_K:=20}"
: "${MIN_P:=0.0}"
: "${PRESENCE_PENALTY:=1.5}"

# Reasoning: the model is an agent checkpoint; keep the trace and project it
# into the API's reasoning_content field rather than literal thinking tags.
: "${REASONING:=on}"
: "${REASONING_FORMAT:=deepseek}"
: "${REASONING_BUDGET:=-1}"

# Chat template: empty means "use the template embedded in the GGUF".
: "${CHAT_TEMPLATE_FILE:=}"
: "${CHAT_TEMPLATE_KWARGS:={\"enable_thinking\":true,\"preserve_thinking\":true}}"

# MTP draft head (separate GGUF, --spec-type draft-mtp).  SPEC_DRAFT_DEVICE
# must name exactly one device: under tensor mode an unsplit draft joins the
# meta-device KV allocation and aborts while building the KV cache
# (GGML_ASSERT(buffer) in ggml-backend.cpp, 2026-09-15).
: "${SPEC_TYPE:=draft-mtp}"
: "${SPEC_DRAFT_N_MAX:=6}"
: "${SPEC_DRAFT_N_MIN:=0}"
: "${SPEC_DRAFT_P_MIN:=0.6}"
: "${SPEC_DRAFT_P_SPLIT:=0.10}"
: "${SPEC_DRAFT_NGL:=999}"
: "${SPEC_DRAFT_DEVICE:=}"
: "${SPEC_DRAFT_TYPE_K:=f16}"
: "${SPEC_DRAFT_TYPE_V:=f16}"
: "${SPEC_DRAFT_POLL:=1}"
: "${SPEC_DRAFT_POLL_BATCH:=1}"

# Text only: the server must not try to autoload the vision tower.  Vision
# stays on frank-large (Dankclaw), which is where the fleet routes images.
NO_MMPROJ="${NO_MMPROJ:-1}"

export HIP_VISIBLE_DEVICES

MODEL="$MODEL_DIR/$MODEL_FILE"
if [[ ! -x "$LLAMA_BIN" ]]; then
    echo "missing executable runtime: $LLAMA_BIN" >&2
    echo "build it with: cd $(dirname "$(dirname "$LLAMA_BIN")")/.. && env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh" >&2
    exit 1
fi
if [[ ! -r "$MODEL" ]]; then
    echo "missing or unreadable required artifact: $MODEL" >&2
    echo "fetch it with: bash $SCRIPT_DIR/download-q8agent.sh" >&2
    exit 1
fi
if [[ -n "$CHAT_TEMPLATE_FILE" && ! -r "$CHAT_TEMPLATE_FILE" ]]; then
    echo "missing or unreadable chat template: $CHAT_TEMPLATE_FILE" >&2
    exit 1
fi

# --- speculative decoding (MTP draft head) --------------------------------
SPEC_ARGS=()
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
    SPEC_ARGS=(
        --spec-type "$SPEC_TYPE"
        --spec-draft-n-max "$SPEC_DRAFT_N_MAX"
        --spec-draft-n-min "$SPEC_DRAFT_N_MIN"
        --spec-draft-p-min "$SPEC_DRAFT_P_MIN"
        --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT"
    )
    if [[ "$SPEC_TYPE" == *draft-mtp* ]]; then
        DRAFT_MODEL="$MODEL_DIR/$DRAFT_MODEL_FILE"
        if [[ ! -r "$DRAFT_MODEL" ]]; then
            echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
            exit 1
        fi
        SPEC_ARGS+=(
            --spec-draft-model "$DRAFT_MODEL"
            --spec-draft-ngl "$SPEC_DRAFT_NGL"
            --spec-draft-type-k "$SPEC_DRAFT_TYPE_K"
            --spec-draft-type-v "$SPEC_DRAFT_TYPE_V"
            --spec-draft-poll "$SPEC_DRAFT_POLL"
            --spec-draft-poll-batch "$SPEC_DRAFT_POLL_BATCH"
        )
        if [[ -n "$SPEC_DRAFT_DEVICE" ]]; then
            SPEC_ARGS+=(--spec-draft-device "$SPEC_DRAFT_DEVICE")
        fi
    fi
fi

# --- template + placement -------------------------------------------------
TEMPLATE_ARGS=()
if [[ -n "$CHAT_TEMPLATE_FILE" ]]; then
    TEMPLATE_ARGS+=(--chat-template-file "$CHAT_TEMPLATE_FILE")
    TEMPLATE_ARGS+=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS")
fi

PLACEMENT_ARGS=(--device "$DEVICES" --split-mode "$SPLIT_MODE")
if [[ -n "$TENSOR_SPLIT" ]]; then
    PLACEMENT_ARGS+=(--tensor-split "$TENSOR_SPLIT")
fi

MMPROJ_ARGS=()
if [[ "$NO_MMPROJ" == "1" ]]; then
    MMPROJ_ARGS=(--no-mmproj)
fi

exec "$LLAMA_BIN" \
    --model "$MODEL" \
    --alias "$ALIAS" \
    --host "$BIND_ADDR" \
    --port "$LISTEN_PORT" \
    --jinja \
    --gpu-layers "$NGL" \
    --fit "$FIT" \
    --parallel "$PARALLEL" \
    --ctx-size "$CTX_SIZE" \
    --batch-size "$BATCH_SIZE" \
    --ubatch-size "$UBATCH_SIZE" \
    --flash-attn "$FLASH_ATTN" \
    --cache-type-k "$CACHE_TYPE_K" \
    --cache-type-v "$CACHE_TYPE_V" \
    --threads "$NUM_THREADS" \
    --threads-batch "$THREAD_BATCH" \
    --ctx-checkpoints "$CTX_CHECKPOINTS" \
    --cache-ram "$CACHE_RAM" \
    --cache-prompt \
    --metrics \
    --temp "$TEMPERATURE" \
    --top-p "$TOP_P" \
    --top-k "$TOP_K" \
    --min-p "$MIN_P" \
    --presence-penalty "$PRESENCE_PENALTY" \
    --reasoning "$REASONING" \
    --reasoning-format "$REASONING_FORMAT" \
    --reasoning-budget "$REASONING_BUDGET" \
    "${TEMPLATE_ARGS[@]}" \
    "${PLACEMENT_ARGS[@]}" \
    "${MMPROJ_ARGS[@]}" \
    "${SPEC_ARGS[@]}"