serverAppConfigs/scripts/.AGENTTRASH/20260920-llama-layersplit/serve-frank-medium.sh.llamacpp-layersplit

#!/usr/bin/env bash
# Serve ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF @ UD-Q8_K_XL as
# frank-medium on Dankserver's two Radeon AI PRO R9700 (gfx1201), with the
# z-lab DFlash2 block-diffusion drafter, using mainline llama.cpp.
#
# Runtime: llama.cpp master fb27a525 (2026-09-16), a gfx1201 HIP build that
# bundles its own ROCm 7.14 runtime (RPATH $ORIGIN/../lib), produced by
# scripts/build-llama-rdna4.sh.  Two features pin that ref: DFlash2
# (ggml-org/llama.cpp#27342) and qwen35 MTP.  The charlie12345/ROCmFPX fork
# this deployment replaced has neither (its tip is c49ebdb, 2026-08-22).
#
# Parallelism: --split-mode layer across both cards.  Tensor split would need a
# cross-card allreduce and this host cannot dispatch one -- RCCL's kernels
# require a hostcall buffer that ROCr refuses on a card without PCIe atomics,
# and the second R9700 sits behind the chipset (AtomicOpsCap: Routing-).  Layer
# split crosses the wire once per forward pass instead of once per layer.
#
# The vLLM replica pair and its routing proxy were retired with this cutover:
# one process, one model, both cards.
#
# Tunables come from frank-medium.env unless FRANK_MEDIUM_ENV_FILE overrides it.
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_MEDIUM_ENV_FILE:-$SCRIPT_DIR/frank-medium.env}"
if [[ -f "$ENV_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    source "$ENV_FILE"
    set +a
fi

# --- defaults (used when the env file does not set them) ------------------
: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}"
: "${MODEL_DIR:=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF}"
: "${MODEL_FILE:=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}"
: "${DRAFT_MODEL_DIR:=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF}"
: "${DRAFT_MODEL_FILE:=Qwen3.8-27B-DFlash2-Q8_0.gguf}"
: "${ALIAS:=frank-medium}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7770}"

# Devices.  HIP_VISIBLE_DEVICES hides the Raphael iGPU (HIP device 2) so nothing
# can drift onto shared system RAM; DEVICES is what llama.cpp splits across.
: "${HIP_VISIBLE_DEVICES:=0,1}"
: "${DEVICES:=ROCm0,ROCm1}"
: "${SPLIT_MODE:=layer}"
: "${TENSOR_SPLIT:=1,1}"
: "${NGL:=999}"
: "${FIT:=off}"
: "${FLASH_ATTN:=on}"

: "${CTX_SIZE:=131072}"
: "${PARALLEL:=1}"
: "${BATCH_SIZE:=2048}"
: "${UBATCH_SIZE:=512}"
: "${NUM_THREADS:=8}"
: "${THREAD_BATCH:=8}"
: "${CACHE_TYPE_K:=f16}"
: "${CACHE_TYPE_V:=f16}"
: "${CTX_CHECKPOINTS:=8}"
: "${CACHE_RAM:=8192}"

# Sampling defaults for silent clients: the values the vLLM predecessor served.
: "${TEMPERATURE:=0.7}"
: "${TOP_P:=0.95}"
: "${TOP_K:=20}"
: "${MIN_P:=0.0}"
: "${PRESENCE_PENALTY:=0.0}"

: "${REASONING:=on}"
: "${REASONING_FORMAT:=deepseek}"
: "${REASONING_BUDGET:=-1}"
: "${CHAT_TEMPLATE_FILE:=}"
: "${CHAT_TEMPLATE_KWARGS:={\"enable_thinking\":true,\"preserve_thinking\":true}}"

# Speculative decoding: draft-dflash or draft-dspark (both are MODEL drafters --
# a file, not a flag), or draft-mtp (the target GGUF's own MTP head, no draft
# file).  The drafter's own tuning knobs (n-min, p-min, p-split, poll) are left
# at the build's defaults: they were only ever passed for dspark, and dspark is
# not the deployed drafter.
: "${SPEC_TYPE:=draft-dflash}"
: "${SPEC_DRAFT_N_MAX:=7}"
: "${SPEC_DRAFT_NGL:=999}"
: "${SPEC_DRAFT_DEVICE:=}"
: "${SPEC_DRAFT_TYPE_K:=f16}"
: "${SPEC_DRAFT_TYPE_V:=f16}"
: "${SPEC_NGRAM_N_MATCH:=24}"
: "${SPEC_NGRAM_N_MIN:=48}"
: "${SPEC_NGRAM_N_MAX:=64}"

NO_MMPROJ="${NO_MMPROJ:-0}"
: "${MM_PROJ_FILE:=mmproj-BF16.gguf}"

export HIP_VISIBLE_DEVICES

MODEL="$MODEL_DIR/$MODEL_FILE"
DRAFT_MODEL="$DRAFT_MODEL_DIR/$DRAFT_MODEL_FILE"

if [[ ! -x "$LLAMA_BIN" ]]; then
    echo "missing executable runtime: $LLAMA_BIN" >&2
    echo "build it with: bash $SCRIPT_DIR/build-llama-rdna4.sh" >&2
    exit 1
fi
if [[ ! -r "$MODEL" ]]; then
    echo "missing or unreadable target model: $MODEL" >&2
    echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2
    exit 1
fi
if [[ "$NO_MMPROJ" != "1" ]]; then
    MM_PROJ="$MODEL_DIR/$MM_PROJ_FILE"
    if [[ ! -r "$MM_PROJ" ]]; then
        echo "vision is enabled but the projector is missing: $MM_PROJ" >&2
        echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2
        exit 1
    fi
fi
if [[ -n "$CHAT_TEMPLATE_FILE" && ! -r "$CHAT_TEMPLATE_FILE" ]]; then
    echo "missing or unreadable chat template: $CHAT_TEMPLATE_FILE" >&2
    exit 1
fi

# --- speculative decoding -------------------------------------------------
SPEC_ARGS=()
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
    SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
    if [[ "$SPEC_TYPE" == *dflash* || "$SPEC_TYPE" == *dspark* ]]; then
        if [[ ! -r "$DRAFT_MODEL" ]]; then
            echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
            exit 1
        fi
        SPEC_ARGS+=(
            --spec-draft-model "$DRAFT_MODEL"
            --spec-draft-ngl "$SPEC_DRAFT_NGL"
            --spec-draft-type-k "$SPEC_DRAFT_TYPE_K"
            --spec-draft-type-v "$SPEC_DRAFT_TYPE_V"
        )
    fi
    # ngram-mod is draftless: it needs no draft context and its window is its
    # own (--spec-ngram-mod-*), so it can ride in the chain beside a model
    # drafter. It also costs no VRAM -- a CPU hash table.
    if [[ "$SPEC_TYPE" == *ngram* ]]; then
        SPEC_ARGS+=(
            --spec-ngram-mod-n-match "$SPEC_NGRAM_N_MATCH"
            --spec-ngram-mod-n-min "$SPEC_NGRAM_N_MIN"
            --spec-ngram-mod-n-max "$SPEC_NGRAM_N_MAX"
        )
    fi
    if [[ -n "$SPEC_DRAFT_DEVICE" ]]; then
        SPEC_ARGS+=(--spec-draft-device "$SPEC_DRAFT_DEVICE")
    fi
fi

# --- template + placement -------------------------------------------------
TEMPLATE_ARGS=()
if [[ -n "$CHAT_TEMPLATE_FILE" ]]; then
    TEMPLATE_ARGS+=(--chat-template-file "$CHAT_TEMPLATE_FILE")
fi
TEMPLATE_ARGS+=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS")

PLACEMENT_ARGS=(--device "$DEVICES" --split-mode "$SPLIT_MODE" --tensor-split "$TENSOR_SPLIT")

MMPROJ_ARGS=()
if [[ "$NO_MMPROJ" == "1" ]]; then
    MMPROJ_ARGS=(--no-mmproj)
else
    # No autoload in this build: a server without the flag starts text-only
    # and answers image requests with a 500, so name the projector.
    MMPROJ_ARGS=(--mmproj "$MM_PROJ")
fi
exec "$LLAMA_BIN" \
    --model "$MODEL" \
    --alias "$ALIAS" \
    --host "$BIND_ADDR" \
    --port "$LISTEN_PORT" \
    --jinja \
    --gpu-layers "$NGL" \
    --fit "$FIT" \
    --parallel "$PARALLEL" \
    --ctx-size "$CTX_SIZE" \
    --batch-size "$BATCH_SIZE" \
    --ubatch-size "$UBATCH_SIZE" \
    --flash-attn "$FLASH_ATTN" \
    --cache-type-k "$CACHE_TYPE_K" \
    --cache-type-v "$CACHE_TYPE_V" \
    --threads "$NUM_THREADS" \
    --threads-batch "$THREAD_BATCH" \
    --ctx-checkpoints "$CTX_CHECKPOINTS" \
    --cache-ram "$CACHE_RAM" \
    --cache-prompt \
    --metrics \
    --temp "$TEMPERATURE" \
    --top-p "$TOP_P" \
    --top-k "$TOP_K" \
    --min-p "$MIN_P" \
    --presence-penalty "$PRESENCE_PENALTY" \
    --reasoning "$REASONING" \
    --reasoning-format "$REASONING_FORMAT" \
    --reasoning-budget "$REASONING_BUDGET" \
    "${TEMPLATE_ARGS[@]}" \
    "${PLACEMENT_ARGS[@]}" \
    "${MMPROJ_ARGS[@]}" \
    "${SPEC_ARGS[@]}"