serverAppConfigs/scripts/serve-frank-primary.sh

#!/usr/bin/env bash
# Serve unsloth/Qwen3.8-Flash-Next-GGUF at Q8_0 as frank-primary on
# Dankserver's two Radeon AI PRO R9700s (gfx1201), using the unsloth
# b11030-mix prebuilt (ROCm gfx120X, bundled runtime).
#
# Shape: one stream, the model's full 262144 context, all non-overridden
# tensors on the GPUs (--n-gpu-layers 999 across both cards), with the
# routed experts and the 51B-param n-gram embedding table computed in host
# RAM via --override-tensor (the x4-pair law: streaming over PCIe loses,
# computing in RAM does not care). mmap stays ON (the PLE table must stay
# page-cache-backed; --no-mmap forces it into anonymous memory and the
# cgroup kill is silent).
#
# Spec chain: draft-mtp (the shared Q8_0 head, loaded via -md) chained with
# the draftless ngram-mod matcher. The shared head borrows the target's
# embedding tensors, which is why per_layer_token_embd.weight must be
# CPU-resident (see frank-primary.env). depth 2.
#
# Vision: the BF16 projector, loaded explicitly via --mmproj (this build
# never autoloads projectors — the frank-medium 2026-09-17 trap: a server
# started without it comes up healthy and text-only, and answers image
# requests with 500).
#
# Template: the GGUF's embedded chat template via --jinja. The Qwen3.8
# family template gates reasoning_effort to {xhigh, medium, low} — the
# harness folds its levels onto that set in models.yml
# (reasoningEffortMap). If the embedded template ever falls back to
# peg-native and bloats prompts (~73,600 tokens for a ~250-token request),
# switch to --chat-template-file scripts/qwen-sharp-v22.4.0.jinja the way
# frank-large does.
#
# Tunables come from frank-primary.env unless FRANK_PRIMARY_ENV_FILE
# overrides it; anything not set there falls through to the defaults below.
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"
if [[ -f "$ENV_FILE" ]]; then
    set -a
    source "$ENV_FILE"
    set +a
fi

# --- defaults (used when the env file does not set them) ------------------
: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}"
: "${MODEL_ROOT:=/home/dank/models/Qwen3.8-Flash-Next-GGUF}"
: "${BIND_HOST:=0.0.0.0}"
: "${PORT:=7770}"
: "${CONTEXT_SIZE:=262144}"
: "${PARALLEL_SLOTS:=1}"
: "${SPEC_DRAFT_N_MAX:=2}"
: "${NGL:=999}"
: "${OT_OVERRIDES:=exps=CPU,ngram_embeddings=CPU,per_layer_token_embd.weight=CPU}"
: "${TENSOR_SPLIT:=1,1}"
: "${SPEC_DRAFT_DEVICE:=ROCm0}"
: "${BATCH_SIZE:=8192}"
: "${UBATCH_SIZE:=512}"
: "${CACHE_RAM:=16384}"

SERVER_BIN="$RUNTIME_DIR/llama-server"
MODEL_PARTS=(
    "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00001-of-00006.gguf"
    "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00002-of-00006.gguf"
    "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00003-of-00006.gguf"
    "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00004-of-00006.gguf"
    "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00005-of-00006.gguf"
    "$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00006-of-00006.gguf"
)
DRAFTER_FILE="$MODEL_ROOT/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
MMPROJ_FILE="$MODEL_ROOT/mmproj-BF16.gguf"

if [[ ! -x "$SERVER_BIN" ]]; then
    echo "runtime not found at $SERVER_BIN" >&2
    echo "  fetch the unsloth b11030-mix gfx120X prebuilt (see the runbook)" >&2
    exit 1
fi
# Every part is checked: part 1 is the -m target and parts 2-6 are followed
# by the split-file loader, so a missing tail part fails at LOAD time with
# a confusing partial-mmap error unless it is caught here.
for f in "${MODEL_PARTS[@]}" "$DRAFTER_FILE" "$MMPROJ_FILE"; do
    if [[ ! -f "$f" ]]; then
        echo "missing asset: $f" >&2
        echo "  run scripts/download-frank-primary.sh on this host" >&2
        exit 1
    fi
done

# --- env (the unsloth build's bundled ROCm runtime) -----------------------
# Both R9700s (gfx1201, HIP devices 0 and 1); the host's Raphael iGPU is
# device 2 and must stay hidden from the enumeration.
export HIP_VISIBLE_DEVICES=0,1
export LD_LIBRARY_PATH="$RUNTIME_DIR:${LD_LIBRARY_PATH:-}"

# --- the serve command ---------------------------------------------------
ARGS=(
    --host "$BIND_HOST"
    --port "$PORT"
    --model "${MODEL_PARTS[0]}"
    --alias frank-primary
    --ctx-size "$CONTEXT_SIZE"
    --parallel "$PARALLEL_SLOTS"
    --n-gpu-layers "$NGL"
    --tensor-split "$TENSOR_SPLIT"
    --jinja
    --reasoning auto
    --reasoning-format deepseek
    --reasoning-budget -1
    --flash-attn on
    --batch-size "$BATCH_SIZE"
    --ubatch-size "$UBATCH_SIZE"
    --cache-ram "$CACHE_RAM"
    # Vision: explicit, never autoloaded by this build
    --mmproj "$MMPROJ_FILE"
    # Spec chain: the shared MTP head + the draftless ngram matcher
    --spec-type draft-mtp,ngram-mod
    --model-draft "$DRAFTER_FILE"
    --spec-draft-n-max "$SPEC_DRAFT_N_MAX"
    --spec-draft-device "$SPEC_DRAFT_DEVICE"
    # Greedy server default: deterministic e2e, and the harness overrides
    # sampling per-request (models.yml compat.extraBody carries Qwen's
    # recommended temp 1.0 / top_p 0.95 / top_k 20).
    --temp 0
    --top-p 1.0
    --top-k 0
    --min-p 0.0
)

# Comma-separated <pattern>=<backend> pairs -> repeated --override-tensor.
IFS=',' read -ra _ots <<< "$OT_OVERRIDES"
for ot in "${_ots[@]}"; do
    ARGS+=(--override-tensor "$ot")
done

exec "$SERVER_BIN" "${ARGS[@]}"