Frank
b149d02187
Replace the CIRU v4.4.1 Orca runtime with the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm gfx1151, bundles its own ROCm) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (87.3 GiB, three HF split parts concatenated) on the Strix Halo iGPU. One conversation, 200,000 ctx, PARALLEL_SLOTS=1, MTP depth 2 via the explicit -md shared-Q8_0 head (sidecar auto-discovery does not search the mtp/ subfolder), text-only. Fleet contract unchanged: port 7770, alias frank-large. - serve-frank-large.sh: unsloth binary, -m/-md, --spec-type draft-mtp --spec-draft-n-max 2, --jinja, preserve_thinking; gates on binary, GGUF, MTP head; conditional --mmproj on ENABLE_VISION=1 - download-frank-large-unsloth.sh: resumable fetch of 3 parts + MTP head + mmproj, each verified against its LFS sha256 fetched live from the Hub tree API; concatenates parts, records/verifies SHA256SUMS - deploy-remote.sh: LLM_LARGE_MODEL/LLM_LARGE_DRAFT paths, pkill pattern on the model path (not the binary name), wait_port_free gate - tests: seams for 200k/1-slot, explicit -md, text-only, handler gates, sweep safety
103 lines
4.1 KiB
Bash
Executable File
103 lines
4.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS as frank-large on
|
|
# Dankclaw's Strix Halo iGPU (gfx1151), using the unsloth llama.cpp
|
|
# prebuilt (b11030-mix-5ff778e, ROCm gfx1151) — the first runtime with an
|
|
# MTP graph for the qwen4exp architecture (stock ggml-org mainline has no
|
|
# MTP graph, no cross-model tensor borrowing and no --spec-type draft-mtp
|
|
# for this model).
|
|
#
|
|
# Runtime: the prebuilt app tarball from the unslothai/llama.cpp releases
|
|
# (tag b11030-mix-5ff778e or newer), extracted under
|
|
# /home/dank/GitPackages/unsloth-llama-b11030/. It bundles its own ROCm
|
|
# runtime, so there is no host SDK to gate on and no venv to symlink.
|
|
#
|
|
# Shape: one stream, 200k context (operator's call, 2026-09-21), all layers
|
|
# on the iGPU (-ngl 999). The 87 GiB model lives in the iGPU's GTT aperture
|
|
# over the shared LPDDR (gtt_total ~114 GiB), the same regime the CIRU
|
|
# Orca predecessor ran in. The MTP shared-Q8_0 head speculates (1.3-1.7x at
|
|
# concurrency 1, per the model card).
|
|
#
|
|
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
|
|
# it; anything not set there falls through to the defaults below.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) ------------------
|
|
: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}"
|
|
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS}"
|
|
: "${HOST:=0.0.0.0}"
|
|
: "${PORT:=7770}"
|
|
: "${CONTEXT_SIZE:=200000}"
|
|
: "${PARALLEL_SLOTS:=1}"
|
|
: "${ENABLE_MTP:=1}"
|
|
: "${SPEC_DRAFT_N_MAX:=2}"
|
|
: "${ENABLE_VISION:=0}"
|
|
|
|
SERVER_BIN="$RUNTIME_DIR/llama-server"
|
|
|
|
if [[ ! -x "$SERVER_BIN" ]]; then
|
|
echo "missing runtime: $SERVER_BIN" >&2
|
|
echo "fetch and extract it:" >&2
|
|
echo " curl -fL -o /tmp/unsloth-llama.tar.gz https://github.com/unslothai/llama.cpp/releases/download/b11030-mix-5ff778e/app-b11030-mix-5ff778e-linux-x64-rocm-gfx1151.tar.gz" >&2
|
|
echo " mkdir -p $RUNTIME_DIR && tar xzf /tmp/unsloth-llama.tar.gz -C $RUNTIME_DIR" >&2
|
|
exit 1
|
|
fi
|
|
|
|
MODEL_FILE="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf"
|
|
if [[ ! -r "$MODEL_FILE" ]]; then
|
|
echo "missing target model: $MODEL_FILE" >&2
|
|
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The MTP shared head borrows the token embedding and the output projection
|
|
# from the main model at runtime. It must be passed explicitly with -md:
|
|
# sidecar auto-discovery does not search the mtp/ subfolder, so
|
|
# --spec-type draft-mtp on its own would find nothing and serve at the
|
|
# main model's speed with no error saying why.
|
|
MTP_FILE="$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
|
|
if [[ "$ENABLE_MTP" == "1" && ! -r "$MTP_FILE" ]]; then
|
|
echo "missing MTP head: $MTP_FILE" >&2
|
|
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The Qwen3.8 family template keeps prior thinking blocks across turns when
|
|
# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents
|
|
# do not lose the trail. A request-level chat_template_kwargs still wins.
|
|
# The template gates reasoning_effort to {xhigh, medium, low} — the harness
|
|
# folds its levels onto that set in models.yml (reasoningEffortMap).
|
|
ARGS=(
|
|
-m "$MODEL_FILE"
|
|
-c "$CONTEXT_SIZE"
|
|
-np "$PARALLEL_SLOTS"
|
|
-ngl 999
|
|
--jinja
|
|
--alias frank-large
|
|
--host "$HOST"
|
|
--port "$PORT"
|
|
--chat-template-kwargs '{"preserve_thinking":true}'
|
|
)
|
|
if [[ "$ENABLE_MTP" == "1" ]]; then
|
|
ARGS+=(-md "$MTP_FILE" --spec-type draft-mtp --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
|
fi
|
|
if [[ "$ENABLE_VISION" == "1" ]]; then
|
|
MMPROJ_FILE="$MODEL_DIR/vision/mmproj-F16.gguf"
|
|
if [[ ! -r "$MMPROJ_FILE" ]]; then
|
|
echo "missing projector: $MMPROJ_FILE" >&2
|
|
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
|
exit 1
|
|
fi
|
|
ARGS+=(--mmproj "$MMPROJ_FILE")
|
|
fi
|
|
|
|
exec "$SERVER_BIN" "${ARGS[@]}"
|