Frank
d6f9b8539a
The first cutover crash-looped the unit with 'invalid split file name': the fetch script concatenated the three split parts into a single GGUF, but part 1's header carries ggml.file_count=3, so the byte-correct concatenation still advertises itself as a split and the loader demands a -NNNNN-of-NNNNN filename. - serve-frank-large.sh: -m points at part 1; llama.cpp loads the split natively (parts 2-3 auto-discovered in the same directory) - deploy-remote.sh: LLM_LARGE_MODEL gate path follows - download-frank-large-unsloth.sh: assembly removed; the parts ARE the canonical payload, verified against live Hub LFS oids - docs: runbook model section + troubleshooting note, AGENTS.md, log - the redundant 87 GiB assembled file is staged on dankclaw at ~/.AGENTTRASH/20260921-frank-large-assembled/
109 lines
4.5 KiB
Bash
Executable File
109 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS as frank-large on
|
|
# Dankclaw's Strix Halo iGPU (gfx1151), using the unsloth llama.cpp
|
|
# prebuilt (b11030-mix-5ff778e, ROCm gfx1151) — the first runtime with an
|
|
# MTP graph for the qwen4exp architecture (stock ggml-org mainline has no
|
|
# MTP graph, no cross-model tensor borrowing and no --spec-type draft-mtp
|
|
# for this model).
|
|
#
|
|
# Runtime: the prebuilt app tarball from the unslothai/llama.cpp releases
|
|
# (tag b11030-mix-5ff778e or newer), extracted under
|
|
# /home/dank/GitPackages/unsloth-llama-b11030/. It bundles its own ROCm
|
|
# runtime, so there is no host SDK to gate on and no venv to symlink.
|
|
#
|
|
# Shape: one stream, 200k context (operator's call, 2026-09-21), all layers
|
|
# on the iGPU (-ngl 999). The 87 GiB model lives in the iGPU's GTT aperture
|
|
# over the shared LPDDR (gtt_total ~114 GiB), the same regime the CIRU
|
|
# Orca predecessor ran in. The MTP shared-Q8_0 head speculates (1.3-1.7x at
|
|
# concurrency 1, per the model card).
|
|
#
|
|
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
|
|
# it; anything not set there falls through to the defaults below.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) ------------------
|
|
: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}"
|
|
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS}"
|
|
: "${HOST:=0.0.0.0}"
|
|
: "${PORT:=7770}"
|
|
: "${CONTEXT_SIZE:=200000}"
|
|
: "${PARALLEL_SLOTS:=1}"
|
|
: "${ENABLE_MTP:=1}"
|
|
: "${SPEC_DRAFT_N_MAX:=2}"
|
|
: "${ENABLE_VISION:=0}"
|
|
|
|
SERVER_BIN="$RUNTIME_DIR/llama-server"
|
|
|
|
if [[ ! -x "$SERVER_BIN" ]]; then
|
|
echo "missing runtime: $SERVER_BIN" >&2
|
|
echo "fetch and extract it:" >&2
|
|
echo " curl -fL -o /tmp/unsloth-llama.tar.gz https://github.com/unslothai/llama.cpp/releases/download/b11030-mix-5ff778e/app-b11030-mix-5ff778e-linux-x64-rocm-gfx1151.tar.gz" >&2
|
|
echo " mkdir -p $RUNTIME_DIR && tar xzf /tmp/unsloth-llama.tar.gz -C $RUNTIME_DIR" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Point -m at the FIRST split part, not a concatenated file: the parts
|
|
# carry ggml.file_count=3 metadata in part 1's header, so even a
|
|
# byte-correct concatenation still advertises itself as a 3-way split and
|
|
# the loader rejects a non-split filename ("invalid split file name").
|
|
# With -m on part 1, llama.cpp loads the 3-way split natively (parts 2-3
|
|
# are auto-discovered in the same directory).
|
|
MODEL_FILE="$MODEL_DIR/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf"
|
|
if [[ ! -r "$MODEL_FILE" ]]; then
|
|
echo "missing target model: $MODEL_FILE" >&2
|
|
echo "fetch it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The MTP shared head borrows the token embedding and the output projection
|
|
# from the main model at runtime. It must be passed explicitly with -md:
|
|
# sidecar auto-discovery does not search the mtp/ subfolder, so
|
|
# --spec-type draft-mtp on its own would find nothing and serve at the
|
|
# main model's speed with no error saying why.
|
|
MTP_FILE="$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
|
|
if [[ "$ENABLE_MTP" == "1" && ! -r "$MTP_FILE" ]]; then
|
|
echo "missing MTP head: $MTP_FILE" >&2
|
|
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The Qwen3.8 family template keeps prior thinking blocks across turns when
|
|
# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents
|
|
# do not lose the trail. A request-level chat_template_kwargs still wins.
|
|
# The template gates reasoning_effort to {xhigh, medium, low} — the harness
|
|
# folds its levels onto that set in models.yml (reasoningEffortMap).
|
|
ARGS=(
|
|
-m "$MODEL_FILE"
|
|
-c "$CONTEXT_SIZE"
|
|
-np "$PARALLEL_SLOTS"
|
|
-ngl 999
|
|
--jinja
|
|
--alias frank-large
|
|
--host "$HOST"
|
|
--port "$PORT"
|
|
--chat-template-kwargs '{"preserve_thinking":true}'
|
|
)
|
|
if [[ "$ENABLE_MTP" == "1" ]]; then
|
|
ARGS+=(-md "$MTP_FILE" --spec-type draft-mtp --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
|
fi
|
|
if [[ "$ENABLE_VISION" == "1" ]]; then
|
|
MMPROJ_FILE="$MODEL_DIR/vision/mmproj-F16.gguf"
|
|
if [[ ! -r "$MMPROJ_FILE" ]]; then
|
|
echo "missing projector: $MMPROJ_FILE" >&2
|
|
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
|
exit 1
|
|
fi
|
|
ARGS+=(--mmproj "$MMPROJ_FILE")
|
|
fi
|
|
|
|
exec "$SERVER_BIN" "${ARGS[@]}"
|