Frank
8551bb006b
frank-small: llama.cpp CUDA (master, sm_86) on MiMo Q6_K GGUF, 2 x 262144, asymmetric KV (f16 K + q8_0 V, flash-attn pinned for the V quant)
122 lines
5.4 KiB
Bash
Executable File
122 lines
5.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) as frank-large on
|
|
# Dankclaw's Strix Halo iGPU (gfx1151), using gufo (gufo-org/gufo) — the
|
|
# inference engine built specifically for this hardware.
|
|
#
|
|
# Runtime: /home/dank/GitPackages/gufo-v0.1.1/bin/gufo, built from source
|
|
# against the host ROCm by scripts/build-gufo.sh. It replaced the stock
|
|
# llama.cpp prebuilt (unsloth b11030) on 2026-09-28; port, alias and client
|
|
# contract did not move.
|
|
#
|
|
# Payload: ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF's
|
|
# UD-Q8_K_XL, fetched by scripts/download-frank-large-swift.sh. It replaced
|
|
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL on 2026-09-28. The two are structurally
|
|
# interchangeable for this engine — same `qwen35` architecture, the same 866
|
|
# tensors in the same order, the same per-tensor Q8_0/F32/BF16 layout, the
|
|
# same MTP head at blk.64 — so the swap needed no engine change. The fine-tune
|
|
# is abliterated (uncensored) and carries the SWIFT OPEN LICENSE 1.0, which
|
|
# is not an OSI licence.
|
|
#
|
|
# Shape: one session, full 262144 context (the model's native window). The
|
|
# 27B dense weights (16.35 GiB at UD-Q4_K_XL) + the DFlash2 drafter (1.07) + KV +
|
|
# compute buffers fit the iGPU's GTT aperture (~114 GiB); a load at this
|
|
# context measured 66.5 GiB of the 116.7 GiB device.
|
|
#
|
|
# Spec method: DFlash2 (a block-diffusion drafter), gufo's adaptive block
|
|
# policy, capped at DRAFT_TOKENS proposals per step. The drafter is the
|
|
# z-lab Qwen3.8-27B-DFlash2-Q4_K_M and it has a sliding window of 2048
|
|
# (small draft KV). It is UNCHANGED by the model swap and still lives beside
|
|
# the previous target, which is why DRAFTER_DIR is its own tunable.
|
|
#
|
|
# Sampling: greedy (--temperature 0) server-side, as every frank-large has
|
|
# served. The harness overrides it per request via extraBody (1.0 / 0.95 /
|
|
# top-k 20).
|
|
#
|
|
# Template: gufo compiles the official Qwen Jinja into the binary and takes
|
|
# NO --chat-template-file — and it is not a Jinja engine at all
|
|
# (src/models/qwen/chat_template.cpp is a hand-written C++ renderer), so the
|
|
# fleet's scripts/qwen-sharp-v22.4.0.jinja cannot be supplied to it. That law
|
|
# was given up when this endpoint moved to gufo on 2026-09-28, by decision:
|
|
# the terse system-prompt injection, the in-band <|think_*|> control, the
|
|
# json|xml tool_call_format switch and the tool-argument truncation are all
|
|
# absent. gufo's own renderer keeps the effort instruction strings and
|
|
# preserve_thinking, which are the parts the harness depends on. Accepted
|
|
# 2026-09-28; recovering the terse prompt from the client side is the
|
|
# cheapest partial restore.
|
|
#
|
|
# Text-only: gufo auto-discovers a mmproj beside the model and this payload
|
|
# has none (the base repo ships one separately, unused), so the endpoint
|
|
# stays text-only.
|
|
#
|
|
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
|
|
# it; anything not set there falls through to the defaults below.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) ------------------
|
|
: "${RUNTIME_DIR:=/home/dank/GitPackages/gufo-v0.1.1}"
|
|
: "${MODEL_DIR:=/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL}"
|
|
: "${ALIAS:=frank-large}"
|
|
: "${HOST:=0.0.0.0}"
|
|
: "${PORT:=7770}"
|
|
: "${CONTEXT_SIZE:=262144}"
|
|
: "${SESSIONS:=1}"
|
|
: "${DRAFT_TOKENS:=7}"
|
|
: "${DRAFT_POLICY:=adaptive}"
|
|
|
|
GUFO_BIN="$RUNTIME_DIR/bin/gufo"
|
|
|
|
# The target GGUF and the drafter are named explicitly because they do not
|
|
# share a directory any more: the target is a different repository's artifact
|
|
# and keeps its own long filename, while the DFlash2 drafter is unchanged and
|
|
# still lives beside the previous target. MODEL_FILE/DRAFTER_FILE default to
|
|
# the deployed payload, so the env file only has to state them when it moves.
|
|
: "${MODEL_FILE:=$MODEL_DIR/Qwen3.8-27B-UD-Q4_K_XL.gguf}"
|
|
: "${DRAFTER_DIR:=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL}"
|
|
: "${DRAFTER_FILE:=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
|
|
|
|
if [[ ! -x "$GUFO_BIN" ]]; then
|
|
echo "runtime not found at $GUFO_BIN" >&2
|
|
echo "frank-large: build it: bash scripts/build-gufo.sh" >&2
|
|
exit 1
|
|
fi
|
|
for f in "$MODEL_FILE" "$DRAFTER_FILE"; do
|
|
if [[ ! -f "$f" ]]; then
|
|
echo "missing asset $f (run the matching scripts/download-frank-large*.sh)" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# --- env -------------------------------------------------------------------
|
|
# gufo links against the host ROCm through its RPATH (/opt/rocm ->
|
|
# /etc/alternatives/rocm), so no LD_LIBRARY_PATH is needed. The device is
|
|
# already gfx1151 — `gufo diagnose` reports it natively — so no
|
|
# HSA_OVERRIDE_GFX_VERSION either; the override existed for the prebuilt
|
|
# llama.cpp runtime and can only misreport the arch here.
|
|
export HIP_VISIBLE_DEVICES=0
|
|
|
|
# --- the serve command ---------------------------------------------------
|
|
# Server options precede the subcommand, model options follow it.
|
|
exec "$GUFO_BIN" serve \
|
|
--host "$HOST" \
|
|
--port "$PORT" \
|
|
--sessions "$SESSIONS" \
|
|
llm \
|
|
--model "$MODEL_FILE" \
|
|
--served-model-name "$ALIAS" \
|
|
--context "$CONTEXT_SIZE" \
|
|
--speculative dflash2 \
|
|
--dflash-model "$DRAFTER_FILE" \
|
|
--draft-tokens "$DRAFT_TOKENS" \
|
|
--draft-policy "$DRAFT_POLICY" \
|
|
--temperature 0 \
|
|
--think auto \
|
|
--preserve-thinking on
|