Frank
293e4e4b09
The serve script only passed --chat-template-kwargs, so the binary fell back to the built-in peg-native template. A 250-token request produced a 73,600-token prompt (280x the expected ~260). Wiring in --chat-template-file drops the prompt to 62 tokens. Draft acceptance 0.79 (mean 4.14), model answers correctly, reasoning lands in reasoning_content.
133 lines
5.6 KiB
Bash
Executable File
133 lines
5.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve jcbtc/Qwen3.8-27B-IU4-Kairic-Edge as frank-large on
|
|
# Dankclaw's Strix Halo iGPU (gfx1151), using the Kairic Edge runtime
|
|
# (ciru-ai/ROCmFPX, branch kairic-edge-qwen38-27b-v1.2, commit 205a3e5) —
|
|
# the qualified IU4 (W4A4) serving path with the FFN, GDN and GDN-Output
|
|
# sidecars.
|
|
#
|
|
# Runtime: built from source (scripts/build-kairic-edge-gfx1151.sh), output at
|
|
# /home/dank/GitPackages/ROCmFPX-kairic/build-kairic/bin/llama-server. The
|
|
# Kairic Edge path is a llama.cpp build with the PromptForge sidecar env
|
|
# block (the certified runner's exact env + flags).
|
|
#
|
|
# Shape: one stream, full 262144 context (the model's native window), all
|
|
# layers on the iGPU (-ngl 999). The 27B dense model (15.5 GiB GGUF + 7.99
|
|
# GiB FFN sidecar + 1.88 GiB GDN sidecar + 0.70 GiB GDN-Output sidecar)
|
|
# fits the iGPU's GTT aperture (~114 GiB).
|
|
#
|
|
# Spec method: native MTP depth 4 (the model's MTP layers) + n-gram drafting
|
|
# (24/64/64). DFlash is NOT available for this model (no DRAFT_DFLASH spec
|
|
# type in the Kairic Edge runtime).
|
|
#
|
|
# Compatibility mode: the harness needs sampling, tools and a reasoning
|
|
# budget, which the fast-greedy path rejects. COMPATIBILITY_MODE=1 enables
|
|
# them (9.7% slower per the build guide). Reasoning is enabled (auto +
|
|
# deepseek format, so thoughts land in reasoning_content).
|
|
#
|
|
# The Qwen3.8 family template keeps prior thinking blocks across turns when
|
|
# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents
|
|
# do not lose the trail. A request-level chat_template_kwargs still wins.
|
|
# The template gates reasoning_effort to {xhigh, medium, low} — the harness
|
|
# folds its levels onto that set in models.yml (reasoningEffortMap).
|
|
#
|
|
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
|
|
# it; anything not set there falls through to the defaults below.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) ------------------
|
|
: "${RUNTIME_DIR:=/home/dank/GitPackages/ROCmFPX-kairic/build-kairic}"
|
|
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge}"
|
|
: "${HOST:=0.0.0.0}"
|
|
: "${PORT:=7770}"
|
|
: "${CONTEXT_SIZE:=262144}"
|
|
: "${PARALLEL_SLOTS:=1}"
|
|
: "${SPEC_DRAFT_N_MAX:=4}"
|
|
: "${COMPATIBILITY_MODE:=1}"
|
|
TEMPLATE_FILE="${TEMPLATE_FILE:-$SCRIPT_DIR/../dankserver/frank-medium/qwen-sharp-v22.4.0.jinja}"
|
|
|
|
SERVER_BIN="$RUNTIME_DIR/bin/llama-server"
|
|
MODEL_FILE="$MODEL_DIR/Qwen3.8-27B-IU4-Kairic-Edge.gguf"
|
|
FFN_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
|
|
GDN_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
|
|
GDN_OUTPUT_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs"
|
|
|
|
if [[ ! -x "$SERVER_BIN" ]]; then
|
|
echo "ERROR: Kairic Edge runtime not found at $SERVER_BIN" >&2
|
|
echo " Build it with: scripts/build-kairic-edge-gfx1151.sh" >&2
|
|
exit 1
|
|
fi
|
|
for f in "$MODEL_FILE" "$FFN_SIDECAR" "$GDN_SIDECAR" "$GDN_OUTPUT_SIDECAR"; do
|
|
if [[ ! -r "$f" ]]; then
|
|
echo "ERROR: missing asset: $f" >&2
|
|
echo " Fetch it with: scripts/download-frank-large-kairic.sh" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# --- PromptForge env block (the Kairic Edge runner's exact env) ----------
|
|
export HIP_VISIBLE_DEVICES=0
|
|
export HSA_OVERRIDE_GFX_VERSION=11.5.1
|
|
export LD_LIBRARY_PATH="/opt/rocm/lib:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}"
|
|
export GGML_CUDA_GRAPH_OPT=0
|
|
# Fast-greedy mode (0) rejects sampling/tools/a reasoning budget; compatibility
|
|
# mode (1) enables them. The harness needs all three, so default to 1.
|
|
if [[ "$COMPATIBILITY_MODE" == "1" ]]; then
|
|
export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0
|
|
export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
|
|
else
|
|
export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1
|
|
export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=1
|
|
fi
|
|
export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1
|
|
export PROMPTFORGE_TARGET_ONLY=0
|
|
export PROMPTFORGE_MODE=iu4_ffn
|
|
export PROMPTFORGE_IU4_SIDECAR="$FFN_SIDECAR"
|
|
export PROMPTFORGE_SIDECAR="$FFN_SIDECAR"
|
|
export PROMPTFORGE_IU4_HADAMARD=1
|
|
export PROMPTFORGE_IU4_SEGMENTED=0
|
|
export PROMPTFORGE_ENABLE_FFN_KEEPERS=0
|
|
export PROMPTFORGE_FFN_KEEPERS=late6
|
|
export PROMPTFORGE_ENABLE_GDN=1
|
|
export PROMPTFORGE_ENABLE_GDN_W8=0
|
|
export PROMPTFORGE_GDN_SIDECAR_OVERRIDE=1
|
|
export PROMPTFORGE_GDN_SIDECAR="$GDN_SIDECAR"
|
|
export PROMPTFORGE_GDN_IU4_HADAMARD=1
|
|
export PROMPTFORGE_ENABLE_GDN_KEEPERS=0
|
|
export PROMPTFORGE_ENABLE_GDN_OUTPUT=1
|
|
export PROMPTFORGE_GDN_OUTPUT_SIDECAR="$GDN_OUTPUT_SIDECAR"
|
|
export PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6
|
|
export PROMPTFORGE_ENABLE_SMALLM_IU4=1
|
|
export PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1
|
|
export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0
|
|
export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0
|
|
|
|
# --- the serve command (the Kairic Edge runner's flags, fleet-adapted) ----
|
|
ARGS=(
|
|
-m "$MODEL_FILE" --alias frank-large
|
|
--host "$HOST" --port "$PORT" --jinja
|
|
--chat-template-kwargs '{"preserve_thinking":true}'
|
|
--chat-template-file "$TEMPLATE_FILE"
|
|
-dev ROCm0 -ngl 999
|
|
-c "$CONTEXT_SIZE" -b 2048 -ub 512 -fa on -ctk f16 -ctv f16
|
|
-t 16 -tb 32 -np "$PARALLEL_SLOTS" -ctxcp 32 --cache-ram "$CACHE_RAM"
|
|
--cache-prompt --cache-idle-slots --metrics
|
|
--kairic-edge
|
|
--spec-type draft-mtp
|
|
--spec-draft-device ROCm0 --spec-draft-ngl 999
|
|
--spec-draft-type-k f16 --spec-draft-type-v f16
|
|
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" --spec-draft-n-min 0
|
|
--spec-draft-p-min 0.0 --spec-draft-p-split 0.10
|
|
--spec-draft-backend-sampling
|
|
--temp 0 --top-p 1.0 --top-k 0 --min-p 0.0
|
|
--reasoning auto --reasoning-format deepseek --reasoning-budget -1
|
|
)
|
|
|
|
exec "$SERVER_BIN" "${ARGS[@]}"
|