feat(frank-large): unsloth llama.cpp Qwen3.8-Flash-Next UD-IQ4_XS (200k/1 slot, MTP depth 2)
Replace the CIRU v4.4.1 Orca runtime with the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm gfx1151, bundles its own ROCm) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (87.3 GiB, three HF split parts concatenated) on the Strix Halo iGPU. One conversation, 200,000 ctx, PARALLEL_SLOTS=1, MTP depth 2 via the explicit -md shared-Q8_0 head (sidecar auto-discovery does not search the mtp/ subfolder), text-only. Fleet contract unchanged: port 7770, alias frank-large. - serve-frank-large.sh: unsloth binary, -m/-md, --spec-type draft-mtp --spec-draft-n-max 2, --jinja, preserve_thinking; gates on binary, GGUF, MTP head; conditional --mmproj on ENABLE_VISION=1 - download-frank-large-unsloth.sh: resumable fetch of 3 parts + MTP head + mmproj, each verified against its LFS sha256 fetched live from the Hub tree API; concatenates parts, records/verifies SHA256SUMS - deploy-remote.sh: LLM_LARGE_MODEL/LLM_LARGE_DRAFT paths, pkill pattern on the model path (not the binary name), wait_port_free gate - tests: seams for 200k/1-slot, explicit -md, text-only, handler gates, sweep safety
This commit is contained in:
Frank
+14
-17
@@ -1028,22 +1028,20 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# frank-large is the CIRU v4.4.1 runtime (QSA window hotfix) serving
|
||||
# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU, as a ROOT
|
||||
# unit since the 2026-09-19 cutover: the runtime is a host build against
|
||||
# the private ROCm 10 SDK, so there is no container image to install and no
|
||||
# generated unit to dodge. relink_unit points
|
||||
# frank-large is the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm
|
||||
# gfx1151) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS on the Strix
|
||||
# Halo iGPU, as a ROOT unit since the 2026-09-19 cutover: the runtime is a
|
||||
# prebuilt tarball with its own bundled ROCm, so there is no container
|
||||
# image to install and no generated unit to dodge. relink_unit points
|
||||
# /etc/systemd/system/frank-large.service at the repo file and boot
|
||||
# persistence is that unit's [Install] section.
|
||||
#
|
||||
# It replaces the halogen-flash-server quadlet, which served
|
||||
# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16.
|
||||
# The quadlet is retired HERE (the unit file moves into the home trash,
|
||||
# never `rm`), and the port gate is the same as the 2026-09-16 cutover:
|
||||
# nothing starts until 7770 is released.
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf}"
|
||||
LLM_LARGE_PLE="${LLM_LARGE_PLE:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/ple/ple.payload.bin}"
|
||||
# It replaces the CIRU v4.4.1 Orca runtime (jcbtc/Qwen3.8-Flash-CIRU-
|
||||
# STRIX-Orca), which had replaced the halogen-flash-server quadlet. The
|
||||
# port gate is the same as every cutover since 2026-09-16: nothing starts
|
||||
# until 7770 is released.
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf}"
|
||||
|
||||
retire_halogen_large_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
@@ -1080,10 +1078,10 @@ deploy_llm_large() {
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the deploy log to say why.
|
||||
local missing=0 f
|
||||
for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT" "$LLM_LARGE_PLE"; do
|
||||
for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: payload absent at $f" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large-orca.sh" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large-unsloth.sh" >&2
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
@@ -1098,8 +1096,7 @@ deploy_llm_large() {
|
||||
# The pattern never appears in the deploying shell's own command line, so
|
||||
# -f is safe here (it is not in `bash deploy-remote.sh`).
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
pkill -f 'Qwen3.8-Flash-CIRU-STRIX-Orca' 2>/dev/null || true
|
||||
sleep 2
|
||||
pkill -f 'Qwen3.8-Flash-Next-UD-IQ4_XS' 2>/dev/null || true
|
||||
# Nothing may start until the predecessor has released the port. On
|
||||
# 2026-09-16 the cutover failed exactly here: the old server logged
|
||||
# "Deactivated successfully", the next engine tried to bind 7770 a second
|
||||
|
||||
Executable
+118
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download and assemble unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (the
|
||||
# frank-large model) on Dankclaw.
|
||||
#
|
||||
# bash scripts/download-frank-large-unsloth.sh
|
||||
# bash scripts/download-frank-large-unsloth.sh --verify-only
|
||||
#
|
||||
# The UD-IQ4_XS GGUF ships as three llama.cpp split parts (HF caps uploads at
|
||||
# 50 GB); concatenating them in numeric order reconstructs the single GGUF.
|
||||
# The MTP shared-Q8_0 head borrows the token embedding and the output
|
||||
# projection from the main model at runtime (it is the 2.6 GB remainder of
|
||||
# the 4B MTP layer). The F16 projector is fetched for a future vision flip;
|
||||
# the serve script does not load it while ENABLE_VISION=0.
|
||||
#
|
||||
# Integrity: every part is checked against its LFS sha256 (fetched live from
|
||||
# the Hub tree API, not trusted from disk), then the assembled GGUF against
|
||||
# the recorded SHA256SUMS. The repo is public (gated: false); an HF token at
|
||||
# ~/.cache/huggingface/token is used when present to clear the anonymous rate
|
||||
# limit on a 94 GB transfer.
|
||||
#
|
||||
# Layout after a successful run (MODEL_DIR):
|
||||
# Qwen3.8-Flash-Next-UD-IQ4_XS.gguf 93,682,584,224 main model
|
||||
# mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf 2,786,568,256 draft head
|
||||
# vision/mmproj-F16.gguf 904,004,000 projector
|
||||
# parts/ the three verified transport parts (kept for --verify-only)
|
||||
# SHA256SUMS
|
||||
set -euo pipefail
|
||||
|
||||
REPO="unsloth/Qwen3.8-Flash-Next-GGUF"
|
||||
MODEL_DIR="/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS"
|
||||
BASE="https://huggingface.co/$REPO/resolve/main"
|
||||
|
||||
mode="download"
|
||||
[[ "${1:-}" == "--verify-only" ]] && mode="verify"
|
||||
|
||||
token=""
|
||||
auth=()
|
||||
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
|
||||
token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")"
|
||||
auth=(-H "Authorization: Bearer $token")
|
||||
fi
|
||||
|
||||
dl() { # dl <relpath>
|
||||
local rel="$1" dest="$MODEL_DIR/$rel"
|
||||
[[ -s "$dest" ]] && return 0
|
||||
mkdir -p "$(dirname "$dest")"
|
||||
echo "fetching $rel"
|
||||
curl -fL -C - "${auth[@]}" --retry 5 --retry-delay 3 -o "$dest" "$BASE/$rel"
|
||||
}
|
||||
|
||||
# relpath size sha256 — the LFS oids from the Hub tree API (2026-09-21)
|
||||
FILES=(
|
||||
"UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf 10946624 5ce89370720f8bf90890f439361282104c1aa1482d4013bb9a50923e758e71a4"
|
||||
"UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00002-of-00003.gguf 49835229856 577a38a2392b40ca2193cea502e1d92f60b8cd370675d308e0ec21885d9daaa7"
|
||||
"UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00003-of-00003.gguf 43836407744 d4634e6d84f0ebb0940be15c90d3790bf6464e3dea3a1cddc567dc0e83ad8833"
|
||||
"MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf 2786568256 5ff54097406a905cf3a724c709124ceb0e3e10235ee862298969e91c96fa96e6"
|
||||
"mmproj-F16.gguf 904004000 1f7b7f0b984cf065c604360c29c8098362ed61b290db0ff12c6f360bb1a8a980"
|
||||
)
|
||||
|
||||
if [[ "$mode" == "download" ]]; then
|
||||
for line in "${FILES[@]}"; do
|
||||
read -r rel size _ <<<"$line"
|
||||
dl "$rel"
|
||||
done
|
||||
fi
|
||||
|
||||
echo "verifying parts against the live Hub tree API"
|
||||
python3 - "$MODEL_DIR" "$REPO" <<'PY'
|
||||
import hashlib, json, os, sys, urllib.request
|
||||
|
||||
model_dir, repo = sys.argv[1], sys.argv[2]
|
||||
tree = json.load(urllib.request.urlopen(
|
||||
f"https://huggingface.co/api/models/{repo}/tree/main?recursive=true"))
|
||||
want = {}
|
||||
for f in tree:
|
||||
if f["type"] == "file" and f.get("lfs"):
|
||||
want[f["path"]] = (f["lfs"]["size"], f["lfs"]["oid"])
|
||||
|
||||
def sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1 << 24), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
ok = True
|
||||
for rel, (size, oid) in sorted(want.items()):
|
||||
if not rel.startswith(("UD-IQ4_XS/", "MTP/")) and rel != "mmproj-F16.gguf":
|
||||
continue
|
||||
path = os.path.join(model_dir, rel)
|
||||
if not os.path.isfile(path):
|
||||
print(f"MISSING {rel}"); ok = False; continue
|
||||
actual_size = os.path.getsize(path)
|
||||
if actual_size != size:
|
||||
print(f"SIZE MISMATCH {rel}: {actual_size} != {size}"); ok = False; continue
|
||||
digest = sha256(path)
|
||||
if digest != oid:
|
||||
print(f"SHA256 MISMATCH {rel}"); ok = False; continue
|
||||
print(f"ok {rel}")
|
||||
sys.exit(0 if ok else 1)
|
||||
PY
|
||||
|
||||
FINAL="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf"
|
||||
if [[ "$mode" == "download" ]]; then
|
||||
echo "assembling the GGUF from the split parts"
|
||||
cat "$MODEL_DIR"/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-0000{1,2,3}-of-00003.gguf > "$FINAL"
|
||||
[[ "$(stat -c %s "$FINAL")" -eq 93682584224 ]] || {
|
||||
echo "assembled size wrong: $(stat -c %s "$FINAL")" >&2; exit 1; }
|
||||
mkdir -p "$MODEL_DIR/mtp" "$MODEL_DIR/vision"
|
||||
[[ -e "$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" ]] ||
|
||||
mv "$MODEL_DIR/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" "$MODEL_DIR/mtp/"
|
||||
[[ -e "$MODEL_DIR/vision/mmproj-F16.gguf" ]] || mv "$MODEL_DIR/mmproj-F16.gguf" "$MODEL_DIR/vision/"
|
||||
(cd "$MODEL_DIR" && sha256sum Qwen3.8-Flash-Next-UD-IQ4_XS.gguf > SHA256SUMS)
|
||||
fi
|
||||
|
||||
echo "verifying assembled files against SHA256SUMS"
|
||||
(cd "$MODEL_DIR" && sha256sum -c SHA256SUMS)
|
||||
echo "frank-large payload complete: $MODEL_DIR"
|
||||
+26
-31
@@ -1,44 +1,39 @@
|
||||
# frank-large — CIRU v4.4.1 (QSA window hotfix) serving
|
||||
# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU (gfx1151).
|
||||
# frank-large — unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm
|
||||
# gfx1151) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS on the Strix
|
||||
# Halo iGPU.
|
||||
#
|
||||
# These are the launcher tunables read by the runtime's
|
||||
# scripts/ciru/run-orca-server.sh (see the serve script). The 36-variable
|
||||
# kernel bundle and the PLE pager defaults live in the runtime's own
|
||||
# profiles/strix-halo-production.env, which the launcher sources AFTER this
|
||||
# file; every profile line is a ${VAR:-default}, so a value set here wins.
|
||||
# These are the tunables read by scripts/serve-frank-large.sh. The flag
|
||||
# list is owned by the serve script; this file only sets fleet values.
|
||||
#
|
||||
# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP runtime
|
||||
# reads those bare (the 2026-08-13 TEMP incident). HOST is the one exception
|
||||
# because the launcher itself consumes it for --host; its value doubles as a
|
||||
# JIT-cache directory name, which is why 0.0.0.0 is safe.
|
||||
# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP
|
||||
# runtime reads those bare (the 2026-08-13 TEMP incident). HOST is the one
|
||||
# exception because the serve script itself consumes it for --host.
|
||||
|
||||
# The assembled payload (scripts/download-frank-large-orca.sh): the launcher
|
||||
# refuses to start without the main GGUF, the MTP head and the PLE sidecar.
|
||||
MODEL_DIR=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca
|
||||
# The assembled payload (scripts/download-frank-large-unsloth.sh): the
|
||||
# serve script refuses to start without the main GGUF and the MTP head.
|
||||
MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS
|
||||
|
||||
# Fleet endpoint: the same port and alias the halogen predecessor served, so
|
||||
# The prebuilt runtime (see the serve script for the fetch line).
|
||||
RUNTIME_DIR=/home/dank/GitPackages/unsloth-llama-b11030
|
||||
|
||||
# Fleet endpoint: the same port and alias the CIRU predecessor served, so
|
||||
# no client, probe or models.yml entry changes address.
|
||||
HOST=0.0.0.0
|
||||
PORT=7770
|
||||
|
||||
# One stream, full context (operator call, 2026-09-19). MTP requires exactly
|
||||
# one slot in this runtime, so the drafter's constraint and the operator's
|
||||
# ask are the same number. 262144 is the model's trained window; the GGUF
|
||||
# carries no rope scaling.
|
||||
CONTEXT_SIZE=262144
|
||||
# One stream, 200k context (operator's call, 2026-09-21). The model's
|
||||
# trained window is 262144; 200000 is the requested serving shape.
|
||||
CONTEXT_SIZE=200000
|
||||
PARALLEL_SLOTS=1
|
||||
|
||||
# MTP head: on. Depth 4 is the Orca variant's own default (mtp_depth_default
|
||||
# in the launcher); the sidecar is the Q8_0 MTP file from the model repo.
|
||||
# MTP shared-Q8_0 head: on. Draft depth 2 is the model card's default
|
||||
# ("higher drafts more but each guess is accepted less often"). The shared
|
||||
# head borrows the token embedding and output projection from the main
|
||||
# model, so the file is 2.6 GB rather than 4.1.
|
||||
ENABLE_MTP=1
|
||||
SPEC_DRAFT_N_MAX=2
|
||||
|
||||
# Text-only for now: the F16 projector is on disk at
|
||||
# vision/mmproj-Qwen3.8-Flash-Orca-F16.mmproj; flip to 1 to enable it. The
|
||||
# fleet's image path is frank-small either way, so models.yml stays
|
||||
# input: [text] until this changes.
|
||||
# Text-only for now: the F16 projector is on disk at vision/mmproj-F16.gguf;
|
||||
# flip to 1 to enable it. The fleet's image path is frank-small either way,
|
||||
# so models.yml stays input: [text] until this changes.
|
||||
ENABLE_VISION=0
|
||||
|
||||
# PLE pager: the 52.4 GiB payload is mmap'd and paged by the engine's own
|
||||
# workers; this is the hot window kept resident. The vendor's production
|
||||
# profile default is 4096.
|
||||
PLE_CACHE_MIB=4096
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=CIRU v4.4.1 Orca Qwen3.8-Flash (dankclaw, frank-large, Strix Halo iGPU gfx1151)
|
||||
Description=unsloth llama.cpp (b11030, ROCm gfx1151) Qwen3.8-Flash-Next UD-IQ4_XS (dankclaw, frank-large, Strix Halo iGPU)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
|
||||
@@ -1,24 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca as frank-large on Dankclaw's
|
||||
# Strix Halo iGPU (gfx1151), using the CIRU v4.4.1 runtime with the QSA
|
||||
# window hotfix.
|
||||
# Serve unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS as frank-large on
|
||||
# Dankclaw's Strix Halo iGPU (gfx1151), using the unsloth llama.cpp
|
||||
# prebuilt (b11030-mix-5ff778e, ROCm gfx1151) — the first runtime with an
|
||||
# MTP graph for the qwen4exp architecture (stock ggml-org mainline has no
|
||||
# MTP graph, no cross-model tensor borrowing and no --spec-type draft-mtp
|
||||
# for this model).
|
||||
#
|
||||
# Runtime: the v4.4.1 source with the QSA window hotfix pre-applied
|
||||
# (ciru-runtime-v4.4.1-qsa-hotfix-source.tar.gz, shipped in the model repo
|
||||
# under runtime/v4.4.1/qsa-window-hotfix/), built ON the host against the
|
||||
# private ROCm 10.0.0 SDK that scripts/ciru/setup-linux-amd.sh installs into
|
||||
# the runtime checkout's own venv. The launcher below re-verifies on every
|
||||
# start that the built libraries carry the v4.4.1 corrections (it greps the
|
||||
# common and HIP libraries for the fix markers) before exec.
|
||||
# Runtime: the prebuilt app tarball from the unslothai/llama.cpp releases
|
||||
# (tag b11030-mix-5ff778e or newer), extracted under
|
||||
# /home/dank/GitPackages/unsloth-llama-b11030/. It bundles its own ROCm
|
||||
# runtime, so there is no host SDK to gate on and no venv to symlink.
|
||||
#
|
||||
# The launcher owns the flag list: -sm none (one iGPU, no split), -ngl all,
|
||||
# f16 KV, flash attention, cont-batching, the PLE sidecar, the MTP drafter
|
||||
# (depth 4 for the Orca variant) and the production kernel-bundle profile.
|
||||
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides it;
|
||||
# anything not set there falls through to the launcher's own defaults.
|
||||
# Shape: one stream, 200k context (operator's call, 2026-09-21), all layers
|
||||
# on the iGPU (-ngl 999). The 87 GiB model lives in the iGPU's GTT aperture
|
||||
# over the shared LPDDR (gtt_total ~114 GiB), the same regime the CIRU
|
||||
# Orca predecessor ran in. The MTP shared-Q8_0 head speculates (1.3-1.7x at
|
||||
# concurrency 1, per the model card).
|
||||
#
|
||||
# It replaces the halogen-flash-server quadlet, which served
|
||||
# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16.
|
||||
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
|
||||
# it; anything not set there falls through to the defaults below.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -31,64 +31,72 @@ if [[ -f "$ENV_FILE" ]]; then
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) ------------------
|
||||
: "${RUNTIME_DIR:=/home/dank/GitPackages/ciru-runtime-v4.4.1-qsa-hotfix}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca}"
|
||||
: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS}"
|
||||
: "${HOST:=0.0.0.0}"
|
||||
: "${PORT:=7770}"
|
||||
: "${CONTEXT_SIZE:=262144}"
|
||||
: "${CONTEXT_SIZE:=200000}"
|
||||
: "${PARALLEL_SLOTS:=1}"
|
||||
: "${ENABLE_MTP:=1}"
|
||||
: "${SPEC_DRAFT_N_MAX:=2}"
|
||||
: "${ENABLE_VISION:=0}"
|
||||
|
||||
LAUNCHER="$RUNTIME_DIR/scripts/ciru/run-orca-server.sh"
|
||||
SERVER_BIN="$RUNTIME_DIR/build-gfx1151-sdk/bin/llama-server"
|
||||
SERVER_BIN="$RUNTIME_DIR/llama-server"
|
||||
|
||||
if [[ ! -x "$LAUNCHER" ]]; then
|
||||
echo "missing runtime launcher: $LAUNCHER" >&2
|
||||
echo "fetch the hotfix source tarball from the model repo" >&2
|
||||
echo "(runtime/v4.4.1/qsa-window-hotfix/) and build it:" >&2
|
||||
echo " bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -x "$SERVER_BIN" ]]; then
|
||||
echo "missing built runtime: $SERVER_BIN" >&2
|
||||
echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
|
||||
echo "missing runtime: $SERVER_BIN" >&2
|
||||
echo "fetch and extract it:" >&2
|
||||
echo " curl -fL -o /tmp/unsloth-llama.tar.gz https://github.com/unslothai/llama.cpp/releases/download/b11030-mix-5ff778e/app-b11030-mix-5ff778e-linux-x64-rocm-gfx1151.tar.gz" >&2
|
||||
echo " mkdir -p $RUNTIME_DIR && tar xzf /tmp/unsloth-llama.tar.gz -C $RUNTIME_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The launcher gates on a runtime root holding hip/lib/libamdhip64.so and
|
||||
# rocr/lib/libhsa-runtime64.so (it prepends both to LD_LIBRARY_PATH). Our
|
||||
# build links the private SDK venv through its RUNPATH, so point those
|
||||
# entries at the venv's unversioned devel symlinks. Idempotent: ln -sfn.
|
||||
VENV_LIB="$(ls -d "$RUNTIME_DIR"/.venv-rocm/lib/python*/site-packages/_rocm_sdk_devel/lib 2>/dev/null | head -1 || true)"
|
||||
if [[ -z "$VENV_LIB" || ! -e "$VENV_LIB/libamdhip64.so" || ! -e "$VENV_LIB/libhsa-runtime64.so" ]]; then
|
||||
echo "missing SDK venv runtime under $RUNTIME_DIR/.venv-rocm (libamdhip64.so, libhsa-runtime64.so)" >&2
|
||||
echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p "$RUNTIME_DIR/runtime/hip/lib" "$RUNTIME_DIR/runtime/rocr/lib"
|
||||
ln -sfn "$VENV_LIB/libamdhip64.so" "$RUNTIME_DIR/runtime/hip/lib/libamdhip64.so"
|
||||
ln -sfn "$VENV_LIB/libhsa-runtime64.so" "$RUNTIME_DIR/runtime/rocr/lib/libhsa-runtime64.so"
|
||||
if [[ ! -r "$MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" ]]; then
|
||||
echo "missing target model: $MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" >&2
|
||||
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2
|
||||
MODEL_FILE="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf"
|
||||
if [[ ! -r "$MODEL_FILE" ]]; then
|
||||
echo "missing target model: $MODEL_FILE" >&2
|
||||
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The launcher checks the MTP head and the PLE sidecar itself, but a missing
|
||||
# payload should be a deploy error with a pointer, not a runtime exit 2.
|
||||
for required in \
|
||||
"$MODEL_DIR/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf" \
|
||||
"$MODEL_DIR/ple/ple.payload.bin" \
|
||||
"$MODEL_DIR/ple/ple.manifest.json" \
|
||||
"$MODEL_DIR/ple/ple.scale.bf16"; do
|
||||
if [[ ! -r "$required" ]]; then
|
||||
echo "missing payload: $required" >&2
|
||||
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
# The MTP shared head borrows the token embedding and the output projection
|
||||
# from the main model at runtime. It must be passed explicitly with -md:
|
||||
# sidecar auto-discovery does not search the mtp/ subfolder, so
|
||||
# --spec-type draft-mtp on its own would find nothing and serve at the
|
||||
# main model's speed with no error saying why.
|
||||
MTP_FILE="$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
|
||||
if [[ "$ENABLE_MTP" == "1" && ! -r "$MTP_FILE" ]]; then
|
||||
echo "missing MTP head: $MTP_FILE" >&2
|
||||
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The Qwen3.8 family template keeps prior thinking blocks across turns when
|
||||
# preserve_thinking is set; the launcher passes --jinja but no template
|
||||
# kwargs, so name them here. A request-level chat_template_kwargs still
|
||||
# wins over this.
|
||||
exec "$LAUNCHER" --chat-template-kwargs '{"preserve_thinking":true}'
|
||||
# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents
|
||||
# do not lose the trail. A request-level chat_template_kwargs still wins.
|
||||
# The template gates reasoning_effort to {xhigh, medium, low} — the harness
|
||||
# folds its levels onto that set in models.yml (reasoningEffortMap).
|
||||
ARGS=(
|
||||
-m "$MODEL_FILE"
|
||||
-c "$CONTEXT_SIZE"
|
||||
-np "$PARALLEL_SLOTS"
|
||||
-ngl 999
|
||||
--jinja
|
||||
--alias frank-large
|
||||
--host "$HOST"
|
||||
--port "$PORT"
|
||||
--chat-template-kwargs '{"preserve_thinking":true}'
|
||||
)
|
||||
if [[ "$ENABLE_MTP" == "1" ]]; then
|
||||
ARGS+=(-md "$MTP_FILE" --spec-type draft-mtp --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
||||
fi
|
||||
if [[ "$ENABLE_VISION" == "1" ]]; then
|
||||
MMPROJ_FILE="$MODEL_DIR/vision/mmproj-F16.gguf"
|
||||
if [[ ! -r "$MMPROJ_FILE" ]]; then
|
||||
echo "missing projector: $MMPROJ_FILE" >&2
|
||||
echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
ARGS+=(--mmproj "$MMPROJ_FILE")
|
||||
fi
|
||||
|
||||
exec "$SERVER_BIN" "${ARGS[@]}"
|
||||
|
||||
Reference in New Issue
Block a user