Frank
b149d02187
Replace the CIRU v4.4.1 Orca runtime with the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm gfx1151, bundles its own ROCm) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (87.3 GiB, three HF split parts concatenated) on the Strix Halo iGPU. One conversation, 200,000 ctx, PARALLEL_SLOTS=1, MTP depth 2 via the explicit -md shared-Q8_0 head (sidecar auto-discovery does not search the mtp/ subfolder), text-only. Fleet contract unchanged: port 7770, alias frank-large. - serve-frank-large.sh: unsloth binary, -m/-md, --spec-type draft-mtp --spec-draft-n-max 2, --jinja, preserve_thinking; gates on binary, GGUF, MTP head; conditional --mmproj on ENABLE_VISION=1 - download-frank-large-unsloth.sh: resumable fetch of 3 parts + MTP head + mmproj, each verified against its LFS sha256 fetched live from the Hub tree API; concatenates parts, records/verifies SHA256SUMS - deploy-remote.sh: LLM_LARGE_MODEL/LLM_LARGE_DRAFT paths, pkill pattern on the model path (not the binary name), wait_port_free gate - tests: seams for 200k/1-slot, explicit -md, text-only, handler gates, sweep safety
119 lines
4.9 KiB
Bash
Executable File
119 lines
4.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Download and assemble unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (the
|
|
# frank-large model) on Dankclaw.
|
|
#
|
|
# bash scripts/download-frank-large-unsloth.sh
|
|
# bash scripts/download-frank-large-unsloth.sh --verify-only
|
|
#
|
|
# The UD-IQ4_XS GGUF ships as three llama.cpp split parts (HF caps uploads at
|
|
# 50 GB); concatenating them in numeric order reconstructs the single GGUF.
|
|
# The MTP shared-Q8_0 head borrows the token embedding and the output
|
|
# projection from the main model at runtime (it is the 2.6 GB remainder of
|
|
# the 4B MTP layer). The F16 projector is fetched for a future vision flip;
|
|
# the serve script does not load it while ENABLE_VISION=0.
|
|
#
|
|
# Integrity: every part is checked against its LFS sha256 (fetched live from
|
|
# the Hub tree API, not trusted from disk), then the assembled GGUF against
|
|
# the recorded SHA256SUMS. The repo is public (gated: false); an HF token at
|
|
# ~/.cache/huggingface/token is used when present to clear the anonymous rate
|
|
# limit on a 94 GB transfer.
|
|
#
|
|
# Layout after a successful run (MODEL_DIR):
|
|
# Qwen3.8-Flash-Next-UD-IQ4_XS.gguf 93,682,584,224 main model
|
|
# mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf 2,786,568,256 draft head
|
|
# vision/mmproj-F16.gguf 904,004,000 projector
|
|
# parts/ the three verified transport parts (kept for --verify-only)
|
|
# SHA256SUMS
|
|
set -euo pipefail
|
|
|
|
REPO="unsloth/Qwen3.8-Flash-Next-GGUF"
|
|
MODEL_DIR="/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS"
|
|
BASE="https://huggingface.co/$REPO/resolve/main"
|
|
|
|
mode="download"
|
|
[[ "${1:-}" == "--verify-only" ]] && mode="verify"
|
|
|
|
token=""
|
|
auth=()
|
|
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
|
|
token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")"
|
|
auth=(-H "Authorization: Bearer $token")
|
|
fi
|
|
|
|
dl() { # dl <relpath>
|
|
local rel="$1" dest="$MODEL_DIR/$rel"
|
|
[[ -s "$dest" ]] && return 0
|
|
mkdir -p "$(dirname "$dest")"
|
|
echo "fetching $rel"
|
|
curl -fL -C - "${auth[@]}" --retry 5 --retry-delay 3 -o "$dest" "$BASE/$rel"
|
|
}
|
|
|
|
# relpath size sha256 — the LFS oids from the Hub tree API (2026-09-21)
|
|
FILES=(
|
|
"UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf 10946624 5ce89370720f8bf90890f439361282104c1aa1482d4013bb9a50923e758e71a4"
|
|
"UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00002-of-00003.gguf 49835229856 577a38a2392b40ca2193cea502e1d92f60b8cd370675d308e0ec21885d9daaa7"
|
|
"UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00003-of-00003.gguf 43836407744 d4634e6d84f0ebb0940be15c90d3790bf6464e3dea3a1cddc567dc0e83ad8833"
|
|
"MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf 2786568256 5ff54097406a905cf3a724c709124ceb0e3e10235ee862298969e91c96fa96e6"
|
|
"mmproj-F16.gguf 904004000 1f7b7f0b984cf065c604360c29c8098362ed61b290db0ff12c6f360bb1a8a980"
|
|
)
|
|
|
|
if [[ "$mode" == "download" ]]; then
|
|
for line in "${FILES[@]}"; do
|
|
read -r rel size _ <<<"$line"
|
|
dl "$rel"
|
|
done
|
|
fi
|
|
|
|
echo "verifying parts against the live Hub tree API"
|
|
python3 - "$MODEL_DIR" "$REPO" <<'PY'
|
|
import hashlib, json, os, sys, urllib.request
|
|
|
|
model_dir, repo = sys.argv[1], sys.argv[2]
|
|
tree = json.load(urllib.request.urlopen(
|
|
f"https://huggingface.co/api/models/{repo}/tree/main?recursive=true"))
|
|
want = {}
|
|
for f in tree:
|
|
if f["type"] == "file" and f.get("lfs"):
|
|
want[f["path"]] = (f["lfs"]["size"], f["lfs"]["oid"])
|
|
|
|
def sha256(path):
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as fh:
|
|
for chunk in iter(lambda: fh.read(1 << 24), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
ok = True
|
|
for rel, (size, oid) in sorted(want.items()):
|
|
if not rel.startswith(("UD-IQ4_XS/", "MTP/")) and rel != "mmproj-F16.gguf":
|
|
continue
|
|
path = os.path.join(model_dir, rel)
|
|
if not os.path.isfile(path):
|
|
print(f"MISSING {rel}"); ok = False; continue
|
|
actual_size = os.path.getsize(path)
|
|
if actual_size != size:
|
|
print(f"SIZE MISMATCH {rel}: {actual_size} != {size}"); ok = False; continue
|
|
digest = sha256(path)
|
|
if digest != oid:
|
|
print(f"SHA256 MISMATCH {rel}"); ok = False; continue
|
|
print(f"ok {rel}")
|
|
sys.exit(0 if ok else 1)
|
|
PY
|
|
|
|
FINAL="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf"
|
|
if [[ "$mode" == "download" ]]; then
|
|
echo "assembling the GGUF from the split parts"
|
|
cat "$MODEL_DIR"/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-0000{1,2,3}-of-00003.gguf > "$FINAL"
|
|
[[ "$(stat -c %s "$FINAL")" -eq 93682584224 ]] || {
|
|
echo "assembled size wrong: $(stat -c %s "$FINAL")" >&2; exit 1; }
|
|
mkdir -p "$MODEL_DIR/mtp" "$MODEL_DIR/vision"
|
|
[[ -e "$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" ]] ||
|
|
mv "$MODEL_DIR/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" "$MODEL_DIR/mtp/"
|
|
[[ -e "$MODEL_DIR/vision/mmproj-F16.gguf" ]] || mv "$MODEL_DIR/mmproj-F16.gguf" "$MODEL_DIR/vision/"
|
|
(cd "$MODEL_DIR" && sha256sum Qwen3.8-Flash-Next-UD-IQ4_XS.gguf > SHA256SUMS)
|
|
fi
|
|
|
|
echo "verifying assembled files against SHA256SUMS"
|
|
(cd "$MODEL_DIR" && sha256sum -c SHA256SUMS)
|
|
echo "frank-large payload complete: $MODEL_DIR"
|