From b149d021877678d6d21cd4505359340416dd6651 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 21 Sep 2026 12:30:30 -0700 Subject: [PATCH] feat(frank-large): unsloth llama.cpp Qwen3.8-Flash-Next UD-IQ4_XS (200k/1 slot, MTP depth 2) Replace the CIRU v4.4.1 Orca runtime with the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm gfx1151, bundles its own ROCm) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (87.3 GiB, three HF split parts concatenated) on the Strix Halo iGPU. One conversation, 200,000 ctx, PARALLEL_SLOTS=1, MTP depth 2 via the explicit -md shared-Q8_0 head (sidecar auto-discovery does not search the mtp/ subfolder), text-only. Fleet contract unchanged: port 7770, alias frank-large. - serve-frank-large.sh: unsloth binary, -m/-md, --spec-type draft-mtp --spec-draft-n-max 2, --jinja, preserve_thinking; gates on binary, GGUF, MTP head; conditional --mmproj on ENABLE_VISION=1 - download-frank-large-unsloth.sh: resumable fetch of 3 parts + MTP head + mmproj, each verified against its LFS sha256 fetched live from the Hub tree API; concatenates parts, records/verifies SHA256SUMS - deploy-remote.sh: LLM_LARGE_MODEL/LLM_LARGE_DRAFT paths, pkill pattern on the model path (not the binary name), wait_port_free gate - tests: seams for 200k/1-slot, explicit -md, text-only, handler gates, sweep safety --- scripts/deploy-remote.sh | 31 +++--- scripts/download-frank-large-unsloth.sh | 118 ++++++++++++++++++++ scripts/frank-large.env | 57 +++++----- scripts/frank-large.service | 2 +- scripts/serve-frank-large.sh | 136 +++++++++++++----------- tests/frank_large_unit.bats | 85 +++++++-------- 6 files changed, 272 insertions(+), 157 deletions(-) create mode 100755 scripts/download-frank-large-unsloth.sh diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 466cbd2..115045e 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1028,22 +1028,20 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large is the CIRU v4.4.1 runtime (QSA window hotfix) serving -# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU, as a ROOT -# unit since the 2026-09-19 cutover: the runtime is a host build against -# the private ROCm 10 SDK, so there is no container image to install and no -# generated unit to dodge. relink_unit points +# frank-large is the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm +# gfx1151) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS on the Strix +# Halo iGPU, as a ROOT unit since the 2026-09-19 cutover: the runtime is a +# prebuilt tarball with its own bundled ROCm, so there is no container +# image to install and no generated unit to dodge. relink_unit points # /etc/systemd/system/frank-large.service at the repo file and boot # persistence is that unit's [Install] section. # -# It replaces the halogen-flash-server quadlet, which served -# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16. -# The quadlet is retired HERE (the unit file moves into the home trash, -# never `rm`), and the port gate is the same as the 2026-09-16 cutover: -# nothing starts until 7770 is released. -LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf}" -LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf}" -LLM_LARGE_PLE="${LLM_LARGE_PLE:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/ple/ple.payload.bin}" +# It replaces the CIRU v4.4.1 Orca runtime (jcbtc/Qwen3.8-Flash-CIRU- +# STRIX-Orca), which had replaced the halogen-flash-server quadlet. The +# port gate is the same as every cutover since 2026-09-16: nothing starts +# until 7770 is released. +LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf}" +LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf}" retire_halogen_large_quadlet() { local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" @@ -1080,10 +1078,10 @@ deploy_llm_large() { # missing never becomes healthy, and a green deploy would leave the # endpoint dark with nothing in the deploy log to say why. local missing=0 f - for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT" "$LLM_LARGE_PLE"; do + for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do if [[ ! -f "$f" ]]; then echo "ERROR: payload absent at $f" >&2 - echo "frank-large: fetch it: bash scripts/download-frank-large-orca.sh" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-unsloth.sh" >&2 missing=1 fi done @@ -1098,8 +1096,7 @@ deploy_llm_large() { # The pattern never appears in the deploying shell's own command line, so # -f is safe here (it is not in `bash deploy-remote.sh`). run_privileged systemctl stop frank-large.service 2>/dev/null || true - pkill -f 'Qwen3.8-Flash-CIRU-STRIX-Orca' 2>/dev/null || true - sleep 2 + pkill -f 'Qwen3.8-Flash-Next-UD-IQ4_XS' 2>/dev/null || true # Nothing may start until the predecessor has released the port. On # 2026-09-16 the cutover failed exactly here: the old server logged # "Deactivated successfully", the next engine tried to bind 7770 a second diff --git a/scripts/download-frank-large-unsloth.sh b/scripts/download-frank-large-unsloth.sh new file mode 100755 index 0000000..ea68d2e --- /dev/null +++ b/scripts/download-frank-large-unsloth.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Download and assemble unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (the +# frank-large model) on Dankclaw. +# +# bash scripts/download-frank-large-unsloth.sh +# bash scripts/download-frank-large-unsloth.sh --verify-only +# +# The UD-IQ4_XS GGUF ships as three llama.cpp split parts (HF caps uploads at +# 50 GB); concatenating them in numeric order reconstructs the single GGUF. +# The MTP shared-Q8_0 head borrows the token embedding and the output +# projection from the main model at runtime (it is the 2.6 GB remainder of +# the 4B MTP layer). The F16 projector is fetched for a future vision flip; +# the serve script does not load it while ENABLE_VISION=0. +# +# Integrity: every part is checked against its LFS sha256 (fetched live from +# the Hub tree API, not trusted from disk), then the assembled GGUF against +# the recorded SHA256SUMS. The repo is public (gated: false); an HF token at +# ~/.cache/huggingface/token is used when present to clear the anonymous rate +# limit on a 94 GB transfer. +# +# Layout after a successful run (MODEL_DIR): +# Qwen3.8-Flash-Next-UD-IQ4_XS.gguf 93,682,584,224 main model +# mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf 2,786,568,256 draft head +# vision/mmproj-F16.gguf 904,004,000 projector +# parts/ the three verified transport parts (kept for --verify-only) +# SHA256SUMS +set -euo pipefail + +REPO="unsloth/Qwen3.8-Flash-Next-GGUF" +MODEL_DIR="/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS" +BASE="https://huggingface.co/$REPO/resolve/main" + +mode="download" +[[ "${1:-}" == "--verify-only" ]] && mode="verify" + +token="" +auth=() +if [[ -r "$HOME/.cache/huggingface/token" ]]; then + token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")" + auth=(-H "Authorization: Bearer $token") +fi + +dl() { # dl + local rel="$1" dest="$MODEL_DIR/$rel" + [[ -s "$dest" ]] && return 0 + mkdir -p "$(dirname "$dest")" + echo "fetching $rel" + curl -fL -C - "${auth[@]}" --retry 5 --retry-delay 3 -o "$dest" "$BASE/$rel" +} + +# relpath size sha256 — the LFS oids from the Hub tree API (2026-09-21) +FILES=( + "UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf 10946624 5ce89370720f8bf90890f439361282104c1aa1482d4013bb9a50923e758e71a4" + "UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00002-of-00003.gguf 49835229856 577a38a2392b40ca2193cea502e1d92f60b8cd370675d308e0ec21885d9daaa7" + "UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00003-of-00003.gguf 43836407744 d4634e6d84f0ebb0940be15c90d3790bf6464e3dea3a1cddc567dc0e83ad8833" + "MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf 2786568256 5ff54097406a905cf3a724c709124ceb0e3e10235ee862298969e91c96fa96e6" + "mmproj-F16.gguf 904004000 1f7b7f0b984cf065c604360c29c8098362ed61b290db0ff12c6f360bb1a8a980" +) + +if [[ "$mode" == "download" ]]; then + for line in "${FILES[@]}"; do + read -r rel size _ <<<"$line" + dl "$rel" + done +fi + +echo "verifying parts against the live Hub tree API" +python3 - "$MODEL_DIR" "$REPO" <<'PY' +import hashlib, json, os, sys, urllib.request + +model_dir, repo = sys.argv[1], sys.argv[2] +tree = json.load(urllib.request.urlopen( + f"https://huggingface.co/api/models/{repo}/tree/main?recursive=true")) +want = {} +for f in tree: + if f["type"] == "file" and f.get("lfs"): + want[f["path"]] = (f["lfs"]["size"], f["lfs"]["oid"]) + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 24), b""): + h.update(chunk) + return h.hexdigest() + +ok = True +for rel, (size, oid) in sorted(want.items()): + if not rel.startswith(("UD-IQ4_XS/", "MTP/")) and rel != "mmproj-F16.gguf": + continue + path = os.path.join(model_dir, rel) + if not os.path.isfile(path): + print(f"MISSING {rel}"); ok = False; continue + actual_size = os.path.getsize(path) + if actual_size != size: + print(f"SIZE MISMATCH {rel}: {actual_size} != {size}"); ok = False; continue + digest = sha256(path) + if digest != oid: + print(f"SHA256 MISMATCH {rel}"); ok = False; continue + print(f"ok {rel}") +sys.exit(0 if ok else 1) +PY + +FINAL="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf" +if [[ "$mode" == "download" ]]; then + echo "assembling the GGUF from the split parts" + cat "$MODEL_DIR"/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-0000{1,2,3}-of-00003.gguf > "$FINAL" + [[ "$(stat -c %s "$FINAL")" -eq 93682584224 ]] || { + echo "assembled size wrong: $(stat -c %s "$FINAL")" >&2; exit 1; } + mkdir -p "$MODEL_DIR/mtp" "$MODEL_DIR/vision" + [[ -e "$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" ]] || + mv "$MODEL_DIR/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" "$MODEL_DIR/mtp/" + [[ -e "$MODEL_DIR/vision/mmproj-F16.gguf" ]] || mv "$MODEL_DIR/mmproj-F16.gguf" "$MODEL_DIR/vision/" + (cd "$MODEL_DIR" && sha256sum Qwen3.8-Flash-Next-UD-IQ4_XS.gguf > SHA256SUMS) +fi + +echo "verifying assembled files against SHA256SUMS" +(cd "$MODEL_DIR" && sha256sum -c SHA256SUMS) +echo "frank-large payload complete: $MODEL_DIR" diff --git a/scripts/frank-large.env b/scripts/frank-large.env index 562f6c0..203dafb 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -1,44 +1,39 @@ -# frank-large — CIRU v4.4.1 (QSA window hotfix) serving -# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU (gfx1151). +# frank-large — unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm +# gfx1151) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS on the Strix +# Halo iGPU. # -# These are the launcher tunables read by the runtime's -# scripts/ciru/run-orca-server.sh (see the serve script). The 36-variable -# kernel bundle and the PLE pager defaults live in the runtime's own -# profiles/strix-halo-production.env, which the launcher sources AFTER this -# file; every profile line is a ${VAR:-default}, so a value set here wins. +# These are the tunables read by scripts/serve-frank-large.sh. The flag +# list is owned by the serve script; this file only sets fleet values. # -# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP runtime -# reads those bare (the 2026-08-13 TEMP incident). HOST is the one exception -# because the launcher itself consumes it for --host; its value doubles as a -# JIT-cache directory name, which is why 0.0.0.0 is safe. +# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP +# runtime reads those bare (the 2026-08-13 TEMP incident). HOST is the one +# exception because the serve script itself consumes it for --host. -# The assembled payload (scripts/download-frank-large-orca.sh): the launcher -# refuses to start without the main GGUF, the MTP head and the PLE sidecar. -MODEL_DIR=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca +# The assembled payload (scripts/download-frank-large-unsloth.sh): the +# serve script refuses to start without the main GGUF and the MTP head. +MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS -# Fleet endpoint: the same port and alias the halogen predecessor served, so +# The prebuilt runtime (see the serve script for the fetch line). +RUNTIME_DIR=/home/dank/GitPackages/unsloth-llama-b11030 + +# Fleet endpoint: the same port and alias the CIRU predecessor served, so # no client, probe or models.yml entry changes address. HOST=0.0.0.0 PORT=7770 -# One stream, full context (operator call, 2026-09-19). MTP requires exactly -# one slot in this runtime, so the drafter's constraint and the operator's -# ask are the same number. 262144 is the model's trained window; the GGUF -# carries no rope scaling. -CONTEXT_SIZE=262144 +# One stream, 200k context (operator's call, 2026-09-21). The model's +# trained window is 262144; 200000 is the requested serving shape. +CONTEXT_SIZE=200000 PARALLEL_SLOTS=1 -# MTP head: on. Depth 4 is the Orca variant's own default (mtp_depth_default -# in the launcher); the sidecar is the Q8_0 MTP file from the model repo. +# MTP shared-Q8_0 head: on. Draft depth 2 is the model card's default +# ("higher drafts more but each guess is accepted less often"). The shared +# head borrows the token embedding and output projection from the main +# model, so the file is 2.6 GB rather than 4.1. ENABLE_MTP=1 +SPEC_DRAFT_N_MAX=2 -# Text-only for now: the F16 projector is on disk at -# vision/mmproj-Qwen3.8-Flash-Orca-F16.mmproj; flip to 1 to enable it. The -# fleet's image path is frank-small either way, so models.yml stays -# input: [text] until this changes. +# Text-only for now: the F16 projector is on disk at vision/mmproj-F16.gguf; +# flip to 1 to enable it. The fleet's image path is frank-small either way, +# so models.yml stays input: [text] until this changes. ENABLE_VISION=0 - -# PLE pager: the 52.4 GiB payload is mmap'd and paged by the engine's own -# workers; this is the hot window kept resident. The vendor's production -# profile default is 4096. -PLE_CACHE_MIB=4096 diff --git a/scripts/frank-large.service b/scripts/frank-large.service index 9f76cb6..4e08322 100644 --- a/scripts/frank-large.service +++ b/scripts/frank-large.service @@ -1,5 +1,5 @@ [Unit] -Description=CIRU v4.4.1 Orca Qwen3.8-Flash (dankclaw, frank-large, Strix Halo iGPU gfx1151) +Description=unsloth llama.cpp (b11030, ROCm gfx1151) Qwen3.8-Flash-Next UD-IQ4_XS (dankclaw, frank-large, Strix Halo iGPU) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index aa56899..0fbe09d 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,24 +1,24 @@ #!/usr/bin/env bash -# Serve jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca as frank-large on Dankclaw's -# Strix Halo iGPU (gfx1151), using the CIRU v4.4.1 runtime with the QSA -# window hotfix. +# Serve unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS as frank-large on +# Dankclaw's Strix Halo iGPU (gfx1151), using the unsloth llama.cpp +# prebuilt (b11030-mix-5ff778e, ROCm gfx1151) — the first runtime with an +# MTP graph for the qwen4exp architecture (stock ggml-org mainline has no +# MTP graph, no cross-model tensor borrowing and no --spec-type draft-mtp +# for this model). # -# Runtime: the v4.4.1 source with the QSA window hotfix pre-applied -# (ciru-runtime-v4.4.1-qsa-hotfix-source.tar.gz, shipped in the model repo -# under runtime/v4.4.1/qsa-window-hotfix/), built ON the host against the -# private ROCm 10.0.0 SDK that scripts/ciru/setup-linux-amd.sh installs into -# the runtime checkout's own venv. The launcher below re-verifies on every -# start that the built libraries carry the v4.4.1 corrections (it greps the -# common and HIP libraries for the fix markers) before exec. +# Runtime: the prebuilt app tarball from the unslothai/llama.cpp releases +# (tag b11030-mix-5ff778e or newer), extracted under +# /home/dank/GitPackages/unsloth-llama-b11030/. It bundles its own ROCm +# runtime, so there is no host SDK to gate on and no venv to symlink. # -# The launcher owns the flag list: -sm none (one iGPU, no split), -ngl all, -# f16 KV, flash attention, cont-batching, the PLE sidecar, the MTP drafter -# (depth 4 for the Orca variant) and the production kernel-bundle profile. -# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides it; -# anything not set there falls through to the launcher's own defaults. +# Shape: one stream, 200k context (operator's call, 2026-09-21), all layers +# on the iGPU (-ngl 999). The 87 GiB model lives in the iGPU's GTT aperture +# over the shared LPDDR (gtt_total ~114 GiB), the same regime the CIRU +# Orca predecessor ran in. The MTP shared-Q8_0 head speculates (1.3-1.7x at +# concurrency 1, per the model card). # -# It replaces the halogen-flash-server quadlet, which served -# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16. +# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides +# it; anything not set there falls through to the defaults below. set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" @@ -31,64 +31,72 @@ if [[ -f "$ENV_FILE" ]]; then fi # --- defaults (used when the env file does not set them) ------------------ -: "${RUNTIME_DIR:=/home/dank/GitPackages/ciru-runtime-v4.4.1-qsa-hotfix}" -: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca}" +: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}" +: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS}" : "${HOST:=0.0.0.0}" : "${PORT:=7770}" -: "${CONTEXT_SIZE:=262144}" +: "${CONTEXT_SIZE:=200000}" : "${PARALLEL_SLOTS:=1}" +: "${ENABLE_MTP:=1}" +: "${SPEC_DRAFT_N_MAX:=2}" +: "${ENABLE_VISION:=0}" -LAUNCHER="$RUNTIME_DIR/scripts/ciru/run-orca-server.sh" -SERVER_BIN="$RUNTIME_DIR/build-gfx1151-sdk/bin/llama-server" +SERVER_BIN="$RUNTIME_DIR/llama-server" -if [[ ! -x "$LAUNCHER" ]]; then - echo "missing runtime launcher: $LAUNCHER" >&2 - echo "fetch the hotfix source tarball from the model repo" >&2 - echo "(runtime/v4.4.1/qsa-window-hotfix/) and build it:" >&2 - echo " bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2 - exit 1 -fi if [[ ! -x "$SERVER_BIN" ]]; then - echo "missing built runtime: $SERVER_BIN" >&2 - echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2 + echo "missing runtime: $SERVER_BIN" >&2 + echo "fetch and extract it:" >&2 + echo " curl -fL -o /tmp/unsloth-llama.tar.gz https://github.com/unslothai/llama.cpp/releases/download/b11030-mix-5ff778e/app-b11030-mix-5ff778e-linux-x64-rocm-gfx1151.tar.gz" >&2 + echo " mkdir -p $RUNTIME_DIR && tar xzf /tmp/unsloth-llama.tar.gz -C $RUNTIME_DIR" >&2 exit 1 fi -# The launcher gates on a runtime root holding hip/lib/libamdhip64.so and -# rocr/lib/libhsa-runtime64.so (it prepends both to LD_LIBRARY_PATH). Our -# build links the private SDK venv through its RUNPATH, so point those -# entries at the venv's unversioned devel symlinks. Idempotent: ln -sfn. -VENV_LIB="$(ls -d "$RUNTIME_DIR"/.venv-rocm/lib/python*/site-packages/_rocm_sdk_devel/lib 2>/dev/null | head -1 || true)" -if [[ -z "$VENV_LIB" || ! -e "$VENV_LIB/libamdhip64.so" || ! -e "$VENV_LIB/libhsa-runtime64.so" ]]; then - echo "missing SDK venv runtime under $RUNTIME_DIR/.venv-rocm (libamdhip64.so, libhsa-runtime64.so)" >&2 - echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2 - exit 1 -fi -mkdir -p "$RUNTIME_DIR/runtime/hip/lib" "$RUNTIME_DIR/runtime/rocr/lib" -ln -sfn "$VENV_LIB/libamdhip64.so" "$RUNTIME_DIR/runtime/hip/lib/libamdhip64.so" -ln -sfn "$VENV_LIB/libhsa-runtime64.so" "$RUNTIME_DIR/runtime/rocr/lib/libhsa-runtime64.so" -if [[ ! -r "$MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" ]]; then - echo "missing target model: $MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" >&2 - echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2 +MODEL_FILE="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf" +if [[ ! -r "$MODEL_FILE" ]]; then + echo "missing target model: $MODEL_FILE" >&2 + echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2 exit 1 fi -# The launcher checks the MTP head and the PLE sidecar itself, but a missing -# payload should be a deploy error with a pointer, not a runtime exit 2. -for required in \ - "$MODEL_DIR/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf" \ - "$MODEL_DIR/ple/ple.payload.bin" \ - "$MODEL_DIR/ple/ple.manifest.json" \ - "$MODEL_DIR/ple/ple.scale.bf16"; do - if [[ ! -r "$required" ]]; then - echo "missing payload: $required" >&2 - echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2 - exit 1 - fi -done +# The MTP shared head borrows the token embedding and the output projection +# from the main model at runtime. It must be passed explicitly with -md: +# sidecar auto-discovery does not search the mtp/ subfolder, so +# --spec-type draft-mtp on its own would find nothing and serve at the +# main model's speed with no error saying why. +MTP_FILE="$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" +if [[ "$ENABLE_MTP" == "1" && ! -r "$MTP_FILE" ]]; then + echo "missing MTP head: $MTP_FILE" >&2 + echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2 + exit 1 +fi # The Qwen3.8 family template keeps prior thinking blocks across turns when -# preserve_thinking is set; the launcher passes --jinja but no template -# kwargs, so name them here. A request-level chat_template_kwargs still -# wins over this. -exec "$LAUNCHER" --chat-template-kwargs '{"preserve_thinking":true}' +# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents +# do not lose the trail. A request-level chat_template_kwargs still wins. +# The template gates reasoning_effort to {xhigh, medium, low} — the harness +# folds its levels onto that set in models.yml (reasoningEffortMap). +ARGS=( + -m "$MODEL_FILE" + -c "$CONTEXT_SIZE" + -np "$PARALLEL_SLOTS" + -ngl 999 + --jinja + --alias frank-large + --host "$HOST" + --port "$PORT" + --chat-template-kwargs '{"preserve_thinking":true}' +) +if [[ "$ENABLE_MTP" == "1" ]]; then + ARGS+=(-md "$MTP_FILE" --spec-type draft-mtp --spec-draft-n-max "$SPEC_DRAFT_N_MAX") +fi +if [[ "$ENABLE_VISION" == "1" ]]; then + MMPROJ_FILE="$MODEL_DIR/vision/mmproj-F16.gguf" + if [[ ! -r "$MMPROJ_FILE" ]]; then + echo "missing projector: $MMPROJ_FILE" >&2 + echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2 + exit 1 + fi + ARGS+=(--mmproj "$MMPROJ_FILE") +fi + +exec "$SERVER_BIN" "${ARGS[@]}" diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 5f8ca54..aaa8f18 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,23 +1,24 @@ #!/usr/bin/env bats -# Seam: the frank-large CIRU v4.4.1 root unit on dankclaw. +# Seam: the frank-large unsloth-llama.cpp root unit on dankclaw. # -# The 2026-09-19 cutover moved this service out of the halogen-flash-server -# quadlet and onto the CIRU v4.4.1 runtime (QSA window hotfix) serving -# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca, as a ROOT unit on a host build -- the -# same shape as frank-small and frank-medium, because a private-SDK host -# binary has no container image to install and no generated unit to dodge. -# This seam pins what a flag list cannot show: the operator's one-stream / -# full-context shape, the payload the launcher refuses to start without, a -# deploy handler that retires the halogen quadlet without deleting it and -# gates on the port, and the fact that the sweep for a stray benchmark run -# cannot take frank-small down with it. +# The 2026-09-21 cutover moved this service from the CIRU v4.4.1 runtime +# (jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca) onto the unsloth llama.cpp prebuilt +# (b11030-mix-5ff778e, ROCm gfx1151) serving +# unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS, still as a ROOT unit: a +# prebuilt tarball with its own bundled ROCm has no container image to +# install and no generated unit to dodge. This seam pins what a flag list +# cannot show: the operator's one-stream / 200k shape, the payload the +# serve script refuses to start without, the explicit -md MTP wiring +# (sidecar auto-discovery does not search the mtp/ subfolder), a deploy +# handler that gates on the port, and the fact that the sweep for a stray +# benchmark run cannot take frank-small down with it. load helpers ENV_FILE="$REPO_ROOT/scripts/frank-large.env" UNIT="$REPO_ROOT/scripts/frank-large.service" SERVE="$REPO_ROOT/scripts/serve-frank-large.sh" -FETCH="$REPO_ROOT/scripts/download-frank-large-orca.sh" +FETCH="$REPO_ROOT/scripts/download-frank-large-unsloth.sh" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" @test "the frank-large unit, env and serve files exist where the deployer installs them from" { @@ -31,23 +32,22 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" grep -q '^Restart=on-failure$' "$UNIT" grep -q '^RestartSec=' "$UNIT" grep -q '^User=dank$' "$UNIT" - # The engine pins staging buffers and the PLE pager's hot window; the - # memlock budget must be unbounded or the load fails on the RLIMIT. + # The engine pins staging buffers; the memlock budget must be unbounded + # or the load fails on the RLIMIT. grep -q '^LimitMEMLOCK=infinity$' "$UNIT" # The unit runs the repo's own serve script with the repo's env. grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh' "$UNIT" grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT" } -@test "the operator's shape is one stream of the full trained window" { +@test "the operator's shape is one stream of 200k context with MTP on" { local ctx slots ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" slots="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" - # 262144 is the model's trained window and the GGUF carries no rope - # scaling; the operator asked for full context. - [ "$ctx" -eq 262144 ] - # One slot. This is also the drafter's constraint: the launcher refuses - # MTP with more than one slot, so the two cannot drift apart silently. + # 200000 is the operator's requested serving shape (2026-09-21); the + # model's trained window is 262144. + [ "$ctx" -eq 200000 ] + # One stream. [ "$slots" -eq 1 ] grep -q '^ENABLE_MTP=1$' "$ENV_FILE" } @@ -59,44 +59,41 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" grep -q -- '--mmproj' "$REPO_ROOT/scripts/serve-frank-small.sh" } -@test "the serve script pins the hotfix runtime and refuses a missing build" { - grep -q 'ciru-runtime-v4.4.1-qsa-hotfix' "$SERVE" - grep -q 'build-gfx1151-sdk/bin/llama-server' "$SERVE" - grep -q 'run-orca-server.sh' "$SERVE" +@test "the serve script pins the prebuilt runtime and refuses a missing build" { + grep -q 'unsloth-llama-b11030' "$SERVE" + grep -q 'llama-server' "$SERVE" # A missing runtime or payload is a deploy error with a pointer, not a # restart loop. - grep -q 'missing built runtime' "$SERVE" + grep -q 'missing runtime' "$SERVE" grep -q 'missing target model' "$SERVE" + grep -q 'missing MTP head' "$SERVE" # The Qwen3.8 template keeps prior thinking blocks across turns only - # when the kwarg is set; the launcher passes --jinja but no kwargs. + # when the kwarg is set. grep -q 'preserve_thinking' "$SERVE" } -@test "the serve script feeds the launcher its runtime root from the SDK venv" { - # The launcher exits 2 without hip/lib/libamdhip64.so + - # rocr/lib/libhsa-runtime64.so under its runtime root; the build links - # the private venv, so the serve script must point those entries there - # or the unit crash-loops on the gate (seen 2026-09-19). - grep -q '_rocm_sdk_devel/lib' "$SERVE" - grep -q 'runtime/hip/lib/libamdhip64.so' "$SERVE" - grep -q 'runtime/rocr/lib/libhsa-runtime64.so' "$SERVE" - grep -q 'missing SDK venv runtime' "$SERVE" +@test "the serve script wires the MTP head explicitly and speculates" { + # Sidecar auto-discovery does not search the mtp/ subfolder, so the head + # must be passed with -md or the server runs at the main model's speed + # with no error saying why. + grep -q -- '-md "$MTP_FILE"' "$SERVE" + grep -q -- '--spec-type draft-mtp' "$SERVE" + grep -q 'mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf' "$SERVE" } -@test "the payload is fetched resumably and verified by sha256" { - grep -q 'jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca' "$FETCH" +@test "the payload is fetched resumably and verified against the live Hub tree" { + grep -q 'unsloth/Qwen3.8-Flash-Next-GGUF' "$FETCH" grep -q -- '-fL -C -' "$FETCH" - grep -q 'parts.json' "$FETCH" + # The LFS oids come from the Hub tree API at verify time, not from a + # file that could drift. + grep -q 'tree/main' "$FETCH" grep -q 'sha256sum -c SHA256SUMS' "$FETCH" - # The PLE sidecar is the largest transport part; the script must fetch - # and verify it, not just the GGUF. - grep -q 'ple.payload.bin' "$FETCH" } @test "the fleet endpoint and alias survive the cutover" { grep -q '^HOST=0.0.0.0$' "$ENV_FILE" grep -q '^PORT=7770$' "$ENV_FILE" - grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca$' "$ENV_FILE" + grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS$' "$ENV_FILE" } @test "the deploy handler retires the halogen quadlet without deleting it" { @@ -111,14 +108,14 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY" grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY" - grep -q 'LLM_LARGE_PLE' "$REMOTE_DEPLOY" + grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY" } @test "the stray-run sweep cannot take frank-small down" { # frank-small on this host is ALSO a llama-server. The sweep must match # the model path, not the binary name, or a frank-large deploy restarts # the 3090 endpoint. - grep -q "pkill -f 'Qwen3.8-Flash-CIRU-STRIX-Orca'" "$REMOTE_DEPLOY" + grep -q "pkill -f 'Qwen3.8-Flash-Next-UD-IQ4_XS'" "$REMOTE_DEPLOY" # ...and only inside the large handler: extract it and check there is no # bare binary-name pkill in it. local large