feat(frank-large): unsloth llama.cpp Qwen3.8-Flash-Next UD-IQ4_XS (200k/1 slot, MTP depth 2)

Replace the CIRU v4.4.1 Orca runtime with the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm gfx1151, bundles its own ROCm) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (87.3 GiB, three HF split parts concatenated) on the Strix Halo iGPU. One conversation, 200,000 ctx, PARALLEL_SLOTS=1, MTP depth 2 via the explicit -md shared-Q8_0 head (sidecar auto-discovery does not search the mtp/ subfolder), text-only. Fleet contract unchanged: port 7770, alias frank-large. - serve-frank-large.sh: unsloth binary, -m/-md, --spec-type draft-mtp --spec-draft-n-max 2, --jinja, preserve_thinking; gates on binary, GGUF, MTP head; conditional --mmproj on ENABLE_VISION=1 - download-frank-large-unsloth.sh: resumable fetch of 3 parts + MTP head + mmproj, each verified against its LFS sha256 fetched live from the Hub tree API; concatenates parts, records/verifies SHA256SUMS - deploy-remote.sh: LLM_LARGE_MODEL/LLM_LARGE_DRAFT paths, pkill pattern on the model path (not the binary name), wait_port_free gate - tests: seams for 200k/1-slot, explicit -md, text-only, handler gates, sweep safety
2026-09-21 12:30:30 -07:00
1 parent e5a1f8e767
commit b149d02187
@@ -1,24 +1,24 @@
 #!/usr/bin/env bash
-# Serve jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca as frank-large on Dankclaw's
-# Strix Halo iGPU (gfx1151), using the CIRU v4.4.1 runtime with the QSA
-# window hotfix.
+# Serve unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS as frank-large on
+# Dankclaw's Strix Halo iGPU (gfx1151), using the unsloth llama.cpp
+# prebuilt (b11030-mix-5ff778e, ROCm gfx1151) — the first runtime with an
+# MTP graph for the qwen4exp architecture (stock ggml-org mainline has no
+# MTP graph, no cross-model tensor borrowing and no --spec-type draft-mtp
+# for this model).
 #
-# Runtime: the v4.4.1 source with the QSA window hotfix pre-applied
-# (ciru-runtime-v4.4.1-qsa-hotfix-source.tar.gz, shipped in the model repo
-# under runtime/v4.4.1/qsa-window-hotfix/), built ON the host against the
-# private ROCm 10.0.0 SDK that scripts/ciru/setup-linux-amd.sh installs into
-# the runtime checkout's own venv. The launcher below re-verifies on every
-# start that the built libraries carry the v4.4.1 corrections (it greps the
-# common and HIP libraries for the fix markers) before exec.
+# Runtime: the prebuilt app tarball from the unslothai/llama.cpp releases
+# (tag b11030-mix-5ff778e or newer), extracted under
+# /home/dank/GitPackages/unsloth-llama-b11030/. It bundles its own ROCm
+# runtime, so there is no host SDK to gate on and no venv to symlink.
 #
-# The launcher owns the flag list: -sm none (one iGPU, no split), -ngl all,
-# f16 KV, flash attention, cont-batching, the PLE sidecar, the MTP drafter
-# (depth 4 for the Orca variant) and the production kernel-bundle profile.
-# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides it;
-# anything not set there falls through to the launcher's own defaults.
+# Shape: one stream, 200k context (operator's call, 2026-09-21), all layers
+# on the iGPU (-ngl 999). The 87 GiB model lives in the iGPU's GTT aperture
+# over the shared LPDDR (gtt_total ~114 GiB), the same regime the CIRU
+# Orca predecessor ran in. The MTP shared-Q8_0 head speculates (1.3-1.7x at
+# concurrency 1, per the model card).
 #
-# It replaces the halogen-flash-server quadlet, which served
-# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16.
+# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
+# it; anything not set there falls through to the defaults below.
 set -euo pipefail

 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
@@ -31,64 +31,72 @@ if [[ -f "$ENV_FILE" ]]; then
 fi

 # --- defaults (used when the env file does not set them) ------------------
-: "${RUNTIME_DIR:=/home/dank/GitPackages/ciru-runtime-v4.4.1-qsa-hotfix}"
-: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca}"
+: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}"
+: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS}"
 : "${HOST:=0.0.0.0}"
 : "${PORT:=7770}"
-: "${CONTEXT_SIZE:=262144}"
+: "${CONTEXT_SIZE:=200000}"
 : "${PARALLEL_SLOTS:=1}"
+: "${ENABLE_MTP:=1}"
+: "${SPEC_DRAFT_N_MAX:=2}"
+: "${ENABLE_VISION:=0}"

-LAUNCHER="$RUNTIME_DIR/scripts/ciru/run-orca-server.sh"
-SERVER_BIN="$RUNTIME_DIR/build-gfx1151-sdk/bin/llama-server"
+SERVER_BIN="$RUNTIME_DIR/llama-server"

-if [[ ! -x "$LAUNCHER" ]]; then
-    echo "missing runtime launcher: $LAUNCHER" >&2
-    echo "fetch the hotfix source tarball from the model repo" >&2
-    echo "(runtime/v4.4.1/qsa-window-hotfix/) and build it:" >&2
-    echo "  bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
-    exit 1
-fi
 if [[ ! -x "$SERVER_BIN" ]]; then
-    echo "missing built runtime: $SERVER_BIN" >&2
-    echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
+    echo "missing runtime: $SERVER_BIN" >&2
+    echo "fetch and extract it:" >&2
+    echo "  curl -fL -o /tmp/unsloth-llama.tar.gz https://github.com/unslothai/llama.cpp/releases/download/b11030-mix-5ff778e/app-b11030-mix-5ff778e-linux-x64-rocm-gfx1151.tar.gz" >&2
+    echo "  mkdir -p $RUNTIME_DIR && tar xzf /tmp/unsloth-llama.tar.gz -C $RUNTIME_DIR" >&2
    exit 1
 fi

-# The launcher gates on a runtime root holding hip/lib/libamdhip64.so and
-# rocr/lib/libhsa-runtime64.so (it prepends both to LD_LIBRARY_PATH). Our
-# build links the private SDK venv through its RUNPATH, so point those
-# entries at the venv's unversioned devel symlinks. Idempotent: ln -sfn.
-VENV_LIB="$(ls -d "$RUNTIME_DIR"/.venv-rocm/lib/python*/site-packages/_rocm_sdk_devel/lib 2>/dev/null | head -1 || true)"
-if [[ -z "$VENV_LIB" || ! -e "$VENV_LIB/libamdhip64.so" || ! -e "$VENV_LIB/libhsa-runtime64.so" ]]; then
-    echo "missing SDK venv runtime under $RUNTIME_DIR/.venv-rocm (libamdhip64.so, libhsa-runtime64.so)" >&2
-    echo "build it: bash $RUNTIME_DIR/scripts/ciru/setup-linux-amd.sh --install-host-deps" >&2
-    exit 1
-fi
-mkdir -p "$RUNTIME_DIR/runtime/hip/lib" "$RUNTIME_DIR/runtime/rocr/lib"
-ln -sfn "$VENV_LIB/libamdhip64.so" "$RUNTIME_DIR/runtime/hip/lib/libamdhip64.so"
-ln -sfn "$VENV_LIB/libhsa-runtime64.so" "$RUNTIME_DIR/runtime/rocr/lib/libhsa-runtime64.so"
-if [[ ! -r "$MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" ]]; then
-    echo "missing target model: $MODEL_DIR/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf" >&2
-    echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2
+MODEL_FILE="$MODEL_DIR/Qwen3.8-Flash-Next-UD-IQ4_XS.gguf"
+if [[ ! -r "$MODEL_FILE" ]]; then
+    echo "missing target model: $MODEL_FILE" >&2
+    echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
    exit 1
 fi

-# The launcher checks the MTP head and the PLE sidecar itself, but a missing
-# payload should be a deploy error with a pointer, not a runtime exit 2.
-for required in \
-    "$MODEL_DIR/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf" \
-    "$MODEL_DIR/ple/ple.payload.bin" \
-    "$MODEL_DIR/ple/ple.manifest.json" \
-    "$MODEL_DIR/ple/ple.scale.bf16"; do
-    if [[ ! -r "$required" ]]; then
-        echo "missing payload: $required" >&2
-        echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-orca.sh" >&2
-        exit 1
-    fi
-done
+# The MTP shared head borrows the token embedding and the output projection
+# from the main model at runtime. It must be passed explicitly with -md:
+# sidecar auto-discovery does not search the mtp/ subfolder, so
+# --spec-type draft-mtp on its own would find nothing and serve at the
+# main model's speed with no error saying why.
+MTP_FILE="$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
+if [[ "$ENABLE_MTP" == "1" && ! -r "$MTP_FILE" ]]; then
+    echo "missing MTP head: $MTP_FILE" >&2
+    echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
+    exit 1
+fi

 # The Qwen3.8 family template keeps prior thinking blocks across turns when
-# preserve_thinking is set; the launcher passes --jinja but no template
-# kwargs, so name them here. A request-level chat_template_kwargs still
-# wins over this.
-exec "$LAUNCHER" --chat-template-kwargs '{"preserve_thinking":true}'
+# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents
+# do not lose the trail. A request-level chat_template_kwargs still wins.
+# The template gates reasoning_effort to {xhigh, medium, low} — the harness
+# folds its levels onto that set in models.yml (reasoningEffortMap).
+ARGS=(
+    -m "$MODEL_FILE"
+    -c "$CONTEXT_SIZE"
+    -np "$PARALLEL_SLOTS"
+    -ngl 999
+    --jinja
+    --alias frank-large
+    --host "$HOST"
+    --port "$PORT"
+    --chat-template-kwargs '{"preserve_thinking":true}'
+)
+if [[ "$ENABLE_MTP" == "1" ]]; then
+    ARGS+=(-md "$MTP_FILE" --spec-type draft-mtp --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
+fi
+if [[ "$ENABLE_VISION" == "1" ]]; then
+    MMPROJ_FILE="$MODEL_DIR/vision/mmproj-F16.gguf"
+    if [[ ! -r "$MMPROJ_FILE" ]]; then
+        echo "missing projector: $MMPROJ_FILE" >&2
+        echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
+        exit 1
+    fi
+    ARGS+=(--mmproj "$MMPROJ_FILE")
+fi
+
+exec "$SERVER_BIN" "${ARGS[@]}"