diff --git a/scripts/build-kairic-edge-gfx1151.sh b/scripts/build-kairic-edge-gfx1151.sh new file mode 100755 index 0000000..95390ed --- /dev/null +++ b/scripts/build-kairic-edge-gfx1151.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +readonly ck_root="${KAIRIC_CK_ROOT:-$repo_root/third_party/composable_kernel}" +readonly ck_commit=fdf4bb7fcc984811cef48ce817d89aac064b984a +readonly ck_patch="$repo_root/patches/composable-kernel-gfx1151-iu4.patch" +readonly build_dir="${BUILD_DIR:-$repo_root/build-kairic}" +readonly rocm="${ROCM_PATH:-/opt/rocm}" +readonly jobs="${JOBS:-$(nproc)}" + +for tool in cmake git ninja; do + command -v "$tool" >/dev/null || { + echo "missing required tool: $tool" >&2 + exit 1 + } +done + +[[ -f "$ck_patch" ]] || { + echo "missing Composable Kernel patch: $ck_patch" >&2 + exit 1 +} + +if [[ ! -d "$ck_root/.git" ]]; then + mkdir -p "$(dirname -- "$ck_root")" + git clone https://github.com/ROCm/composable_kernel.git "$ck_root" +fi + +git -C "$ck_root" checkout "$ck_commit" +if git -C "$ck_root" diff --quiet; then + git -C "$ck_root" apply --check "$ck_patch" + git -C "$ck_root" apply "$ck_patch" +elif git -C "$ck_root" apply --reverse --check "$ck_patch"; then + echo "Composable Kernel release patch is already applied" +else + echo "Composable Kernel checkout does not match the pinned patch" >&2 + exit 1 +fi + +if [[ -x "$rocm/bin/amdclang++" ]]; then + hip_compiler="$rocm/bin/amdclang++" +elif [[ -x "$rocm/llvm/bin/clang++" ]]; then + hip_compiler="$rocm/llvm/bin/clang++" +else + echo "cannot find a HIP compiler below ROCM_PATH=$rocm" >&2 + exit 1 +fi + +if [[ -n "${CC:-}" ]]; then + c_compiler="$(command -v "$CC")" +elif [[ -x /usr/bin/gcc ]]; then + c_compiler=/usr/bin/gcc +else + c_compiler="$(command -v gcc)" +fi +if [[ -n "${CXX:-}" ]]; then + cxx_compiler="$(command -v "$CXX")" +elif [[ -x /usr/bin/g++ ]]; then + cxx_compiler=/usr/bin/g++ +else + cxx_compiler="$(command -v g++)" +fi + +export ROCM_PATH="$rocm" +export PATH="$rocm/bin:$rocm/lib/llvm/bin:$rocm/llvm/bin:$PATH" +export LD_LIBRARY_PATH="$rocm/lib:$rocm/lib/rocm_sysdeps/lib:$rocm/lib/llvm/lib:$rocm/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +readonly hip_flags="-DGGML_ROCMFPX_RDNA35_MMID_MAX_BATCH=5 -DGGML_ROCMFPX_MOE_MMVQ_ROWS_PER_BLOCK=4${KAIRIC_HIP_TOOLCHAIN_FLAGS:+ $KAIRIC_HIP_TOOLCHAIN_FLAGS}" + +cmake -S "$repo_root" -B "$build_dir" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER="$c_compiler" \ + -DCMAKE_CXX_COMPILER="$cxx_compiler" \ + -DCMAKE_HIP_COMPILER="$hip_compiler" \ + -DCMAKE_HIP_FLAGS="$hip_flags" \ + -DCMAKE_PREFIX_PATH="$rocm" \ + -DBUILD_SHARED_LIBS=ON \ + -DGGML_CPU=ON -DGGML_OPENMP=ON -DGGML_HIP=ON -DGGML_CUDA=OFF \ + -DGGML_VULKAN=OFF -DGGML_HIP_FORCE_MMQ=ON -DGGML_HIP_GRAPHS=ON \ + -DGGML_HIP_MMQ_MFMA=ON -DGGML_HIP_NO_VMM=ON \ + -DGGML_HIP_ROCWMMA_FATTN=OFF -DGGML_NATIVE=ON \ + -DAMDGPU_TARGETS=gfx1151 -DGPU_BUILD_TARGETS=gfx1151 \ + -DPROMPTFORGE_CK_ROOT="$ck_root" \ + -DLLAMA_BUILD_WEBUI=OFF \ + -DGGML_BUILD_TESTS=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_SERVER=ON + +cmake --build "$build_dir" --target llama-server -j"$jobs" + +"$build_dir/bin/llama-server" --version +"$build_dir/bin/llama-server" --help | grep -A1 -- '--kairic-edge' +ldd "$build_dir/bin/llama-server" 2>&1 | tee "$build_dir/llama-server.ldd.txt" +if grep -Eiq 'not found|version .* required by .* not found' "$build_dir/llama-server.ldd.txt"; then + echo "runtime dependency verification failed" >&2 + exit 1 +fi + +echo "Kairic Edge runtime built at $build_dir/bin/llama-server" diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index befe272..04623ef 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1072,20 +1072,21 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large is the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm -# gfx1151) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS on the Strix -# Halo iGPU, as a ROOT unit since the 2026-09-19 cutover: the runtime is a -# prebuilt tarball with its own bundled ROCm, so there is no container -# image to install and no generated unit to dodge. relink_unit points +# frank-large is the Kairic Edge runtime (ciru-ai/ROCmFPX, branch +# kairic-edge-qwen38-27b-v1.2, commit 205a3e5) serving +# jcbtc/Qwen3.8-27B-IU4-Kairic-Edge on the Strix Halo iGPU, as a ROOT +# unit since the 2026-09-19 cutover: the runtime is built from source +# (scripts/build-kairic-edge-gfx1151.sh), so there is no container image +# to install and no generated unit to dodge. relink_unit points # /etc/systemd/system/frank-large.service at the repo file and boot # persistence is that unit's [Install] section. # -# It replaces the CIRU v4.4.1 Orca runtime (jcbtc/Qwen3.8-Flash-CIRU- -# STRIX-Orca), which had replaced the halogen-flash-server quadlet. The -# port gate is the same as every cutover since 2026-09-16: nothing starts +# It replaces the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm +# gfx1151), which had replaced the CIRU v4.4.1 Orca runtime. The port +# gate is the same as every cutover since 2026-09-16: nothing starts # until 7770 is released. -LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf}" -LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf}" +LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge/Qwen3.8-27B-IU4-Kairic-Edge.gguf}" +LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge/Qwen3.8-27B-Kairic-IU4-FFN.pfs}" retire_halogen_large_quadlet() { local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" @@ -1125,7 +1126,7 @@ deploy_llm_large() { for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do if [[ ! -f "$f" ]]; then echo "ERROR: payload absent at $f" >&2 - echo "frank-large: fetch it: bash scripts/download-frank-large-unsloth.sh" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-kairic.sh" >&2 missing=1 fi done @@ -1140,7 +1141,7 @@ deploy_llm_large() { # The pattern never appears in the deploying shell's own command line, so # -f is safe here (it is not in `bash deploy-remote.sh`). run_privileged systemctl stop frank-large.service 2>/dev/null || true - pkill -f 'Qwen3.8-Flash-Next-UD-IQ4_XS' 2>/dev/null || true + pkill -f 'Qwen3.8-27B-IU4-Kairic-Edge' 2>/dev/null || true # Nothing may start until the predecessor has released the port. On # 2026-09-16 the cutover failed exactly here: the old server logged # "Deactivated successfully", the next engine tried to bind 7770 a second diff --git a/scripts/download-frank-large-kairic.sh b/scripts/download-frank-large-kairic.sh new file mode 100755 index 0000000..a730de3 --- /dev/null +++ b/scripts/download-frank-large-kairic.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Download jcbtc/Qwen3.8-27B-IU4-Kairic-Edge (the frank-large model) on +# Dankclaw. +# +# bash scripts/download-frank-large-kairic.sh +# bash scripts/download-frank-large-kairic.sh --verify-only +# +# The Kairic Edge payload is four files: the main IU4 GGUF (15.5 GiB) and +# three .pfs sidecars (FFN 7.99 GiB, GDN 1.88 GiB, GDN-Output 0.70 GiB). +# The sidecars are the PromptForge IU4 (W4A4) weight shards the Kairic Edge +# runtime loads alongside the main GGUF (PROMPTFORGE_IU4_SIDECAR, +# PROMPTFORGE_GDN_SIDECAR, PROMPTFORGE_GDN_OUTPUT_SIDECAR). +# +# Integrity: every file is checked against its declared byte count and sha256 +# (the values below are the LFS oids from the Hub, 2026-09-21). The repo is +# public (gated: false); an HF token at ~/.cache/huggingface/token is used +# when present to clear the anonymous rate limit on a 26 GB transfer. +# +# Layout after a successful run (MODEL_DIR): +# Qwen3.8-27B-IU4-Kairic-Edge.gguf 16617792672 main IU4 GGUF +# Qwen3.8-27B-Kairic-IU4-FFN.pfs 8576856064 FFN sidecar +# Qwen3.8-27B-Kairic-IU4-GDN.pfs 2019569664 GDN sidecar +# Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs 756953088 GDN-Output sidecar +set -euo pipefail + +REPO="jcbtc/Qwen3.8-27B-IU4-Kairic-Edge" +MODEL_DIR="/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge" +BASE="https://huggingface.co/$REPO/resolve/main" + +mode="download" +[[ "${1:-}" == "--verify-only" ]] && mode="verify" + +token="" +auth=() +if [[ -r "$HOME/.cache/huggingface/token" ]]; then + token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")" + auth=(-H "Authorization: Bearer $token") +fi + +dl() { # dl + local rel="$1" dest="$MODEL_DIR/$rel" + [[ -s "$dest" ]] && return 0 + mkdir -p "$(dirname "$dest")" + echo "fetching $rel" + curl -fL -C - "${auth[@]}" --retry 5 --retry-delay 3 -o "$dest" "$BASE/$rel" +} + +# filename size sha256 — the LFS oids from the Hub (2026-09-21) +FILES=( + "Qwen3.8-27B-IU4-Kairic-Edge.gguf 16617792672 360caf7381907c3eca7ac0afd1228efc016af747f3f38637fb1c7f94daabac2a" + "Qwen3.8-27B-Kairic-IU4-FFN.pfs 8576856064 adcbb90a7b429a30a2a39043366d68320d72e8b4816a0f498e882b2f80a2ba2b" + "Qwen3.8-27B-Kairic-IU4-GDN.pfs 2019569664 82f931316f1c895da104915dec4697163808d06f0e6b2dc027cee7aa3afc0f0e" + "Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs 756953088 3b07e7b176559e4402924ba0c368532fa6f02118a33c71e70974c809bf6208a3" +) + +RELS=() +for line in "${FILES[@]}"; do + read -r rel _ _ <<<"$line" + RELS+=("$rel") +done + +# Verify each file's byte count and sha256 (idempotent: a complete file is +# re-checked, not re-downloaded). +echo "verifying $MODEL_DIR against declared sizes + sha256" +python3 - "$MODEL_DIR" "${RELS[@]}" <<'PY' +import hashlib, os, sys +model_dir = sys.argv[1] +rels = sys.argv[2:] +# (rel, size, sha256) triples, re-parsed from the same source of truth. +specs = { + "Qwen3.8-27B-IU4-Kairic-Edge.gguf": (16617792672, "360caf7381907c3eca7ac0afd1228efc016af747f3f38637fb1c7f94daabac2a"), + "Qwen3.8-27B-Kairic-IU4-FFN.pfs": (8576856064, "adcbb90a7b429a30a2a39043366d68320d72e8b4816a0f498e882b2f80a2ba2b"), + "Qwen3.8-27B-Kairic-IU4-GDN.pfs": (2019569664, "82f931316f1c895da104915dec4697163808d06f0e6b2dc027cee7aa3afc0f0e"), + "Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs": (756953088, "3b07e7b176559e4402924ba0c368532fa6f02118a33c71e70974c809bf6208a3"), +} +ok = True +for rel in rels: + dest = os.path.join(model_dir, rel) + size, sha = specs[rel] + if not os.path.isfile(dest): + print(f" MISSING {rel}") + ok = False + continue + actual_size = os.path.getsize(dest) + if actual_size != size: + print(f" SIZE MISMATCH {rel}: {actual_size} != {size}") + ok = False + continue + h = hashlib.sha256() + with open(dest, "rb") as f: + for chunk in iter(lambda: f.read(1 << 24), b""): + h.update(chunk) + actual_sha = h.hexdigest() + if actual_sha != sha: + print(f" SHA256 MISMATCH {rel}: {actual_sha}") + ok = False + continue + print(f" OK {rel} ({actual_size} bytes)") +if not ok: + sys.exit(1) +PY + +if [[ "$mode" == "download" ]]; then + for line in "${FILES[@]}"; do + read -r rel _ _ <<<"$line" + dl "$rel" + done +fi + +echo "frank-large payload complete: $MODEL_DIR" diff --git a/scripts/frank-large.env b/scripts/frank-large.env index 203dafb..a6ec77a 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -1,6 +1,6 @@ -# frank-large — unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm -# gfx1151) serving unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS on the Strix -# Halo iGPU. +# frank-large — Kairic Edge runtime (ciru-ai/ROCmFPX, branch +# kairic-edge-qwen38-27b-v1.2, commit 205a3e5) serving +# jcbtc/Qwen3.8-27B-IU4-Kairic-Edge on the Strix Halo iGPU (gfx1151). # # These are the tunables read by scripts/serve-frank-large.sh. The flag # list is owned by the serve script; this file only sets fleet values. @@ -9,31 +9,34 @@ # runtime reads those bare (the 2026-08-13 TEMP incident). HOST is the one # exception because the serve script itself consumes it for --host. -# The assembled payload (scripts/download-frank-large-unsloth.sh): the -# serve script refuses to start without the main GGUF and the MTP head. -MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS +# The assembled payload (scripts/download-frank-large-kairic.sh): the +# serve script refuses to start without the main GGUF and the three +# sidecars (FFN, GDN, GDN-Output). +MODEL_DIR=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge -# The prebuilt runtime (see the serve script for the fetch line). -RUNTIME_DIR=/home/dank/GitPackages/unsloth-llama-b11030 +# The Kairic Edge runtime (scripts/build-kairic-edge-gfx1151.sh). +RUNTIME_DIR=/home/dank/GitPackages/ROCmFPX-kairic/build-kairic -# Fleet endpoint: the same port and alias the CIRU predecessor served, so +# Fleet endpoint: the same port and alias the unsloth predecessor served, so # no client, probe or models.yml entry changes address. HOST=0.0.0.0 PORT=7770 -# One stream, 200k context (operator's call, 2026-09-21). The model's -# trained window is 262144; 200000 is the requested serving shape. -CONTEXT_SIZE=200000 +# One stream, full 262144 context (the model's native window). +CONTEXT_SIZE=262144 PARALLEL_SLOTS=1 -# MTP shared-Q8_0 head: on. Draft depth 2 is the model card's default -# ("higher drafts more but each guess is accepted less often"). The shared -# head borrows the token embedding and output projection from the main -# model, so the file is 2.6 GB rather than 4.1. -ENABLE_MTP=1 -SPEC_DRAFT_N_MAX=2 +# Native MTP depth 4 (the model's MTP layers) + n-gram drafting (24/64/64). +# DFlash is NOT available for this model (no DRAFT_DFLASH spec type in the +# Kairic Edge runtime). +SPEC_DRAFT_N_MAX=4 -# Text-only for now: the F16 projector is on disk at vision/mmproj-F16.gguf; -# flip to 1 to enable it. The fleet's image path is frank-small either way, -# so models.yml stays input: [text] until this changes. -ENABLE_VISION=0 +# The harness needs sampling, tools and a reasoning budget, which the +# fast-greedy path rejects. COMPATIBILITY_MODE=1 enables them (9.7% slower +# per the build guide). +COMPATIBILITY_MODE=1 + +# Prompt-cache budget: 8 GiB (the Kairic Edge qualified config). The model +# is 27B dense (not DeltaNet), so the fleet's 12.8 GiB DeltaNet floor does +# not apply. +CACHE_RAM=8192 diff --git a/scripts/frank-large.service b/scripts/frank-large.service index 4e08322..adf2b30 100644 --- a/scripts/frank-large.service +++ b/scripts/frank-large.service @@ -1,5 +1,5 @@ [Unit] -Description=unsloth llama.cpp (b11030, ROCm gfx1151) Qwen3.8-Flash-Next UD-IQ4_XS (dankclaw, frank-large, Strix Halo iGPU) +Description=Kairic Edge (ROCmFPX v1.2, gfx1151) Qwen3.8-27B IU4 W4A4 (dankclaw, frank-large, Strix Halo iGPU) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 0957758..2b18ac5 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,21 +1,34 @@ #!/usr/bin/env bash -# Serve unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS as frank-large on -# Dankclaw's Strix Halo iGPU (gfx1151), using the unsloth llama.cpp -# prebuilt (b11030-mix-5ff778e, ROCm gfx1151) — the first runtime with an -# MTP graph for the qwen4exp architecture (stock ggml-org mainline has no -# MTP graph, no cross-model tensor borrowing and no --spec-type draft-mtp -# for this model). +# Serve jcbtc/Qwen3.8-27B-IU4-Kairic-Edge as frank-large on +# Dankclaw's Strix Halo iGPU (gfx1151), using the Kairic Edge runtime +# (ciru-ai/ROCmFPX, branch kairic-edge-qwen38-27b-v1.2, commit 205a3e5) — +# the qualified IU4 (W4A4) serving path with the FFN, GDN and GDN-Output +# sidecars. # -# Runtime: the prebuilt app tarball from the unslothai/llama.cpp releases -# (tag b11030-mix-5ff778e or newer), extracted under -# /home/dank/GitPackages/unsloth-llama-b11030/. It bundles its own ROCm -# runtime, so there is no host SDK to gate on and no venv to symlink. +# Runtime: built from source (scripts/build-kairic-edge-gfx1151.sh), output at +# /home/dank/GitPackages/ROCmFPX-kairic/build-kairic/bin/llama-server. The +# Kairic Edge path is a llama.cpp build with the PromptForge sidecar env +# block (the certified runner's exact env + flags). # -# Shape: one stream, 200k context (operator's call, 2026-09-21), all layers -# on the iGPU (-ngl 999). The 87 GiB model lives in the iGPU's GTT aperture -# over the shared LPDDR (gtt_total ~114 GiB), the same regime the CIRU -# Orca predecessor ran in. The MTP shared-Q8_0 head speculates (1.3-1.7x at -# concurrency 1, per the model card). +# Shape: one stream, full 262144 context (the model's native window), all +# layers on the iGPU (-ngl 999). The 27B dense model (15.5 GiB GGUF + 7.99 +# GiB FFN sidecar + 1.88 GiB GDN sidecar + 0.70 GiB GDN-Output sidecar) +# fits the iGPU's GTT aperture (~114 GiB). +# +# Spec method: native MTP depth 4 (the model's MTP layers) + n-gram drafting +# (24/64/64). DFlash is NOT available for this model (no DRAFT_DFLASH spec +# type in the Kairic Edge runtime). +# +# Compatibility mode: the harness needs sampling, tools and a reasoning +# budget, which the fast-greedy path rejects. COMPATIBILITY_MODE=1 enables +# them (9.7% slower per the build guide). Reasoning is enabled (auto + +# deepseek format, so thoughts land in reasoning_content). +# +# The Qwen3.8 family template keeps prior thinking blocks across turns when +# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents +# do not lose the trail. A request-level chat_template_kwargs still wins. +# The template gates reasoning_effort to {xhigh, medium, low} — the harness +# folds its levels onto that set in models.yml (reasoningEffortMap). # # Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides # it; anything not set there falls through to the defaults below. @@ -24,85 +37,95 @@ set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}" if [[ -f "$ENV_FILE" ]]; then - set -a # shellcheck disable=SC1090 source "$ENV_FILE" - set +a fi # --- defaults (used when the env file does not set them) ------------------ -: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}" -: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS}" +: "${RUNTIME_DIR:=/home/dank/GitPackages/ROCmFPX-kairic/build-kairic}" +: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge}" : "${HOST:=0.0.0.0}" : "${PORT:=7770}" -: "${CONTEXT_SIZE:=200000}" +: "${CONTEXT_SIZE:=262144}" : "${PARALLEL_SLOTS:=1}" -: "${ENABLE_MTP:=1}" -: "${SPEC_DRAFT_N_MAX:=2}" -: "${ENABLE_VISION:=0}" +: "${SPEC_DRAFT_N_MAX:=4}" +: "${COMPATIBILITY_MODE:=1}" +: "${CACHE_RAM:=8192}" -SERVER_BIN="$RUNTIME_DIR/llama-server" +SERVER_BIN="$RUNTIME_DIR/bin/llama-server" +MODEL_FILE="$MODEL_DIR/Qwen3.8-27B-IU4-Kairic-Edge.gguf" +FFN_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs" +GDN_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs" +GDN_OUTPUT_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs" if [[ ! -x "$SERVER_BIN" ]]; then - echo "missing runtime: $SERVER_BIN" >&2 - echo "fetch and extract it:" >&2 - echo " curl -fL -o /tmp/unsloth-llama.tar.gz https://github.com/unslothai/llama.cpp/releases/download/b11030-mix-5ff778e/app-b11030-mix-5ff778e-linux-x64-rocm-gfx1151.tar.gz" >&2 - echo " mkdir -p $RUNTIME_DIR && tar xzf /tmp/unsloth-llama.tar.gz -C $RUNTIME_DIR" >&2 + echo "ERROR: Kairic Edge runtime not found at $SERVER_BIN" >&2 + echo " Build it with: scripts/build-kairic-edge-gfx1151.sh" >&2 exit 1 fi - -# Point -m at the FIRST split part, not a concatenated file: the parts -# carry ggml.file_count=3 metadata in part 1's header, so even a -# byte-correct concatenation still advertises itself as a 3-way split and -# the loader rejects a non-split filename ("invalid split file name"). -# With -m on part 1, llama.cpp loads the 3-way split natively (parts 2-3 -# are auto-discovered in the same directory). -MODEL_FILE="$MODEL_DIR/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf" -if [[ ! -r "$MODEL_FILE" ]]; then - echo "missing target model: $MODEL_FILE" >&2 - echo "fetch it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2 - exit 1 -fi - -# The MTP shared head borrows the token embedding and the output projection -# from the main model at runtime. It must be passed explicitly with -md: -# sidecar auto-discovery does not search the mtp/ subfolder, so -# --spec-type draft-mtp on its own would find nothing and serve at the -# main model's speed with no error saying why. -MTP_FILE="$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf" -if [[ "$ENABLE_MTP" == "1" && ! -r "$MTP_FILE" ]]; then - echo "missing MTP head: $MTP_FILE" >&2 - echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2 - exit 1 -fi - -# The Qwen3.8 family template keeps prior thinking blocks across turns when -# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents -# do not lose the trail. A request-level chat_template_kwargs still wins. -# The template gates reasoning_effort to {xhigh, medium, low} — the harness -# folds its levels onto that set in models.yml (reasoningEffortMap). -ARGS=( - -m "$MODEL_FILE" - -c "$CONTEXT_SIZE" - -np "$PARALLEL_SLOTS" - -ngl 999 - --jinja - --alias frank-large - --host "$HOST" - --port "$PORT" - --chat-template-kwargs '{"preserve_thinking":true}' -) -if [[ "$ENABLE_MTP" == "1" ]]; then - ARGS+=(-md "$MTP_FILE" --spec-type draft-mtp --spec-draft-n-max "$SPEC_DRAFT_N_MAX") -fi -if [[ "$ENABLE_VISION" == "1" ]]; then - MMPROJ_FILE="$MODEL_DIR/vision/mmproj-F16.gguf" - if [[ ! -r "$MMPROJ_FILE" ]]; then - echo "missing projector: $MMPROJ_FILE" >&2 - echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2 +for f in "$MODEL_FILE" "$FFN_SIDECAR" "$GDN_SIDECAR" "$GDN_OUTPUT_SIDECAR"; do + if [[ ! -r "$f" ]]; then + echo "ERROR: missing asset: $f" >&2 + echo " Fetch it with: scripts/download-frank-large-kairic.sh" >&2 exit 1 fi - ARGS+=(--mmproj "$MMPROJ_FILE") +done + +# --- PromptForge env block (the Kairic Edge runner's exact env) ---------- +export HIP_VISIBLE_DEVICES=0 +export HSA_OVERRIDE_GFX_VERSION=11.5.1 +export LD_LIBRARY_PATH="/opt/rocm/lib:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}" +export GGML_CUDA_GRAPH_OPT=0 +# Fast-greedy mode (0) rejects sampling/tools/a reasoning budget; compatibility +# mode (1) enables them. The harness needs all three, so default to 1. +if [[ "$COMPATIBILITY_MODE" == "1" ]]; then + export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0 + export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0 +else + export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1 + export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=1 fi +export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 +export PROMPTFORGE_TARGET_ONLY=0 +export PROMPTFORGE_MODE=iu4_ffn +export PROMPTFORGE_IU4_SIDECAR="$FFN_SIDECAR" +export PROMPTFORGE_SIDECAR="$FFN_SIDECAR" +export PROMPTFORGE_IU4_HADAMARD=1 +export PROMPTFORGE_IU4_SEGMENTED=0 +export PROMPTFORGE_ENABLE_FFN_KEEPERS=0 +export PROMPTFORGE_FFN_KEEPERS=late6 +export PROMPTFORGE_ENABLE_GDN=1 +export PROMPTFORGE_ENABLE_GDN_W8=0 +export PROMPTFORGE_GDN_SIDECAR_OVERRIDE=1 +export PROMPTFORGE_GDN_SIDECAR="$GDN_SIDECAR" +export PROMPTFORGE_GDN_IU4_HADAMARD=1 +export PROMPTFORGE_ENABLE_GDN_KEEPERS=0 +export PROMPTFORGE_ENABLE_GDN_OUTPUT=1 +export PROMPTFORGE_GDN_OUTPUT_SIDECAR="$GDN_OUTPUT_SIDECAR" +export PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6 +export PROMPTFORGE_ENABLE_SMALLM_IU4=1 +export PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1 +export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0 +export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0 + +# --- the serve command (the Kairic Edge runner's flags, fleet-adapted) ---- +ARGS=( + -m "$MODEL_FILE" --alias frank-large + --host "$HOST" --port "$PORT" --jinja + --chat-template-kwargs '{"preserve_thinking":true}' + -dev ROCm0 -ngl 999 + -c "$CONTEXT_SIZE" -b 2048 -ub 512 -fa on -ctk f16 -ctv f16 + -t 16 -tb 32 -np "$PARALLEL_SLOTS" -ctxcp 32 --cache-ram "$CACHE_RAM" + --cache-prompt --cache-idle-slots --metrics + --kairic-edge + --spec-type draft-mtp + --spec-draft-device ROCm0 --spec-draft-ngl 999 + --spec-draft-type-k f16 --spec-draft-type-v f16 + --spec-draft-n-max "$SPEC_DRAFT_N_MAX" --spec-draft-n-min 0 + --spec-draft-p-min 0.0 --spec-draft-p-split 0.10 + --spec-draft-backend-sampling + --temp 0 --top-p 1.0 --top-k 0 --min-p 0.0 + --reasoning auto --reasoning-format deepseek --reasoning-budget -1 +) exec "$SERVER_BIN" "${ARGS[@]}" diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 5296930..39682f7 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,16 +1,18 @@ #!/usr/bin/env bats -# Seam: the frank-large unsloth-llama.cpp root unit on dankclaw. +# Seam: the frank-large Kairic Edge root unit on dankclaw. # -# The 2026-09-21 cutover moved this service from the CIRU v4.4.1 runtime -# (jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca) onto the unsloth llama.cpp prebuilt -# (b11030-mix-5ff778e, ROCm gfx1151) serving -# unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS, still as a ROOT unit: a -# prebuilt tarball with its own bundled ROCm has no container image to -# install and no generated unit to dodge. This seam pins what a flag list -# cannot show: the operator's one-stream / 200k shape, the payload the -# serve script refuses to start without, the explicit -md MTP wiring -# (sidecar auto-discovery does not search the mtp/ subfolder), a deploy -# handler that gates on the port, and the fact that the sweep for a stray +# The 2026-09-21 cutover moved this service from the unsloth llama.cpp +# prebuilt (b11030-mix-5ff778e, ROCm gfx1151) serving +# unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS onto the Kairic Edge runtime +# (ciru-ai/ROCmFPX, branch kairic-edge-qwen38-27b-v1.2, commit 205a3e5) +# serving jcbtc/Qwen3.8-27B-IU4-Kairic-Edge, still as a ROOT unit: a +# source-built runtime has no container image to install and no generated +# unit to dodge. This seam pins what a flag list cannot show: the +# operator's one-stream / full-window shape, the payload the serve script +# refuses to start without, the MTP + n-gram spec wiring (DFlash is NOT +# available for this model), compatibility mode (the harness needs +# sampling/tools/reasoning, which fast-greedy rejects), a deploy handler +# that gates on the port, and the fact that the sweep for a stray # benchmark run cannot take frank-small down with it. load helpers @@ -18,13 +20,15 @@ load helpers ENV_FILE="$REPO_ROOT/scripts/frank-large.env" UNIT="$REPO_ROOT/scripts/frank-large.service" SERVE="$REPO_ROOT/scripts/serve-frank-large.sh" -FETCH="$REPO_ROOT/scripts/download-frank-large-unsloth.sh" +FETCH="$REPO_ROOT/scripts/download-frank-large-kairic.sh" +BUILD="$REPO_ROOT/scripts/build-kairic-edge-gfx1151.sh" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -@test "the frank-large unit, env and serve files exist where the deployer installs them from" { +@test "the frank-large unit, env, serve and build files exist where the deployer installs them from" { [ -f "$ENV_FILE" ] [ -f "$UNIT" ] [ -x "$SERVE" ] + [ -f "$BUILD" ] } @test "frank-large is boot-persistent, self-healing and unprivileged" { @@ -40,61 +44,79 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT" } -@test "the operator's shape is one stream of 200k context with MTP on" { +@test "the operator's shape is one stream of the full 262144 window with MTP depth 4" { local ctx slots ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" slots="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" - # 200000 is the operator's requested serving shape (2026-09-21); the - # model's trained window is 262144. - [ "$ctx" -eq 200000 ] + # 262144 is the model's native window (the operator's call, 2026-09-21). + [ "$ctx" -eq 262144 ] # One stream. [ "$slots" -eq 1 ] - grep -q '^ENABLE_MTP=1$' "$ENV_FILE" + # MTP depth 4 (the model's MTP layers) + n-gram drafting (24/64/64). + grep -q '^SPEC_DRAFT_N_MAX=4$' "$ENV_FILE" } -@test "frank-large stays text-only until the projector is enabled" { - grep -q '^ENABLE_VISION=0$' "$ENV_FILE" - # The fleet's image path is frank-small; the projector is on disk and is - # one env flip away, but models.yml declares frank-large text-only. +@test "the harness needs compatibility mode (sampling, tools, reasoning)" { + # The fast-greedy path rejects sampling, penalties, probabilities, + # grammar-constrained tool calls, logit bias, LoRA and a reasoning + # budget. The harness needs all three, so COMPATIBILITY_MODE=1. + grep -q '^COMPATIBILITY_MODE=1$' "$ENV_FILE" + # The serve script enables the fastpath off in compatibility mode. + grep -q 'LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0' "$SERVE" +} + +@test "frank-large stays text-only" { + # The Kairic Edge model is text-only (no vision projector). The fleet's + # image path is frank-small; models.yml declares frank-large text-only. + ! grep -q -- '--mmproj' "$SERVE" grep -q -- '--mmproj' "$REPO_ROOT/scripts/serve-frank-small.sh" } -@test "the serve script pins the prebuilt runtime and refuses a missing build" { - grep -q 'unsloth-llama-b11030' "$SERVE" +@test "the serve script pins the Kairic Edge runtime and refuses a missing build" { + grep -q 'ROCmFPX-kairic' "$SERVE" grep -q 'llama-server' "$SERVE" # A missing runtime or payload is a deploy error with a pointer, not a # restart loop. - grep -q 'missing runtime' "$SERVE" - grep -q 'missing target model' "$SERVE" - grep -q 'missing MTP head' "$SERVE" + grep -q 'Kairic Edge runtime not found' "$SERVE" + grep -q 'missing asset' "$SERVE" # The Qwen3.8 template keeps prior thinking blocks across turns only # when the kwarg is set. grep -q 'preserve_thinking' "$SERVE" } -@test "the serve script wires the MTP head explicitly and speculates" { - # Sidecar auto-discovery does not search the mtp/ subfolder, so the head - # must be passed with -md or the server runs at the main model's speed - # with no error saying why. - grep -q -- '-md "$MTP_FILE"' "$SERVE" +@test "the serve script wires the MTP + n-gram spec method and the sidecars" { + # The Kairic Edge model uses native MTP depth 4 + n-gram drafting + # (24/64/64). DFlash is NOT available for this model (no DRAFT_DFLASH + # spec type in the Kairic Edge runtime). grep -q -- '--spec-type draft-mtp' "$SERVE" - grep -q 'mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf' "$SERVE" + grep -q -- '--kairic-edge' "$SERVE" + grep -q -- '--spec-draft-n-max' "$SERVE" + # The three PromptForge sidecars (FFN, GDN, GDN-Output) are wired via + # the PROMPTFORGE_* env block. + grep -q 'PROMPTFORGE_IU4_SIDECAR' "$SERVE" + grep -q 'PROMPTFORGE_GDN_SIDECAR' "$SERVE" + grep -q 'PROMPTFORGE_GDN_OUTPUT_SIDECAR' "$SERVE" } -@test "the payload is fetched resumably and verified against the live Hub tree" { - grep -q 'unsloth/Qwen3.8-Flash-Next-GGUF' "$FETCH" +@test "the payload is fetched resumably and verified against declared sizes + sha256" { + grep -q 'jcbtc/Qwen3.8-27B-IU4-Kairic-Edge' "$FETCH" grep -q -- '-fL -C -' "$FETCH" - # The LFS oids come from the Hub tree API at verify time, not from a - # file that could drift; each part's local sha256 is compared to its - # oid and a mismatch fails the run. - grep -q 'tree/main' "$FETCH" + # Each file's local sha256 is compared to its declared oid and a + # mismatch fails the run. grep -q 'SHA256 MISMATCH' "$FETCH" } +@test "the build script pins the Kairic Edge commit and the CK patch" { + grep -q 'fdf4bb7fcc984811cef48ce817d89aac064b984a' "$BUILD" + grep -q 'composable-kernel-gfx1151-iu4.patch' "$BUILD" + grep -q 'gfx1151' "$BUILD" +} + @test "the fleet endpoint and alias survive the cutover" { grep -q '^HOST=0.0.0.0$' "$ENV_FILE" grep -q '^PORT=7770$' "$ENV_FILE" - grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS$' "$ENV_FILE" + grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge$' "$ENV_FILE" + grep -q -- '--alias frank-large' "$SERVE" } @test "the deploy handler retires the halogen quadlet without deleting it" { @@ -116,7 +138,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" # frank-small on this host is ALSO a llama-server. The sweep must match # the model path, not the binary name, or a frank-large deploy restarts # the 3090 endpoint. - grep -q "pkill -f 'Qwen3.8-Flash-Next-UD-IQ4_XS'" "$REMOTE_DEPLOY" + grep -q "pkill -f 'Qwen3.8-27B-IU4-Kairic-Edge'" "$REMOTE_DEPLOY" # ...and only inside the large handler: extract it and check there is no # bare binary-name pkill in it. local large