frank-large: swap to Kairic Edge 27B IU4 (W4A4) on Strix Halo

Replace the unsloth llama.cpp prebuilt (b11030, Qwen3.8-Flash-Next UD-IQ4_XS, 125B MoE) with the Kairic Edge runtime (ciru-ai/ROCmFPX, branch kairic-edge-qwen38-27b-v1.2, commit 205a3e5) serving jcbtc/Qwen3.8-27B-IU4-Kairic-Edge (27B dense, IU4 W4A4). - Runtime: built from source (scripts/build-kairic-edge-gfx1151.sh), TheRock 7.15.0a20260718 toolchain, CK fdf4bb7f + gfx1151-iu4 patch. - Shape: one stream, full 262144 context (the model's native window), 1 slot, 8 GiB prompt-cache budget. - Spec method: native MTP depth 4 + n-gram drafting (24/64/64). DFlash is NOT available for this model (no DRAFT_DFLASH spec type). - Compatibility mode: the harness needs sampling, tools and a reasoning budget, which fast-greedy rejects. COMPATIBILITY_MODE=1 enables them. - Reasoning: enabled (auto + deepseek format, thoughts land in reasoning_content). - Payload: 4 files (15.5 GiB GGUF + 7.99 GiB FFN + 1.88 GiB GDN + 0.70 GiB GDN-Output), sha256-verified (scripts/download-frank-large-kairic.sh).
2026-09-21 18:56:35 -07:00
1 parent 2f7aadab88
commit b2a74004df
@@ -1,21 +1,34 @@
 #!/usr/bin/env bash
-# Serve unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS as frank-large on
-# Dankclaw's Strix Halo iGPU (gfx1151), using the unsloth llama.cpp
-# prebuilt (b11030-mix-5ff778e, ROCm gfx1151) — the first runtime with an
-# MTP graph for the qwen4exp architecture (stock ggml-org mainline has no
-# MTP graph, no cross-model tensor borrowing and no --spec-type draft-mtp
-# for this model).
+# Serve jcbtc/Qwen3.8-27B-IU4-Kairic-Edge as frank-large on
+# Dankclaw's Strix Halo iGPU (gfx1151), using the Kairic Edge runtime
+# (ciru-ai/ROCmFPX, branch kairic-edge-qwen38-27b-v1.2, commit 205a3e5) —
+# the qualified IU4 (W4A4) serving path with the FFN, GDN and GDN-Output
+# sidecars.
 #
-# Runtime: the prebuilt app tarball from the unslothai/llama.cpp releases
-# (tag b11030-mix-5ff778e or newer), extracted under
-# /home/dank/GitPackages/unsloth-llama-b11030/. It bundles its own ROCm
-# runtime, so there is no host SDK to gate on and no venv to symlink.
+# Runtime: built from source (scripts/build-kairic-edge-gfx1151.sh), output at
+# /home/dank/GitPackages/ROCmFPX-kairic/build-kairic/bin/llama-server. The
+# Kairic Edge path is a llama.cpp build with the PromptForge sidecar env
+# block (the certified runner's exact env + flags).
 #
-# Shape: one stream, 200k context (operator's call, 2026-09-21), all layers
-# on the iGPU (-ngl 999). The 87 GiB model lives in the iGPU's GTT aperture
-# over the shared LPDDR (gtt_total ~114 GiB), the same regime the CIRU
-# Orca predecessor ran in. The MTP shared-Q8_0 head speculates (1.3-1.7x at
-# concurrency 1, per the model card).
+# Shape: one stream, full 262144 context (the model's native window), all
+# layers on the iGPU (-ngl 999). The 27B dense model (15.5 GiB GGUF + 7.99
+# GiB FFN sidecar + 1.88 GiB GDN sidecar + 0.70 GiB GDN-Output sidecar)
+# fits the iGPU's GTT aperture (~114 GiB).
+#
+# Spec method: native MTP depth 4 (the model's MTP layers) + n-gram drafting
+# (24/64/64). DFlash is NOT available for this model (no DRAFT_DFLASH spec
+# type in the Kairic Edge runtime).
+#
+# Compatibility mode: the harness needs sampling, tools and a reasoning
+# budget, which the fast-greedy path rejects. COMPATIBILITY_MODE=1 enables
+# them (9.7% slower per the build guide). Reasoning is enabled (auto +
+# deepseek format, so thoughts land in reasoning_content).
+#
+# The Qwen3.8 family template keeps prior thinking blocks across turns when
+# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents
+# do not lose the trail. A request-level chat_template_kwargs still wins.
+# The template gates reasoning_effort to {xhigh, medium, low} — the harness
+# folds its levels onto that set in models.yml (reasoningEffortMap).
 #
 # Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
 # it; anything not set there falls through to the defaults below.
@@ -24,85 +37,95 @@ set -euo pipefail
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
 if [[ -f "$ENV_FILE" ]]; then
-    set -a
    # shellcheck disable=SC1090
    source "$ENV_FILE"
-    set +a
 fi

 # --- defaults (used when the env file does not set them) ------------------
-: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}"
-: "${MODEL_DIR:=/home/dank/models/Qwen3.8-Flash-Next-UD-IQ4_XS}"
+: "${RUNTIME_DIR:=/home/dank/GitPackages/ROCmFPX-kairic/build-kairic}"
+: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge}"
 : "${HOST:=0.0.0.0}"
 : "${PORT:=7770}"
-: "${CONTEXT_SIZE:=200000}"
+: "${CONTEXT_SIZE:=262144}"
 : "${PARALLEL_SLOTS:=1}"
-: "${ENABLE_MTP:=1}"
-: "${SPEC_DRAFT_N_MAX:=2}"
-: "${ENABLE_VISION:=0}"
+: "${SPEC_DRAFT_N_MAX:=4}"
+: "${COMPATIBILITY_MODE:=1}"
+: "${CACHE_RAM:=8192}"

-SERVER_BIN="$RUNTIME_DIR/llama-server"
+SERVER_BIN="$RUNTIME_DIR/bin/llama-server"
+MODEL_FILE="$MODEL_DIR/Qwen3.8-27B-IU4-Kairic-Edge.gguf"
+FFN_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
+GDN_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
+GDN_OUTPUT_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs"

 if [[ ! -x "$SERVER_BIN" ]]; then
-    echo "missing runtime: $SERVER_BIN" >&2
-    echo "fetch and extract it:" >&2
-    echo "  curl -fL -o /tmp/unsloth-llama.tar.gz https://github.com/unslothai/llama.cpp/releases/download/b11030-mix-5ff778e/app-b11030-mix-5ff778e-linux-x64-rocm-gfx1151.tar.gz" >&2
-    echo "  mkdir -p $RUNTIME_DIR && tar xzf /tmp/unsloth-llama.tar.gz -C $RUNTIME_DIR" >&2
+    echo "ERROR: Kairic Edge runtime not found at $SERVER_BIN" >&2
+    echo "       Build it with: scripts/build-kairic-edge-gfx1151.sh" >&2
    exit 1
 fi
-
-# Point -m at the FIRST split part, not a concatenated file: the parts
-# carry ggml.file_count=3 metadata in part 1's header, so even a
-# byte-correct concatenation still advertises itself as a 3-way split and
-# the loader rejects a non-split filename ("invalid split file name").
-# With -m on part 1, llama.cpp loads the 3-way split natively (parts 2-3
-# are auto-discovered in the same directory).
-MODEL_FILE="$MODEL_DIR/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf"
-if [[ ! -r "$MODEL_FILE" ]]; then
-    echo "missing target model: $MODEL_FILE" >&2
-    echo "fetch it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
-    exit 1
-fi
-
-# The MTP shared head borrows the token embedding and the output projection
-# from the main model at runtime. It must be passed explicitly with -md:
-# sidecar auto-discovery does not search the mtp/ subfolder, so
-# --spec-type draft-mtp on its own would find nothing and serve at the
-# main model's speed with no error saying why.
-MTP_FILE="$MODEL_DIR/mtp/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
-if [[ "$ENABLE_MTP" == "1" && ! -r "$MTP_FILE" ]]; then
-    echo "missing MTP head: $MTP_FILE" >&2
-    echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
-    exit 1
-fi
-
-# The Qwen3.8 family template keeps prior thinking blocks across turns when
-# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents
-# do not lose the trail. A request-level chat_template_kwargs still wins.
-# The template gates reasoning_effort to {xhigh, medium, low} — the harness
-# folds its levels onto that set in models.yml (reasoningEffortMap).
-ARGS=(
-    -m "$MODEL_FILE"
-    -c "$CONTEXT_SIZE"
-    -np "$PARALLEL_SLOTS"
-    -ngl 999
-    --jinja
-    --alias frank-large
-    --host "$HOST"
-    --port "$PORT"
-    --chat-template-kwargs '{"preserve_thinking":true}'
-)
-if [[ "$ENABLE_MTP" == "1" ]]; then
-    ARGS+=(-md "$MTP_FILE" --spec-type draft-mtp --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
-fi
-if [[ "$ENABLE_VISION" == "1" ]]; then
-    MMPROJ_FILE="$MODEL_DIR/vision/mmproj-F16.gguf"
-    if [[ ! -r "$MMPROJ_FILE" ]]; then
-        echo "missing projector: $MMPROJ_FILE" >&2
-        echo "fetch and assemble it: bash $SCRIPT_DIR/download-frank-large-unsloth.sh" >&2
+for f in "$MODEL_FILE" "$FFN_SIDECAR" "$GDN_SIDECAR" "$GDN_OUTPUT_SIDECAR"; do
+    if [[ ! -r "$f" ]]; then
+        echo "ERROR: missing asset: $f" >&2
+        echo "       Fetch it with: scripts/download-frank-large-kairic.sh" >&2
        exit 1
    fi
-    ARGS+=(--mmproj "$MMPROJ_FILE")
+done
+
+# --- PromptForge env block (the Kairic Edge runner's exact env) ----------
+export HIP_VISIBLE_DEVICES=0
+export HSA_OVERRIDE_GFX_VERSION=11.5.1
+export LD_LIBRARY_PATH="/opt/rocm/lib:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}"
+export GGML_CUDA_GRAPH_OPT=0
+# Fast-greedy mode (0) rejects sampling/tools/a reasoning budget; compatibility
+# mode (1) enables them. The harness needs all three, so default to 1.
+if [[ "$COMPATIBILITY_MODE" == "1" ]]; then
+    export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0
+    export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
+else
+    export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1
+    export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=1
 fi
+export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1
+export PROMPTFORGE_TARGET_ONLY=0
+export PROMPTFORGE_MODE=iu4_ffn
+export PROMPTFORGE_IU4_SIDECAR="$FFN_SIDECAR"
+export PROMPTFORGE_SIDECAR="$FFN_SIDECAR"
+export PROMPTFORGE_IU4_HADAMARD=1
+export PROMPTFORGE_IU4_SEGMENTED=0
+export PROMPTFORGE_ENABLE_FFN_KEEPERS=0
+export PROMPTFORGE_FFN_KEEPERS=late6
+export PROMPTFORGE_ENABLE_GDN=1
+export PROMPTFORGE_ENABLE_GDN_W8=0
+export PROMPTFORGE_GDN_SIDECAR_OVERRIDE=1
+export PROMPTFORGE_GDN_SIDECAR="$GDN_SIDECAR"
+export PROMPTFORGE_GDN_IU4_HADAMARD=1
+export PROMPTFORGE_ENABLE_GDN_KEEPERS=0
+export PROMPTFORGE_ENABLE_GDN_OUTPUT=1
+export PROMPTFORGE_GDN_OUTPUT_SIDECAR="$GDN_OUTPUT_SIDECAR"
+export PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6
+export PROMPTFORGE_ENABLE_SMALLM_IU4=1
+export PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1
+export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0
+export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0
+
+# --- the serve command (the Kairic Edge runner's flags, fleet-adapted) ----
+ARGS=(
+    -m "$MODEL_FILE" --alias frank-large
+    --host "$HOST" --port "$PORT" --jinja
+    --chat-template-kwargs '{"preserve_thinking":true}'
+    -dev ROCm0 -ngl 999
+    -c "$CONTEXT_SIZE" -b 2048 -ub 512 -fa on -ctk f16 -ctv f16
+    -t 16 -tb 32 -np "$PARALLEL_SLOTS" -ctxcp 32 --cache-ram "$CACHE_RAM"
+    --cache-prompt --cache-idle-slots --metrics
+    --kairic-edge
+    --spec-type draft-mtp
+    --spec-draft-device ROCm0 --spec-draft-ngl 999
+    --spec-draft-type-k f16 --spec-draft-type-v f16
+    --spec-draft-n-max "$SPEC_DRAFT_N_MAX" --spec-draft-n-min 0
+    --spec-draft-p-min 0.0 --spec-draft-p-split 0.10
+    --spec-draft-backend-sampling
+    --temp 0 --top-p 1.0 --top-k 0 --min-p 0.0
+    --reasoning auto --reasoning-format deepseek --reasoning-budget -1
+)

 exec "$SERVER_BIN" "${ARGS[@]}"