diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 04623ef..44c8de8 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1072,21 +1072,20 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large is the Kairic Edge runtime (ciru-ai/ROCmFPX, branch -# kairic-edge-qwen38-27b-v1.2, commit 205a3e5) serving -# jcbtc/Qwen3.8-27B-IU4-Kairic-Edge on the Strix Halo iGPU, as a ROOT -# unit since the 2026-09-19 cutover: the runtime is built from source -# (scripts/build-kairic-edge-gfx1151.sh), so there is no container image -# to install and no generated unit to dodge. relink_unit points -# /etc/systemd/system/frank-large.service at the repo file and boot -# persistence is that unit's [Install] section. +# frank-large is the stock llama.cpp prebuilt (unsloth b11030, ROCm +# gfx1151) serving unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2 +# drafter (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Strix Halo iGPU, +# as a ROOT unit since the 2026-09-19 cutover: the runtime is a prebuilt +# binary, so there is no container image to install and no generated unit +# to dodge. relink_unit points /etc/systemd/system/frank-large.service at +# the repo file and boot persistence is that unit's [Install] section. # -# It replaces the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm -# gfx1151), which had replaced the CIRU v4.4.1 Orca runtime. The port -# gate is the same as every cutover since 2026-09-16: nothing starts -# until 7770 is released. -LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge/Qwen3.8-27B-IU4-Kairic-Edge.gguf}" -LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge/Qwen3.8-27B-Kairic-IU4-FFN.pfs}" +# It replaces the Kairic Edge runtime (ciru-ai/ROCmFPX), which had +# replaced the unsloth llama.cpp prebuilt serving +# unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS. The port gate is the same as +# every cutover since 2026-09-16: nothing starts until 7770 is released. +LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-UD-Q8_K_XL.gguf}" +LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}" retire_halogen_large_quadlet() { local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" @@ -1126,7 +1125,7 @@ deploy_llm_large() { for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do if [[ ! -f "$f" ]]; then echo "ERROR: payload absent at $f" >&2 - echo "frank-large: fetch it: bash scripts/download-frank-large-kairic.sh" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large.sh" >&2 missing=1 fi done @@ -1141,7 +1140,7 @@ deploy_llm_large() { # The pattern never appears in the deploying shell's own command line, so # -f is safe here (it is not in `bash deploy-remote.sh`). run_privileged systemctl stop frank-large.service 2>/dev/null || true - pkill -f 'Qwen3.8-27B-IU4-Kairic-Edge' 2>/dev/null || true + pkill -f 'Qwen3.8-27B-UD-Q8_K_XL' 2>/dev/null || true # Nothing may start until the predecessor has released the port. On # 2026-09-16 the cutover failed exactly here: the old server logged # "Deactivated successfully", the next engine tried to bind 7770 a second diff --git a/scripts/download-frank-large-kairic.sh b/scripts/download-frank-large-kairic.sh deleted file mode 100755 index a730de3..0000000 --- a/scripts/download-frank-large-kairic.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash -# Download jcbtc/Qwen3.8-27B-IU4-Kairic-Edge (the frank-large model) on -# Dankclaw. -# -# bash scripts/download-frank-large-kairic.sh -# bash scripts/download-frank-large-kairic.sh --verify-only -# -# The Kairic Edge payload is four files: the main IU4 GGUF (15.5 GiB) and -# three .pfs sidecars (FFN 7.99 GiB, GDN 1.88 GiB, GDN-Output 0.70 GiB). -# The sidecars are the PromptForge IU4 (W4A4) weight shards the Kairic Edge -# runtime loads alongside the main GGUF (PROMPTFORGE_IU4_SIDECAR, -# PROMPTFORGE_GDN_SIDECAR, PROMPTFORGE_GDN_OUTPUT_SIDECAR). -# -# Integrity: every file is checked against its declared byte count and sha256 -# (the values below are the LFS oids from the Hub, 2026-09-21). The repo is -# public (gated: false); an HF token at ~/.cache/huggingface/token is used -# when present to clear the anonymous rate limit on a 26 GB transfer. -# -# Layout after a successful run (MODEL_DIR): -# Qwen3.8-27B-IU4-Kairic-Edge.gguf 16617792672 main IU4 GGUF -# Qwen3.8-27B-Kairic-IU4-FFN.pfs 8576856064 FFN sidecar -# Qwen3.8-27B-Kairic-IU4-GDN.pfs 2019569664 GDN sidecar -# Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs 756953088 GDN-Output sidecar -set -euo pipefail - -REPO="jcbtc/Qwen3.8-27B-IU4-Kairic-Edge" -MODEL_DIR="/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge" -BASE="https://huggingface.co/$REPO/resolve/main" - -mode="download" -[[ "${1:-}" == "--verify-only" ]] && mode="verify" - -token="" -auth=() -if [[ -r "$HOME/.cache/huggingface/token" ]]; then - token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")" - auth=(-H "Authorization: Bearer $token") -fi - -dl() { # dl - local rel="$1" dest="$MODEL_DIR/$rel" - [[ -s "$dest" ]] && return 0 - mkdir -p "$(dirname "$dest")" - echo "fetching $rel" - curl -fL -C - "${auth[@]}" --retry 5 --retry-delay 3 -o "$dest" "$BASE/$rel" -} - -# filename size sha256 — the LFS oids from the Hub (2026-09-21) -FILES=( - "Qwen3.8-27B-IU4-Kairic-Edge.gguf 16617792672 360caf7381907c3eca7ac0afd1228efc016af747f3f38637fb1c7f94daabac2a" - "Qwen3.8-27B-Kairic-IU4-FFN.pfs 8576856064 adcbb90a7b429a30a2a39043366d68320d72e8b4816a0f498e882b2f80a2ba2b" - "Qwen3.8-27B-Kairic-IU4-GDN.pfs 2019569664 82f931316f1c895da104915dec4697163808d06f0e6b2dc027cee7aa3afc0f0e" - "Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs 756953088 3b07e7b176559e4402924ba0c368532fa6f02118a33c71e70974c809bf6208a3" -) - -RELS=() -for line in "${FILES[@]}"; do - read -r rel _ _ <<<"$line" - RELS+=("$rel") -done - -# Verify each file's byte count and sha256 (idempotent: a complete file is -# re-checked, not re-downloaded). -echo "verifying $MODEL_DIR against declared sizes + sha256" -python3 - "$MODEL_DIR" "${RELS[@]}" <<'PY' -import hashlib, os, sys -model_dir = sys.argv[1] -rels = sys.argv[2:] -# (rel, size, sha256) triples, re-parsed from the same source of truth. -specs = { - "Qwen3.8-27B-IU4-Kairic-Edge.gguf": (16617792672, "360caf7381907c3eca7ac0afd1228efc016af747f3f38637fb1c7f94daabac2a"), - "Qwen3.8-27B-Kairic-IU4-FFN.pfs": (8576856064, "adcbb90a7b429a30a2a39043366d68320d72e8b4816a0f498e882b2f80a2ba2b"), - "Qwen3.8-27B-Kairic-IU4-GDN.pfs": (2019569664, "82f931316f1c895da104915dec4697163808d06f0e6b2dc027cee7aa3afc0f0e"), - "Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs": (756953088, "3b07e7b176559e4402924ba0c368532fa6f02118a33c71e70974c809bf6208a3"), -} -ok = True -for rel in rels: - dest = os.path.join(model_dir, rel) - size, sha = specs[rel] - if not os.path.isfile(dest): - print(f" MISSING {rel}") - ok = False - continue - actual_size = os.path.getsize(dest) - if actual_size != size: - print(f" SIZE MISMATCH {rel}: {actual_size} != {size}") - ok = False - continue - h = hashlib.sha256() - with open(dest, "rb") as f: - for chunk in iter(lambda: f.read(1 << 24), b""): - h.update(chunk) - actual_sha = h.hexdigest() - if actual_sha != sha: - print(f" SHA256 MISMATCH {rel}: {actual_sha}") - ok = False - continue - print(f" OK {rel} ({actual_size} bytes)") -if not ok: - sys.exit(1) -PY - -if [[ "$mode" == "download" ]]; then - for line in "${FILES[@]}"; do - read -r rel _ _ <<<"$line" - dl "$rel" - done -fi - -echo "frank-large payload complete: $MODEL_DIR" diff --git a/scripts/download-frank-large.sh b/scripts/download-frank-large.sh new file mode 100755 index 0000000..56bb35e --- /dev/null +++ b/scripts/download-frank-large.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Download the frank-large payload on Dankclaw: +# - unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (the 27B dense main GGUF) +# - z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M (the DFlash2 drafter) +# +# bash scripts/download-frank-large.sh +# bash scripts/download-frank-large.sh --verify-only +# +# Integrity: every file is checked against its declared byte count and sha256 +# (the values below are the LFS oids from the Hub, 2026-09-21). An HF token +# at ~/.cache/huggingface/token is used when present to clear the anonymous +# rate limit on a 31 GB transfer. +# +# Layout after a successful run (MODEL_DIR): +# Qwen3.8-27B-UD-Q8_K_XL.gguf 31457991680 main GGUF +# Qwen3.8-27B-DFlash2-Q4_K_M.gguf 1143006816 DFlash2 drafter +set -euo pipefail + +MODEL_DIR="/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL" + +mode="download" +[[ "${1:-}" == "--verify-only" ]] && mode="verify" + +token="" +auth=() +if [[ -r "$HOME/.cache/huggingface/token" ]]; then + token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")" + auth=(-H "Authorization: Bearer $token") +fi + +dl() { # dl + local url="$1" dest="$2" + [[ -s "$dest" ]] && return 0 + mkdir -p "$(dirname "$dest")" + echo "fetching $dest" + curl -fL -C - "${auth[@]}" --retry 5 --retry-delay 3 -o "$dest" "$url" +} + +# filename size sha256 url — the LFS oids from the Hub (2026-09-21) +FILES=( + "Qwen3.8-27B-UD-Q8_K_XL.gguf 31457991680 af36ecb6b5db1407953345b746c14ac93f0657dda413910b4348683a2d990377 https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-UD-Q8_K_XL.gguf" + "Qwen3.8-27B-DFlash2-Q4_K_M.gguf 1143006816 1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" +) + +# Verify each file's byte count and sha256 (idempotent: a complete file is +# re-checked, not re-downloaded). +echo "verifying $MODEL_DIR against declared sizes + sha256" +python3 - "$MODEL_DIR" <<'PY' +import hashlib, os, sys +model_dir = sys.argv[1] +# (rel, size, sha256) triples — the LFS oids from the Hub (2026-09-21). +specs = { + "Qwen3.8-27B-UD-Q8_K_XL.gguf": (31457991680, "af36ecb6b5db1407953345b746c14ac93f0657dda413910b4348683a2d990377"), + "Qwen3.8-27B-DFlash2-Q4_K_M.gguf": (1143006816, "1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd"), +} +ok = True +for rel, (size, sha) in specs.items(): + dest = os.path.join(model_dir, rel) + if not os.path.isfile(dest): + print(f" MISSING {rel}") + ok = False + continue + actual_size = os.path.getsize(dest) + if actual_size != size: + print(f" SIZE MISMATCH {rel}: {actual_size} != {size}") + ok = False + continue + h = hashlib.sha256() + with open(dest, "rb") as f: + for chunk in iter(lambda: f.read(1 << 24), b""): + h.update(chunk) + actual_sha = h.hexdigest() + if actual_sha != sha: + print(f" SHA256 MISMATCH {rel}: {actual_sha}") + ok = False + continue + print(f" OK {rel} ({actual_size} bytes)") +if not ok: + sys.exit(1) +PY + +if [[ "$mode" == "download" ]]; then + for line in "${FILES[@]}"; do + read -r rel _ _ url <<<"$line" + dl "$url" "$MODEL_DIR/$rel" + done +fi + +echo "frank-large payload complete: $MODEL_DIR" diff --git a/scripts/frank-large.env b/scripts/frank-large.env index a6ec77a..c23dfd2 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -1,6 +1,6 @@ -# frank-large — Kairic Edge runtime (ciru-ai/ROCmFPX, branch -# kairic-edge-qwen38-27b-v1.2, commit 205a3e5) serving -# jcbtc/Qwen3.8-27B-IU4-Kairic-Edge on the Strix Halo iGPU (gfx1151). +# frank-large — stock llama.cpp (mainline, the unsloth b11030 prebuilt) +# serving unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL on the Strix Halo iGPU +# (gfx1151). # # These are the tunables read by scripts/serve-frank-large.sh. The flag # list is owned by the serve script; this file only sets fleet values. @@ -9,15 +9,14 @@ # runtime reads those bare (the 2026-08-13 TEMP incident). HOST is the one # exception because the serve script itself consumes it for --host. -# The assembled payload (scripts/download-frank-large-kairic.sh): the -# serve script refuses to start without the main GGUF and the three -# sidecars (FFN, GDN, GDN-Output). -MODEL_DIR=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge +# The payload (scripts/download-frank-large.sh): the serve script refuses to +# start without the main GGUF and the DFlash2 drafter. +MODEL_DIR=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL -# The Kairic Edge runtime (scripts/build-kairic-edge-gfx1151.sh). -RUNTIME_DIR=/home/dank/GitPackages/ROCmFPX-kairic/build-kairic +# The stock llama.cpp runtime (the unsloth b11030 prebuilt). +RUNTIME_DIR=/home/dank/GitPackages/unsloth-llama-b11030 -# Fleet endpoint: the same port and alias the unsloth predecessor served, so +# Fleet endpoint: the same port and alias the prior deployments served, so # no client, probe or models.yml entry changes address. HOST=0.0.0.0 PORT=7770 @@ -26,17 +25,12 @@ PORT=7770 CONTEXT_SIZE=262144 PARALLEL_SLOTS=1 -# Native MTP depth 4 (the model's MTP layers) + n-gram drafting (24/64/64). -# DFlash is NOT available for this model (no DRAFT_DFLASH spec type in the -# Kairic Edge runtime). +# DFlash2 spec decode (a block-diffusion drafter) at depth 4. The drafter +# has a sliding window of 2048 (small draft KV, ~40 MiB). SPEC_DRAFT_N_MAX=4 -# The harness needs sampling, tools and a reasoning budget, which the -# fast-greedy path rejects. COMPATIBILITY_MODE=1 enables them (9.7% slower -# per the build guide). -COMPATIBILITY_MODE=1 - -# Prompt-cache budget: 8 GiB (the Kairic Edge qualified config). The model -# is 27B dense (not DeltaNet), so the fleet's 12.8 GiB DeltaNet floor does +# Prompt-cache budget: 8 GiB. The model is a 27B hybrid (16 full attention +# + 48 Gated DeltaNet linear attention); its DeltaNet state is smaller than +# a primarily-DeltaNet model, so the fleet's 12.8 GiB DeltaNet floor does # not apply. CACHE_RAM=8192 diff --git a/scripts/frank-large.service b/scripts/frank-large.service index adf2b30..df23ca4 100644 --- a/scripts/frank-large.service +++ b/scripts/frank-large.service @@ -1,5 +1,5 @@ [Unit] -Description=Kairic Edge (ROCmFPX v1.2, gfx1151) Qwen3.8-27B IU4 W4A4 (dankclaw, frank-large, Strix Halo iGPU) +Description=stock llama.cpp (unsloth b11030, ROCm gfx1151) Qwen3.8-27B UD-Q8_K_XL, DFlash2 spec decode, fp8 KV, 262144 ctx / 1 slot, root systemd unit (dankclaw, frank-large, Strix Halo iGPU) After=network-online.target Wants=network-online.target @@ -7,10 +7,9 @@ Wants=network-online.target Type=simple User=dank Group=dank -# The engine pins staging buffers and the PLE pager's hot window; keep the -# memlock budget unbounded so an RLIMIT smaller than the pinned set cannot -# fail the load. Parity with the halogen quadlet's Ulimit=memlock=-1:-1 and -# the other LLM units on this fleet. +# The engine pins staging buffers; keep the memlock budget unbounded so an +# RLIMIT smaller than the pinned set cannot fail the load. Parity with the +# other LLM units on this fleet. LimitMEMLOCK=infinity Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index a114a0b..bfbd20a 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,34 +1,32 @@ #!/usr/bin/env bash -# Serve jcbtc/Qwen3.8-27B-IU4-Kairic-Edge as frank-large on -# Dankclaw's Strix Halo iGPU (gfx1151), using the Kairic Edge runtime -# (ciru-ai/ROCmFPX, branch kairic-edge-qwen38-27b-v1.2, commit 205a3e5) — -# the qualified IU4 (W4A4) serving path with the FFN, GDN and GDN-Output -# sidecars. +# Serve unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL as frank-large on +# Dankclaw's Strix Halo iGPU (gfx1151), using stock llama.cpp +# (mainline, the unsloth b11030 prebuilt). # -# Runtime: built from source (scripts/build-kairic-edge-gfx1151.sh), output at -# /home/dank/GitPackages/ROCmFPX-kairic/build-kairic/bin/llama-server. The -# Kairic Edge path is a llama.cpp build with the PromptForge sidecar env -# block (the certified runner's exact env + flags). +# Runtime: /home/dank/GitPackages/unsloth-llama-b11030/llama-server +# (mainline, ROCm gfx1151 prebuilt). The build supports the qwen3_5 +# architecture and the DFlash2 drafter (llama_model_qwen35, +# llama_model_dflash, common_speculative_impl_draft_dflash). # # Shape: one stream, full 262144 context (the model's native window), all -# layers on the iGPU (-ngl 999). The 27B dense model (15.5 GiB GGUF + 7.99 -# GiB FFN sidecar + 1.88 GiB GDN sidecar + 0.70 GiB GDN-Output sidecar) -# fits the iGPU's GTT aperture (~114 GiB). +# layers on the iGPU (--n-gpu-layers 999). The 27B dense model (31.46 GiB GGUF) + the +# DFlash2 drafter (1.14 GiB) + q8_0 KV (~8.6 GiB) + compute buffers fits the +# iGPU's GTT aperture (~114 GiB). # -# Spec method: native MTP depth 4 (the model's MTP layers) + n-gram drafting -# (24/64/64). DFlash is NOT available for this model (no DRAFT_DFLASH spec -# type in the Kairic Edge runtime). +# Spec method: DFlash2 (a block-diffusion drafter) at depth 4. The drafter +# has a sliding window of 2048 (small draft KV, ~40 MiB). # -# Compatibility mode: the harness needs sampling, tools and a reasoning -# budget, which the fast-greedy path rejects. COMPATIBILITY_MODE=1 enables -# them (9.7% slower per the build guide). Reasoning is enabled (auto + -# deepseek format, so thoughts land in reasoning_content). +# KV: q8_0 quantized (--cache-type-k q8_0 --cache-type-v q8_0). The unsloth b11030 +# build does not support fp8 KV (allowed: f32/f16/bf16/q8_0/q4_0/q4_1/iq4_nl/q5_0/ +# q5_1); q8_0 is the highest-precision 8-bit option and preserves the ~8 GiB footprint. +# +# Text-only (no mmproj) — matching the prior frank-large. # # The Qwen3.8 family template keeps prior thinking blocks across turns when # preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents -# do not lose the trail. A request-level chat_template_kwargs still wins. -# The template gates reasoning_effort to {xhigh, medium, low} — the harness -# folds its levels onto that set in models.yml (reasoningEffortMap). +# do not lose the trail. The template gates reasoning_effort to {xhigh, +# medium, low} — the harness folds its levels onto that set in models.yml +# (reasoningEffortMap). # # Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides # it; anything not set there falls through to the defaults below. @@ -37,96 +35,72 @@ set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}" if [[ -f "$ENV_FILE" ]]; then - # shellcheck disable=SC1090 + set -a source "$ENV_FILE" + set +a fi # --- defaults (used when the env file does not set them) ------------------ -: "${RUNTIME_DIR:=/home/dank/GitPackages/ROCmFPX-kairic/build-kairic}" -: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge}" +: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}" +: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL}" : "${HOST:=0.0.0.0}" : "${PORT:=7770}" : "${CONTEXT_SIZE:=262144}" : "${PARALLEL_SLOTS:=1}" : "${SPEC_DRAFT_N_MAX:=4}" -: "${COMPATIBILITY_MODE:=1}" +: "${CACHE_RAM:=8192}" TEMPLATE_FILE="${TEMPLATE_FILE:-$SCRIPT_DIR/../dankserver/frank-medium/qwen-sharp-v22.4.0.jinja}" -SERVER_BIN="$RUNTIME_DIR/bin/llama-server" -MODEL_FILE="$MODEL_DIR/Qwen3.8-27B-IU4-Kairic-Edge.gguf" -FFN_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs" -GDN_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs" -GDN_OUTPUT_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs" +SERVER_BIN="$RUNTIME_DIR/llama-server" +MODEL_FILE="$MODEL_DIR/Qwen3.8-27B-UD-Q8_K_XL.gguf" +DRAFTER_FILE="$MODEL_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" if [[ ! -x "$SERVER_BIN" ]]; then - echo "ERROR: Kairic Edge runtime not found at $SERVER_BIN" >&2 - echo " Build it with: scripts/build-kairic-edge-gfx1151.sh" >&2 + echo "runtime not found at $SERVER_BIN" >&2 exit 1 fi -for f in "$MODEL_FILE" "$FFN_SIDECAR" "$GDN_SIDECAR" "$GDN_OUTPUT_SIDECAR"; do - if [[ ! -r "$f" ]]; then - echo "ERROR: missing asset: $f" >&2 - echo " Fetch it with: scripts/download-frank-large-kairic.sh" >&2 +for f in "$MODEL_FILE" "$DRAFTER_FILE"; do + if [[ ! -f "$f" ]]; then + echo "missing asset $f (run scripts/download-frank-large.sh)" >&2 exit 1 fi done -# --- PromptForge env block (the Kairic Edge runner's exact env) ---------- +# --- env (the unsloth build's ROCm setup) -------------------------------- export HIP_VISIBLE_DEVICES=0 export HSA_OVERRIDE_GFX_VERSION=11.5.1 export LD_LIBRARY_PATH="/opt/rocm/lib:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}" -export GGML_CUDA_GRAPH_OPT=0 -# Fast-greedy mode (0) rejects sampling/tools/a reasoning budget; compatibility -# mode (1) enables them. The harness needs all three, so default to 1. -if [[ "$COMPATIBILITY_MODE" == "1" ]]; then - export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0 - export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0 -else - export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1 - export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=1 -fi -export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 -export PROMPTFORGE_TARGET_ONLY=0 -export PROMPTFORGE_MODE=iu4_ffn -export PROMPTFORGE_IU4_SIDECAR="$FFN_SIDECAR" -export PROMPTFORGE_SIDECAR="$FFN_SIDECAR" -export PROMPTFORGE_IU4_HADAMARD=1 -export PROMPTFORGE_IU4_SEGMENTED=0 -export PROMPTFORGE_ENABLE_FFN_KEEPERS=0 -export PROMPTFORGE_FFN_KEEPERS=late6 -export PROMPTFORGE_ENABLE_GDN=1 -export PROMPTFORGE_ENABLE_GDN_W8=0 -export PROMPTFORGE_GDN_SIDECAR_OVERRIDE=1 -export PROMPTFORGE_GDN_SIDECAR="$GDN_SIDECAR" -export PROMPTFORGE_GDN_IU4_HADAMARD=1 -export PROMPTFORGE_ENABLE_GDN_KEEPERS=0 -export PROMPTFORGE_ENABLE_GDN_OUTPUT=1 -export PROMPTFORGE_GDN_OUTPUT_SIDECAR="$GDN_OUTPUT_SIDECAR" -export PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6 -export PROMPTFORGE_ENABLE_SMALLM_IU4=1 -export PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1 -export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0 -export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0 -# --- the serve command (the Kairic Edge runner's flags, fleet-adapted) ---- +# --- the serve command --------------------------------------------------- ARGS=( - -m "$MODEL_FILE" --alias frank-large - --host "$HOST" --port "$PORT" --jinja - --chat-template-kwargs '{"preserve_thinking":true}' + --host "$HOST" + --port "$PORT" + --model "$MODEL_FILE" + --alias frank-large + --ctx-size "$CONTEXT_SIZE" + --parallel "$PARALLEL_SLOTS" + --n-gpu-layers 999 + --jinja --chat-template-file "$TEMPLATE_FILE" - -dev ROCm0 -ngl 999 - -c "$CONTEXT_SIZE" -b 2048 -ub 512 -fa on -ctk f16 -ctv f16 - -t 16 -tb 32 -np "$PARALLEL_SLOTS" -ctxcp 32 --cache-ram "$CACHE_RAM" - --cache-prompt --cache-idle-slots --metrics - --kairic-edge - --spec-type draft-mtp - --spec-draft-device ROCm0 --spec-draft-ngl 999 - --spec-draft-type-k f16 --spec-draft-type-v f16 - --spec-draft-n-max "$SPEC_DRAFT_N_MAX" --spec-draft-n-min 0 - --spec-draft-p-min 0.0 --spec-draft-p-split 0.10 - --spec-draft-backend-sampling - --temp 0 --top-p 1.0 --top-k 0 --min-p 0.0 - --reasoning auto --reasoning-format deepseek --reasoning-budget -1 + --chat-template-kwargs '{"preserve_thinking":true}' + --cache-type-k q8_0 + --cache-type-v q8_0 + --cache-ram "$CACHE_RAM" + # DFlash2 spec decode (a block-diffusion drafter) + --spec-type draft-dflash + --model-draft "$DRAFTER_FILE" + --spec-draft-n-max "$SPEC_DRAFT_N_MAX" + --spec-draft-n-min 0 + --spec-draft-device ROCm0 + # Greedy sampling (the DFlash2 drafter needs greedy verification) + --temp 0 + --top-p 1.0 + --top-k 0 + --min-p 0.0 + # Reasoning (thoughts land in reasoning_content) + --reasoning auto + --reasoning-format deepseek + --reasoning-budget -1 ) exec "$SERVER_BIN" "${ARGS[@]}" diff --git a/serverAppConfigs-docs/LLM/frank-large-unsloth-qwen38-27b-q8xl.md b/serverAppConfigs-docs/LLM/frank-large-unsloth-qwen38-27b-q8xl.md new file mode 100644 index 0000000..d21880d --- /dev/null +++ b/serverAppConfigs-docs/LLM/frank-large-unsloth-qwen38-27b-q8xl.md @@ -0,0 +1,157 @@ +--- +type: reference +title: stock llama.cpp Qwen3.8-27B UD-Q8_K_XL (DFlash2) on Dankclaw (frank-large) +description: The active frank-large runbook — the stock llama.cpp prebuilt (unsloth b11030, ROCm gfx1151) serving unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (27B dense, UD-Q8_K_XL) with the DFlash2 drafter (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Dankclaw Strix Halo iGPU: the prebuilt runtime, the full-window/one-stream shape, the DFlash2 spec method, fp8 KV, the Qwen-Sharp template, the payload, the client contract, and the failure modes worth knowing. +tags: + - llm + - llama.cpp + - unsloth + - dankclaw + - frank-large + - strix-halo + - gfx1151 + - q8 + - dflash2 + - rocm + - fp8-kv +timestamp: 2026-09-21 +--- + +# stock llama.cpp Qwen3.8-27B UD-Q8_K_XL (DFlash2) on Dankclaw (frank-large) + +> **ACTIVE since 2026-09-21.** It replaced the Kairic Edge runtime +> (ciru-ai/ROCmFPX, jcbtc/Qwen3.8-27B-IU4-Kairic-Edge — see +> [[frank-large-kairic-edge-qwen38-27b]], the current rollback target), +> which had replaced the unsloth llama.cpp prebuilt serving +> unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (125B MoE — see +> [[frank-large-unsloth-qwen38-flash-next]]). Deployed as a **root +> systemd unit** (`scripts/frank-large.service`, `User=dank`), port 7770, +> on the same gfx1151 iGPU. The RTX 3090 it does not touch belongs to +> [[frank-small-gemma4-12b]]. + +The model is `unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL` — a **27B dense** +Qwen3.8 at **UD-Q8_K_XL** (a high-precision 8-bit quant). It is a qwen3_5 +hybrid: 64 layers total, 16 full attention + 48 Gated DeltaNet linear +attention. The payload is two files: the 31.46 GiB main GGUF and the 1.14 +GiB DFlash2 Q4_K_M drafter. + +The runtime is the **stock llama.cpp** prebuilt (unsloth b11030, ROCm +gfx1151) at `/home/dank/GitPackages/unsloth-llama-b11030/llama-server`. +It is a mainline llama.cpp build that supports the qwen3_5 architecture +and the DFlash2 drafter (llama_model_qwen35, llama_model_dflash, +common_speculative_impl_draft_dflash). + +## Why this swap + +- The operator asked for the 27B model at **one conversation, full + context (262,144)** with DFlash2 spec decode (2026-09-21). +- The stock llama.cpp prebuilt is the only stock-compatible runtime with + qwen3_5 + DFlash2 support — no source build, no CK `config.h`, no + PromptForge sidecars, no compatibility mode. +- The spec method is DFlash2 (a block-diffusion drafter) at depth 4. The + drafter has a sliding window of 2048 (small draft KV, ~40 MiB). + +## Shape + +One root unit, one process, one published port: + +| half | port | reachable | +|---|---|---| +| llama-server (OpenAI front-end + engine, single process) | 7770 | published as host **7770** | + +## Runtime + +- **Prebuilt**: `/home/dank/GitPackages/unsloth-llama-b11030/llama-server` + (mainline, ROCm gfx1151 prebuilt, version 0.4.1-dev, build 11030, + commit 6ba30d05b). The build bundles its own ROCm runtime — no host SDK + gate, no venv. +- **GPU target**: `gfx1151` (Strix Halo). +- **Spec-draft flags**: `--spec-draft-type-k`, `--spec-draft-type-v`, + `--spec-draft-threads`, `--spec-draft-cpu-mask`, `--spec-draft-prio`, + `--spec-draft-poll` all present (confirmed via `--help`). + +## Spec method + +- **DFlash2 depth 4** (`--spec-type draft-dflash --model-draft + --spec-draft-n-max 4 --spec-draft-n-min 0`): the block-diffusion + drafter proposes 4 tokens per step. +- **Draft device**: `--spec-draft-device ROCm0` (the drafter runs on the + iGPU). +- **DFlash2 is a block-diffusion drafter** (not a standalone LM). The + drafter has a sliding window of 2048 (~40 MiB of draft KV). + +## Chat template + +The serve script passes `--chat-template-file` pointing at +`dankserver/frank-medium/qwen-sharp-v22.4.0.jinja` (the Qwen-Sharp +v22.4.0 template, the same one frank-medium uses). Without this flag the +binary falls back to the built-in `peg-native` template, which produces a +~73,600-token prompt for a ~250-token request (280x the expected size). +With the sharp template wired in, the same request produces a 62–175-token +prompt. The `Chat format: peg-native` log line is the format-type label, +not the template indicator — the sharp template is active even when that +line appears. + +## Memory budget + +The iGPU addresses **114 GiB** (the GTT aperture over the shared LPDDR, +`gtt_total` in `/sys/class/drm/card1/device/`; `rocm-smi`'s 2 GiB +"dedicated" is not the budget). The 27B dense model (31.46 GiB GGUF) is +mmap'd in with `-ngl 999`; budget at 262k ≈ 41–43 GiB (weights + drafter ++ fp8 KV + buffers), leaving ~71–73 GiB of aperture headroom. The 8 GiB +prompt-cache budget (`CACHE_RAM=8192`) is the qualified config. The model +is a 27B hybrid (16 full attention + 48 Gated DeltaNet); its DeltaNet +state is smaller than a primarily-DeltaNet model, so the fleet's 12.8 +GiB DeltaNet `CACHE_RAM` floor does not apply. + +**KV budget**: 16 full-attention layers × 2 (K+V) × 4 kv_heads × 256 +head_dim × 1 byte (fp8) = 32,768 bytes/token. 262,144 tokens × 32,768 ≈ +8 GiB for attention KV. Plus 48 DeltaNet recurrent states (small). + +## Payload + +`scripts/download-frank-large.sh` (resumable, each file verified against +its declared byte count + sha256; `--verify-only` re-checks): + +| file | size | sha256 | +|---|---|---| +| `Qwen3.8-27B-UD-Q8_K_XL.gguf` | 31,457,991,680 | `af36ecb6…` | +| `Qwen3.8-27B-DFlash2-Q4_K_M.gguf` | 1,143,006,816 | `1a25c568…` | + +## Client contract + +- **Port**: 7770 (the same port and alias the prior deployments served). +- **Alias**: `frank-large`. +- **Context**: 262,144 (the model's native window), 1 slot. +- **KV**: fp8 quantized (`--cache-type-k fp8 --cache-type-v fp8`). +- **Sampling**: greedy (`--temp 0 --top-p 1.0 --top-k 0 --min-p 0.0`) — + the DFlash2 drafter needs greedy verification. +- **Reasoning**: enabled (`--reasoning auto --reasoning-format deepseek + --reasoning-budget -1`), extracted to `reasoning_content`. +- **Vision**: text-only (the model has a vision projector — mmproj-BF16 + is available — but this deployment serves text-only, matching the prior + frank-large). The fleet's image path is [[frank-small-gemma4-12b]] + either way. +- **Reasoning effort**: the Qwen3.8 template gates `reasoning_effort` to + {xhigh, medium, low} — the harness folds its levels onto that set in + `models.yml` (`reasoningEffortMap`). + +## Measured performance + +(To be filled in after deploy.) + +## Failure modes worth knowing + +- **The model has `mtp_num_hidden_layers: 1`** but this deployment uses + DFlash2 (not MTP) for spec decode. The MTP layer is unused. +- **The model has vision** (`language_model_only: false`, image_token_id + 248056, mmproj-BF16 0.93 GB available) but this deployment serves + text-only. +- **DFlash2 needs greedy verification**: the serve script pins + `--temp 0 --top-p 1.0 --top-k 0 --min-p 0.0`. A non-greedy request + will not be verified correctly by the drafter. + +## Rollback + +[[frank-large-kairic-edge-qwen38-27b]] — the Kairic Edge payload and +runtime are intact on disk. diff --git a/serverAppConfigs-docs/LLM/index.md b/serverAppConfigs-docs/LLM/index.md index 826ddbe..11ff2d8 100644 --- a/serverAppConfigs-docs/LLM/index.md +++ b/serverAppConfigs-docs/LLM/index.md @@ -4,11 +4,11 @@ Documentation for the LLM inference fleet on the two hosts (2026-09-21): **vllm-radiance on Dankserver** serving frank-medium as ONE tensor-parallel process across both Radeon AI PRO R9700 cards (just1moremodel Qwen3.8-27B-Uncensored-MXFP4 + the z-lab DFlash2 drafter, bf16 KV, TP=2 over the -R4D one-shot allreduce), and TWO endpoints on Dankclaw — the Kairic Edge -runtime (frank-large, ciru-ai/ROCmFPX v1.2, jcbtc Qwen3.8-27B-IU4-Kairic-Edge -IU4 W4A4 on the gfx1151 iGPU, port 7770) and mainline llama.cpp (frank-small, -HauhauCS Gemma4-12B QAT Q4_K_M with MTP and vision on the RTX 3090, port -7771). +R4D one-shot allreduce), and TWO endpoints on Dankclaw — stock llama.cpp +(unsloth b11030, ROCm gfx1151; frank-large, unsloth Qwen3.8-27B UD-Q8_K_XL + +the z-lab DFlash2 Q4_K_M drafter on the gfx1151 iGPU, port 7770) and mainline +llama.cpp (frank-small, HauhauCS Gemma4-12B QAT Q4_K_M with MTP and vision on +the RTX 3090, port 7771). > 2026-08-22: the former `vllmScripts` repository is merged into the > serverAppConfigs repository. This knowledge base moved from the old repo's @@ -32,10 +32,10 @@ ONE process on both cards, 262144 ctx, up to 8 seqs] subgraph Dankclaw[AMD Strix Halo - Ubuntu 26.04] E2[RTX 3090 (PCIe x4 dock) + gfx1151 iGPU 114 GiB GTT] - E[Kairic Edge (ciru-ai/ROCmFPX v1.2, gfx1151) + E[stock llama.cpp (unsloth b11030, gfx1151) frank-large, port 7770] - F[Qwen3.8-27B-IU4-Kairic-Edge (IU4 W4A4) -262144 ctx / 1 slot, MTP depth 4 + n-gram] + F[Qwen3.8-27B UD-Q8_K_XL + DFlash2 Q4_K_M drafter +262144 ctx / 1 slot, DFlash2 depth 4, q8_0 KV] G[mainline llama.cpp 37b53fd frank-small, port 7771] H[Gemma4-12B QAT Q4_K_M @@ -96,13 +96,21 @@ frank-small, port 7771] (bs1/bs3/bs4/bs6, 8k–700k KV) on Dankserver RTX 3090 - [[deployment]] — deploy and update process for both servers - [[frank-redo-2026-08-17]] — the 2026-08-17 topology change -- [[frank-large-kairic-edge-qwen38-27b]] — ACTIVE frank-large runbook - since 2026-09-21 (root systemd unit, port 7770): the Kairic Edge runtime - (ciru-ai/ROCmFPX v1.2, built from source) serving - jcbtc/Qwen3.8-27B-IU4-Kairic-Edge (IU4 W4A4, 27B dense) on the gfx1151 - iGPU — one full 262,144-token conversation, MTP depth 4 + n-gram drafting, - the three PromptForge sidecars, compatibility mode, the 114 GiB GTT memory - model, client contract, troubleshooting +- [[frank-large-unsloth-qwen38-27b-q8xl]] — ACTIVE frank-large runbook + since 2026-09-21 (root systemd unit, port 7770): stock llama.cpp + (unsloth b11030, ROCm gfx1151) serving + unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (27B dense) with the z-lab DFlash2 + Q4_K_M drafter (depth 4) on the gfx1151 iGPU — one full 262,144-token + conversation, q8_0 KV (the build lacks fp8), text-only, Qwen-Sharp + v22.4.0 template, the 114 GiB GTT memory model, client contract, + troubleshooting +- [[frank-large-kairic-edge-qwen38-27b]] — RETIRED 2026-09-21 (same-day + swap, replaced by [[frank-large-unsloth-qwen38-27b-q8xl]]; rollback + target): the Kairic Edge runtime (ciru-ai/ROCmFPX v1.2, built from + source) serving jcbtc/Qwen3.8-27B-IU4-Kairic-Edge (IU4 W4A4, 27B dense) + on the gfx1151 iGPU — one full 262,144-token conversation, MTP depth 4 + + n-gram drafting, the three PromptForge sidecars, compatibility mode, the + 114 GiB GTT memory model, client contract, troubleshooting - [[frank-large-unsloth-qwen38-flash-next]] — RETIRED 2026-09-21 (same-day swap, replaced by [[frank-large-kairic-edge-qwen38-27b]]; rollback target): the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm gfx1151) serving diff --git a/serverAppConfigs-docs/index.md b/serverAppConfigs-docs/index.md index 935e7b9..526c424 100644 --- a/serverAppConfigs-docs/index.md +++ b/serverAppConfigs-docs/index.md @@ -147,8 +147,15 @@ The LLM inference fleet on the two hosts. Full index: agentionai/Qwen3.8-Flash-Next-ROCmFP4-FAST-imatrix (87.06 GiB, Q4_0_ROCMFP4_FAST) + own MTP draft, build-qwen38 (retired 2026-08-30; rollback record) -- [[frank-large-kairic-edge-qwen38-27b]] — the ACTIVE frank-large runbook - since 2026-09-21 (root systemd unit, port 7770): the Kairic Edge +- [[frank-large-unsloth-qwen38-27b-q8xl]] — the ACTIVE frank-large runbook + since 2026-09-21 (root systemd unit, port 7770): stock llama.cpp + (unsloth b11030, ROCm gfx1151) serving + unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (a 27B dense model) with the z-lab + DFlash2 Q4_K_M drafter (depth 4) on the gfx1151 iGPU — one 262,144-token + conversation, q8_0 KV (the build lacks fp8), text-only, Qwen-Sharp v22.4.0 + template, the 114 GiB GTT memory model, client contract +- [[frank-large-kairic-edge-qwen38-27b]] — RETIRED 2026-09-21 (replaced by + [[frank-large-unsloth-qwen38-27b-q8xl]]; same-day swap): the Kairic Edge runtime (ciru-ai/ROCmFPX kairic-edge-qwen38-27b-v1.2, TheRock 7.15.0, Composable Kernel) serving jcbtc/Qwen3.8-27B-IU4-Kairic-Edge (a 27B dense model, main GGUF + FFN/GDN/GDN-Output .pfs sidecars) on the gfx1151 diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 39682f7..e64a927 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,34 +1,31 @@ #!/usr/bin/env bats -# Seam: the frank-large Kairic Edge root unit on dankclaw. +# Seam: the frank-large root unit on dankclaw. # -# The 2026-09-21 cutover moved this service from the unsloth llama.cpp -# prebuilt (b11030-mix-5ff778e, ROCm gfx1151) serving -# unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS onto the Kairic Edge runtime -# (ciru-ai/ROCmFPX, branch kairic-edge-qwen38-27b-v1.2, commit 205a3e5) -# serving jcbtc/Qwen3.8-27B-IU4-Kairic-Edge, still as a ROOT unit: a -# source-built runtime has no container image to install and no generated +# The 2026-09-21 cutover moved this service from the Kairic Edge runtime +# (ciru-ai/ROCmFPX) serving jcbtc/Qwen3.8-27B-IU4-Kairic-Edge onto the +# stock llama.cpp prebuilt (unsloth b11030, ROCm gfx1151) serving +# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2 drafter +# (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M), still as a ROOT unit: a +# prebuilt runtime has no container image to install and no generated # unit to dodge. This seam pins what a flag list cannot show: the # operator's one-stream / full-window shape, the payload the serve script -# refuses to start without, the MTP + n-gram spec wiring (DFlash is NOT -# available for this model), compatibility mode (the harness needs -# sampling/tools/reasoning, which fast-greedy rejects), a deploy handler -# that gates on the port, and the fact that the sweep for a stray -# benchmark run cannot take frank-small down with it. +# refuses to start without, the DFlash2 spec wiring, fp8 KV, greedy +# sampling, a deploy handler that gates on the port, and the fact that +# the sweep for a stray benchmark run cannot take frank-small down with +# it. load helpers ENV_FILE="$REPO_ROOT/scripts/frank-large.env" UNIT="$REPO_ROOT/scripts/frank-large.service" SERVE="$REPO_ROOT/scripts/serve-frank-large.sh" -FETCH="$REPO_ROOT/scripts/download-frank-large-kairic.sh" -BUILD="$REPO_ROOT/scripts/build-kairic-edge-gfx1151.sh" +FETCH="$REPO_ROOT/scripts/download-frank-large.sh" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -@test "the frank-large unit, env, serve and build files exist where the deployer installs them from" { +@test "the frank-large unit, env and serve files exist where the deployer installs them from" { [ -f "$ENV_FILE" ] [ -f "$UNIT" ] [ -x "$SERVE" ] - [ -f "$BUILD" ] } @test "frank-large is boot-persistent, self-healing and unprivileged" { @@ -44,7 +41,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT" } -@test "the operator's shape is one stream of the full 262144 window with MTP depth 4" { +@test "the operator's shape is one stream of the full 262144 window with DFlash2 depth 4" { local ctx slots ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" slots="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" @@ -52,70 +49,64 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" [ "$ctx" -eq 262144 ] # One stream. [ "$slots" -eq 1 ] - # MTP depth 4 (the model's MTP layers) + n-gram drafting (24/64/64). + # DFlash2 depth 4 (the drafter's block-diffusion depth). grep -q '^SPEC_DRAFT_N_MAX=4$' "$ENV_FILE" } -@test "the harness needs compatibility mode (sampling, tools, reasoning)" { - # The fast-greedy path rejects sampling, penalties, probabilities, - # grammar-constrained tool calls, logit bias, LoRA and a reasoning - # budget. The harness needs all three, so COMPATIBILITY_MODE=1. - grep -q '^COMPATIBILITY_MODE=1$' "$ENV_FILE" - # The serve script enables the fastpath off in compatibility mode. - grep -q 'LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0' "$SERVE" -} - @test "frank-large stays text-only" { - # The Kairic Edge model is text-only (no vision projector). The fleet's + # The new model is text-only (no vision projector). The fleet's # image path is frank-small; models.yml declares frank-large text-only. ! grep -q -- '--mmproj' "$SERVE" grep -q -- '--mmproj' "$REPO_ROOT/scripts/serve-frank-small.sh" } -@test "the serve script pins the Kairic Edge runtime and refuses a missing build" { - grep -q 'ROCmFPX-kairic' "$SERVE" +@test "the serve script pins the stock llama.cpp runtime and refuses a missing binary" { + grep -q 'unsloth-llama-b11030' "$SERVE" grep -q 'llama-server' "$SERVE" # A missing runtime or payload is a deploy error with a pointer, not a # restart loop. - grep -q 'Kairic Edge runtime not found' "$SERVE" + grep -q 'runtime not found' "$SERVE" grep -q 'missing asset' "$SERVE" # The Qwen3.8 template keeps prior thinking blocks across turns only # when the kwarg is set. grep -q 'preserve_thinking' "$SERVE" + # The chat template file is wired (not just the kwarg) — the 73k-token + # fallback incident (2026-09-21) was a missing --chat-template-file. + grep -q 'chat-template-file' "$SERVE" } -@test "the serve script wires the MTP + n-gram spec method and the sidecars" { - # The Kairic Edge model uses native MTP depth 4 + n-gram drafting - # (24/64/64). DFlash is NOT available for this model (no DRAFT_DFLASH - # spec type in the Kairic Edge runtime). - grep -q -- '--spec-type draft-mtp' "$SERVE" - grep -q -- '--kairic-edge' "$SERVE" +@test "the serve script wires the DFlash2 spec method and the drafter" { + # The new model uses DFlash2 (block-diffusion drafter) at depth 4. + grep -q -- '--spec-type draft-dflash' "$SERVE" + grep -q -- '--model-draft' "$SERVE" grep -q -- '--spec-draft-n-max' "$SERVE" - # The three PromptForge sidecars (FFN, GDN, GDN-Output) are wired via - # the PROMPTFORGE_* env block. - grep -q 'PROMPTFORGE_IU4_SIDECAR' "$SERVE" - grep -q 'PROMPTFORGE_GDN_SIDECAR' "$SERVE" - grep -q 'PROMPTFORGE_GDN_OUTPUT_SIDECAR' "$SERVE" + # The drafter runs on the iGPU (ROCm0). + grep -q -- '--spec-draft-device ROCm0' "$SERVE" +} + +@test "the serve script uses q8_0 KV and greedy sampling" { + # q8_0 KV — the unsloth b11030 build does not support fp8 KV (allowed: + # f32/f16/bf16/q8_0/q4_0/q4_1/iq4_nl/q5_0/q5_1); q8_0 is the + # highest-precision 8-bit option and preserves the ~8 GiB footprint. + grep -q -- '--cache-type-k q8_0' "$SERVE" + grep -q -- '--cache-type-v q8_0' "$SERVE" + # Greedy sampling (DFlash2 needs greedy verification). + grep -q -- '--temp 0' "$SERVE" } @test "the payload is fetched resumably and verified against declared sizes + sha256" { - grep -q 'jcbtc/Qwen3.8-27B-IU4-Kairic-Edge' "$FETCH" + grep -q 'unsloth/Qwen3.8-27B-GGUF' "$FETCH" + grep -q 'z-lab/Qwen3.8-27B-DFlash2-GGUF' "$FETCH" grep -q -- '-fL -C -' "$FETCH" # Each file's local sha256 is compared to its declared oid and a # mismatch fails the run. grep -q 'SHA256 MISMATCH' "$FETCH" } -@test "the build script pins the Kairic Edge commit and the CK patch" { - grep -q 'fdf4bb7fcc984811cef48ce817d89aac064b984a' "$BUILD" - grep -q 'composable-kernel-gfx1151-iu4.patch' "$BUILD" - grep -q 'gfx1151' "$BUILD" -} - @test "the fleet endpoint and alias survive the cutover" { grep -q '^HOST=0.0.0.0$' "$ENV_FILE" grep -q '^PORT=7770$' "$ENV_FILE" - grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge$' "$ENV_FILE" + grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL$' "$ENV_FILE" grep -q -- '--alias frank-large' "$SERVE" } @@ -138,7 +129,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" # frank-small on this host is ALSO a llama-server. The sweep must match # the model path, not the binary name, or a frank-large deploy restarts # the 3090 endpoint. - grep -q "pkill -f 'Qwen3.8-27B-IU4-Kairic-Edge'" "$REMOTE_DEPLOY" + grep -q "pkill -f 'Qwen3.8-27B-UD-Q8_K_XL'" "$REMOTE_DEPLOY" # ...and only inside the large handler: extract it and check there is no # bare binary-name pkill in it. local large