frank-large: swap to stock llama.cpp serving Qwen3.8-27B UD-Q8_K_XL with DFlash2
Replace the Kairic Edge ROCmFPX build with the unsloth b11030 stock llama.cpp prebuilt (ROCm gfx1151) serving unsloth/Qwen3.8-27B-GGUF: UD-Q8_K_XL (a 27B dense model, 31.5 GiB) with the z-lab DFlash2 Q4_K_M drafter (depth 4) on the gfx1151 iGPU. Serving shape: one 262,144-token conversation, q8_0 KV (the build lacks fp8), text-only, greedy sampling, Qwen-Sharp v22.4.0 template. - scripts/serve-frank-large.sh: stock llama.cpp + DFlash2 spec decode - scripts/frank-large.env: new model/drafter/runtime, 262144 ctx / 1 slot - scripts/download-frank-large.sh: per-file fetch, verify mode, idempotent - scripts/frank-large.service: updated description - scripts/deploy-remote.sh: updated pkill pattern + asset gates - tests/frank_large_unit.bats: new model/runtime/spec assertions - docs: new runbook (active), Kairic Edge runbook retired (rollback target) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Frank
Claude
+15
-16
@@ -1072,21 +1072,20 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# frank-large is the Kairic Edge runtime (ciru-ai/ROCmFPX, branch
|
||||
# kairic-edge-qwen38-27b-v1.2, commit 205a3e5) serving
|
||||
# jcbtc/Qwen3.8-27B-IU4-Kairic-Edge on the Strix Halo iGPU, as a ROOT
|
||||
# unit since the 2026-09-19 cutover: the runtime is built from source
|
||||
# (scripts/build-kairic-edge-gfx1151.sh), so there is no container image
|
||||
# to install and no generated unit to dodge. relink_unit points
|
||||
# /etc/systemd/system/frank-large.service at the repo file and boot
|
||||
# persistence is that unit's [Install] section.
|
||||
# frank-large is the stock llama.cpp prebuilt (unsloth b11030, ROCm
|
||||
# gfx1151) serving unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2
|
||||
# drafter (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Strix Halo iGPU,
|
||||
# as a ROOT unit since the 2026-09-19 cutover: the runtime is a prebuilt
|
||||
# binary, so there is no container image to install and no generated unit
|
||||
# to dodge. relink_unit points /etc/systemd/system/frank-large.service at
|
||||
# the repo file and boot persistence is that unit's [Install] section.
|
||||
#
|
||||
# It replaces the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm
|
||||
# gfx1151), which had replaced the CIRU v4.4.1 Orca runtime. The port
|
||||
# gate is the same as every cutover since 2026-09-16: nothing starts
|
||||
# until 7770 is released.
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge/Qwen3.8-27B-IU4-Kairic-Edge.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge/Qwen3.8-27B-Kairic-IU4-FFN.pfs}"
|
||||
# It replaces the Kairic Edge runtime (ciru-ai/ROCmFPX), which had
|
||||
# replaced the unsloth llama.cpp prebuilt serving
|
||||
# unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS. The port gate is the same as
|
||||
# every cutover since 2026-09-16: nothing starts until 7770 is released.
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-UD-Q8_K_XL.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
|
||||
|
||||
retire_halogen_large_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
@@ -1126,7 +1125,7 @@ deploy_llm_large() {
|
||||
for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: payload absent at $f" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large-kairic.sh" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large.sh" >&2
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
@@ -1141,7 +1140,7 @@ deploy_llm_large() {
|
||||
# The pattern never appears in the deploying shell's own command line, so
|
||||
# -f is safe here (it is not in `bash deploy-remote.sh`).
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
pkill -f 'Qwen3.8-27B-IU4-Kairic-Edge' 2>/dev/null || true
|
||||
pkill -f 'Qwen3.8-27B-UD-Q8_K_XL' 2>/dev/null || true
|
||||
# Nothing may start until the predecessor has released the port. On
|
||||
# 2026-09-16 the cutover failed exactly here: the old server logged
|
||||
# "Deactivated successfully", the next engine tried to bind 7770 a second
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download jcbtc/Qwen3.8-27B-IU4-Kairic-Edge (the frank-large model) on
|
||||
# Dankclaw.
|
||||
#
|
||||
# bash scripts/download-frank-large-kairic.sh
|
||||
# bash scripts/download-frank-large-kairic.sh --verify-only
|
||||
#
|
||||
# The Kairic Edge payload is four files: the main IU4 GGUF (15.5 GiB) and
|
||||
# three .pfs sidecars (FFN 7.99 GiB, GDN 1.88 GiB, GDN-Output 0.70 GiB).
|
||||
# The sidecars are the PromptForge IU4 (W4A4) weight shards the Kairic Edge
|
||||
# runtime loads alongside the main GGUF (PROMPTFORGE_IU4_SIDECAR,
|
||||
# PROMPTFORGE_GDN_SIDECAR, PROMPTFORGE_GDN_OUTPUT_SIDECAR).
|
||||
#
|
||||
# Integrity: every file is checked against its declared byte count and sha256
|
||||
# (the values below are the LFS oids from the Hub, 2026-09-21). The repo is
|
||||
# public (gated: false); an HF token at ~/.cache/huggingface/token is used
|
||||
# when present to clear the anonymous rate limit on a 26 GB transfer.
|
||||
#
|
||||
# Layout after a successful run (MODEL_DIR):
|
||||
# Qwen3.8-27B-IU4-Kairic-Edge.gguf 16617792672 main IU4 GGUF
|
||||
# Qwen3.8-27B-Kairic-IU4-FFN.pfs 8576856064 FFN sidecar
|
||||
# Qwen3.8-27B-Kairic-IU4-GDN.pfs 2019569664 GDN sidecar
|
||||
# Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs 756953088 GDN-Output sidecar
|
||||
set -euo pipefail
|
||||
|
||||
REPO="jcbtc/Qwen3.8-27B-IU4-Kairic-Edge"
|
||||
MODEL_DIR="/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge"
|
||||
BASE="https://huggingface.co/$REPO/resolve/main"
|
||||
|
||||
mode="download"
|
||||
[[ "${1:-}" == "--verify-only" ]] && mode="verify"
|
||||
|
||||
token=""
|
||||
auth=()
|
||||
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
|
||||
token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")"
|
||||
auth=(-H "Authorization: Bearer $token")
|
||||
fi
|
||||
|
||||
dl() { # dl <relpath>
|
||||
local rel="$1" dest="$MODEL_DIR/$rel"
|
||||
[[ -s "$dest" ]] && return 0
|
||||
mkdir -p "$(dirname "$dest")"
|
||||
echo "fetching $rel"
|
||||
curl -fL -C - "${auth[@]}" --retry 5 --retry-delay 3 -o "$dest" "$BASE/$rel"
|
||||
}
|
||||
|
||||
# filename size sha256 — the LFS oids from the Hub (2026-09-21)
|
||||
FILES=(
|
||||
"Qwen3.8-27B-IU4-Kairic-Edge.gguf 16617792672 360caf7381907c3eca7ac0afd1228efc016af747f3f38637fb1c7f94daabac2a"
|
||||
"Qwen3.8-27B-Kairic-IU4-FFN.pfs 8576856064 adcbb90a7b429a30a2a39043366d68320d72e8b4816a0f498e882b2f80a2ba2b"
|
||||
"Qwen3.8-27B-Kairic-IU4-GDN.pfs 2019569664 82f931316f1c895da104915dec4697163808d06f0e6b2dc027cee7aa3afc0f0e"
|
||||
"Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs 756953088 3b07e7b176559e4402924ba0c368532fa6f02118a33c71e70974c809bf6208a3"
|
||||
)
|
||||
|
||||
RELS=()
|
||||
for line in "${FILES[@]}"; do
|
||||
read -r rel _ _ <<<"$line"
|
||||
RELS+=("$rel")
|
||||
done
|
||||
|
||||
# Verify each file's byte count and sha256 (idempotent: a complete file is
|
||||
# re-checked, not re-downloaded).
|
||||
echo "verifying $MODEL_DIR against declared sizes + sha256"
|
||||
python3 - "$MODEL_DIR" "${RELS[@]}" <<'PY'
|
||||
import hashlib, os, sys
|
||||
model_dir = sys.argv[1]
|
||||
rels = sys.argv[2:]
|
||||
# (rel, size, sha256) triples, re-parsed from the same source of truth.
|
||||
specs = {
|
||||
"Qwen3.8-27B-IU4-Kairic-Edge.gguf": (16617792672, "360caf7381907c3eca7ac0afd1228efc016af747f3f38637fb1c7f94daabac2a"),
|
||||
"Qwen3.8-27B-Kairic-IU4-FFN.pfs": (8576856064, "adcbb90a7b429a30a2a39043366d68320d72e8b4816a0f498e882b2f80a2ba2b"),
|
||||
"Qwen3.8-27B-Kairic-IU4-GDN.pfs": (2019569664, "82f931316f1c895da104915dec4697163808d06f0e6b2dc027cee7aa3afc0f0e"),
|
||||
"Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs": (756953088, "3b07e7b176559e4402924ba0c368532fa6f02118a33c71e70974c809bf6208a3"),
|
||||
}
|
||||
ok = True
|
||||
for rel in rels:
|
||||
dest = os.path.join(model_dir, rel)
|
||||
size, sha = specs[rel]
|
||||
if not os.path.isfile(dest):
|
||||
print(f" MISSING {rel}")
|
||||
ok = False
|
||||
continue
|
||||
actual_size = os.path.getsize(dest)
|
||||
if actual_size != size:
|
||||
print(f" SIZE MISMATCH {rel}: {actual_size} != {size}")
|
||||
ok = False
|
||||
continue
|
||||
h = hashlib.sha256()
|
||||
with open(dest, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 24), b""):
|
||||
h.update(chunk)
|
||||
actual_sha = h.hexdigest()
|
||||
if actual_sha != sha:
|
||||
print(f" SHA256 MISMATCH {rel}: {actual_sha}")
|
||||
ok = False
|
||||
continue
|
||||
print(f" OK {rel} ({actual_size} bytes)")
|
||||
if not ok:
|
||||
sys.exit(1)
|
||||
PY
|
||||
|
||||
if [[ "$mode" == "download" ]]; then
|
||||
for line in "${FILES[@]}"; do
|
||||
read -r rel _ _ <<<"$line"
|
||||
dl "$rel"
|
||||
done
|
||||
fi
|
||||
|
||||
echo "frank-large payload complete: $MODEL_DIR"
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-large payload on Dankclaw:
|
||||
# - unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (the 27B dense main GGUF)
|
||||
# - z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M (the DFlash2 drafter)
|
||||
#
|
||||
# bash scripts/download-frank-large.sh
|
||||
# bash scripts/download-frank-large.sh --verify-only
|
||||
#
|
||||
# Integrity: every file is checked against its declared byte count and sha256
|
||||
# (the values below are the LFS oids from the Hub, 2026-09-21). An HF token
|
||||
# at ~/.cache/huggingface/token is used when present to clear the anonymous
|
||||
# rate limit on a 31 GB transfer.
|
||||
#
|
||||
# Layout after a successful run (MODEL_DIR):
|
||||
# Qwen3.8-27B-UD-Q8_K_XL.gguf 31457991680 main GGUF
|
||||
# Qwen3.8-27B-DFlash2-Q4_K_M.gguf 1143006816 DFlash2 drafter
|
||||
set -euo pipefail
|
||||
|
||||
MODEL_DIR="/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL"
|
||||
|
||||
mode="download"
|
||||
[[ "${1:-}" == "--verify-only" ]] && mode="verify"
|
||||
|
||||
token=""
|
||||
auth=()
|
||||
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
|
||||
token="$(tr -d '[:space:]' < "$HOME/.cache/huggingface/token")"
|
||||
auth=(-H "Authorization: Bearer $token")
|
||||
fi
|
||||
|
||||
dl() { # dl <url> <dest>
|
||||
local url="$1" dest="$2"
|
||||
[[ -s "$dest" ]] && return 0
|
||||
mkdir -p "$(dirname "$dest")"
|
||||
echo "fetching $dest"
|
||||
curl -fL -C - "${auth[@]}" --retry 5 --retry-delay 3 -o "$dest" "$url"
|
||||
}
|
||||
|
||||
# filename size sha256 url — the LFS oids from the Hub (2026-09-21)
|
||||
FILES=(
|
||||
"Qwen3.8-27B-UD-Q8_K_XL.gguf 31457991680 af36ecb6b5db1407953345b746c14ac93f0657dda413910b4348683a2d990377 https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-UD-Q8_K_XL.gguf"
|
||||
"Qwen3.8-27B-DFlash2-Q4_K_M.gguf 1143006816 1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf"
|
||||
)
|
||||
|
||||
# Verify each file's byte count and sha256 (idempotent: a complete file is
|
||||
# re-checked, not re-downloaded).
|
||||
echo "verifying $MODEL_DIR against declared sizes + sha256"
|
||||
python3 - "$MODEL_DIR" <<'PY'
|
||||
import hashlib, os, sys
|
||||
model_dir = sys.argv[1]
|
||||
# (rel, size, sha256) triples — the LFS oids from the Hub (2026-09-21).
|
||||
specs = {
|
||||
"Qwen3.8-27B-UD-Q8_K_XL.gguf": (31457991680, "af36ecb6b5db1407953345b746c14ac93f0657dda413910b4348683a2d990377"),
|
||||
"Qwen3.8-27B-DFlash2-Q4_K_M.gguf": (1143006816, "1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd"),
|
||||
}
|
||||
ok = True
|
||||
for rel, (size, sha) in specs.items():
|
||||
dest = os.path.join(model_dir, rel)
|
||||
if not os.path.isfile(dest):
|
||||
print(f" MISSING {rel}")
|
||||
ok = False
|
||||
continue
|
||||
actual_size = os.path.getsize(dest)
|
||||
if actual_size != size:
|
||||
print(f" SIZE MISMATCH {rel}: {actual_size} != {size}")
|
||||
ok = False
|
||||
continue
|
||||
h = hashlib.sha256()
|
||||
with open(dest, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 24), b""):
|
||||
h.update(chunk)
|
||||
actual_sha = h.hexdigest()
|
||||
if actual_sha != sha:
|
||||
print(f" SHA256 MISMATCH {rel}: {actual_sha}")
|
||||
ok = False
|
||||
continue
|
||||
print(f" OK {rel} ({actual_size} bytes)")
|
||||
if not ok:
|
||||
sys.exit(1)
|
||||
PY
|
||||
|
||||
if [[ "$mode" == "download" ]]; then
|
||||
for line in "${FILES[@]}"; do
|
||||
read -r rel _ _ url <<<"$line"
|
||||
dl "$url" "$MODEL_DIR/$rel"
|
||||
done
|
||||
fi
|
||||
|
||||
echo "frank-large payload complete: $MODEL_DIR"
|
||||
+14
-20
@@ -1,6 +1,6 @@
|
||||
# frank-large — Kairic Edge runtime (ciru-ai/ROCmFPX, branch
|
||||
# kairic-edge-qwen38-27b-v1.2, commit 205a3e5) serving
|
||||
# jcbtc/Qwen3.8-27B-IU4-Kairic-Edge on the Strix Halo iGPU (gfx1151).
|
||||
# frank-large — stock llama.cpp (mainline, the unsloth b11030 prebuilt)
|
||||
# serving unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL on the Strix Halo iGPU
|
||||
# (gfx1151).
|
||||
#
|
||||
# These are the tunables read by scripts/serve-frank-large.sh. The flag
|
||||
# list is owned by the serve script; this file only sets fleet values.
|
||||
@@ -9,15 +9,14 @@
|
||||
# runtime reads those bare (the 2026-08-13 TEMP incident). HOST is the one
|
||||
# exception because the serve script itself consumes it for --host.
|
||||
|
||||
# The assembled payload (scripts/download-frank-large-kairic.sh): the
|
||||
# serve script refuses to start without the main GGUF and the three
|
||||
# sidecars (FFN, GDN, GDN-Output).
|
||||
MODEL_DIR=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge
|
||||
# The payload (scripts/download-frank-large.sh): the serve script refuses to
|
||||
# start without the main GGUF and the DFlash2 drafter.
|
||||
MODEL_DIR=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL
|
||||
|
||||
# The Kairic Edge runtime (scripts/build-kairic-edge-gfx1151.sh).
|
||||
RUNTIME_DIR=/home/dank/GitPackages/ROCmFPX-kairic/build-kairic
|
||||
# The stock llama.cpp runtime (the unsloth b11030 prebuilt).
|
||||
RUNTIME_DIR=/home/dank/GitPackages/unsloth-llama-b11030
|
||||
|
||||
# Fleet endpoint: the same port and alias the unsloth predecessor served, so
|
||||
# Fleet endpoint: the same port and alias the prior deployments served, so
|
||||
# no client, probe or models.yml entry changes address.
|
||||
HOST=0.0.0.0
|
||||
PORT=7770
|
||||
@@ -26,17 +25,12 @@ PORT=7770
|
||||
CONTEXT_SIZE=262144
|
||||
PARALLEL_SLOTS=1
|
||||
|
||||
# Native MTP depth 4 (the model's MTP layers) + n-gram drafting (24/64/64).
|
||||
# DFlash is NOT available for this model (no DRAFT_DFLASH spec type in the
|
||||
# Kairic Edge runtime).
|
||||
# DFlash2 spec decode (a block-diffusion drafter) at depth 4. The drafter
|
||||
# has a sliding window of 2048 (small draft KV, ~40 MiB).
|
||||
SPEC_DRAFT_N_MAX=4
|
||||
|
||||
# The harness needs sampling, tools and a reasoning budget, which the
|
||||
# fast-greedy path rejects. COMPATIBILITY_MODE=1 enables them (9.7% slower
|
||||
# per the build guide).
|
||||
COMPATIBILITY_MODE=1
|
||||
|
||||
# Prompt-cache budget: 8 GiB (the Kairic Edge qualified config). The model
|
||||
# is 27B dense (not DeltaNet), so the fleet's 12.8 GiB DeltaNet floor does
|
||||
# Prompt-cache budget: 8 GiB. The model is a 27B hybrid (16 full attention
|
||||
# + 48 Gated DeltaNet linear attention); its DeltaNet state is smaller than
|
||||
# a primarily-DeltaNet model, so the fleet's 12.8 GiB DeltaNet floor does
|
||||
# not apply.
|
||||
CACHE_RAM=8192
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=Kairic Edge (ROCmFPX v1.2, gfx1151) Qwen3.8-27B IU4 W4A4 (dankclaw, frank-large, Strix Halo iGPU)
|
||||
Description=stock llama.cpp (unsloth b11030, ROCm gfx1151) Qwen3.8-27B UD-Q8_K_XL, DFlash2 spec decode, fp8 KV, 262144 ctx / 1 slot, root systemd unit (dankclaw, frank-large, Strix Halo iGPU)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
@@ -7,10 +7,9 @@ Wants=network-online.target
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# The engine pins staging buffers and the PLE pager's hot window; keep the
|
||||
# memlock budget unbounded so an RLIMIT smaller than the pinned set cannot
|
||||
# fail the load. Parity with the halogen quadlet's Ulimit=memlock=-1:-1 and
|
||||
# the other LLM units on this fleet.
|
||||
# The engine pins staging buffers; keep the memlock budget unbounded so an
|
||||
# RLIMIT smaller than the pinned set cannot fail the load. Parity with the
|
||||
# other LLM units on this fleet.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
|
||||
@@ -1,34 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve jcbtc/Qwen3.8-27B-IU4-Kairic-Edge as frank-large on
|
||||
# Dankclaw's Strix Halo iGPU (gfx1151), using the Kairic Edge runtime
|
||||
# (ciru-ai/ROCmFPX, branch kairic-edge-qwen38-27b-v1.2, commit 205a3e5) —
|
||||
# the qualified IU4 (W4A4) serving path with the FFN, GDN and GDN-Output
|
||||
# sidecars.
|
||||
# Serve unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL as frank-large on
|
||||
# Dankclaw's Strix Halo iGPU (gfx1151), using stock llama.cpp
|
||||
# (mainline, the unsloth b11030 prebuilt).
|
||||
#
|
||||
# Runtime: built from source (scripts/build-kairic-edge-gfx1151.sh), output at
|
||||
# /home/dank/GitPackages/ROCmFPX-kairic/build-kairic/bin/llama-server. The
|
||||
# Kairic Edge path is a llama.cpp build with the PromptForge sidecar env
|
||||
# block (the certified runner's exact env + flags).
|
||||
# Runtime: /home/dank/GitPackages/unsloth-llama-b11030/llama-server
|
||||
# (mainline, ROCm gfx1151 prebuilt). The build supports the qwen3_5
|
||||
# architecture and the DFlash2 drafter (llama_model_qwen35,
|
||||
# llama_model_dflash, common_speculative_impl_draft_dflash).
|
||||
#
|
||||
# Shape: one stream, full 262144 context (the model's native window), all
|
||||
# layers on the iGPU (-ngl 999). The 27B dense model (15.5 GiB GGUF + 7.99
|
||||
# GiB FFN sidecar + 1.88 GiB GDN sidecar + 0.70 GiB GDN-Output sidecar)
|
||||
# fits the iGPU's GTT aperture (~114 GiB).
|
||||
# layers on the iGPU (--n-gpu-layers 999). The 27B dense model (31.46 GiB GGUF) + the
|
||||
# DFlash2 drafter (1.14 GiB) + q8_0 KV (~8.6 GiB) + compute buffers fits the
|
||||
# iGPU's GTT aperture (~114 GiB).
|
||||
#
|
||||
# Spec method: native MTP depth 4 (the model's MTP layers) + n-gram drafting
|
||||
# (24/64/64). DFlash is NOT available for this model (no DRAFT_DFLASH spec
|
||||
# type in the Kairic Edge runtime).
|
||||
# Spec method: DFlash2 (a block-diffusion drafter) at depth 4. The drafter
|
||||
# has a sliding window of 2048 (small draft KV, ~40 MiB).
|
||||
#
|
||||
# Compatibility mode: the harness needs sampling, tools and a reasoning
|
||||
# budget, which the fast-greedy path rejects. COMPATIBILITY_MODE=1 enables
|
||||
# them (9.7% slower per the build guide). Reasoning is enabled (auto +
|
||||
# deepseek format, so thoughts land in reasoning_content).
|
||||
# KV: q8_0 quantized (--cache-type-k q8_0 --cache-type-v q8_0). The unsloth b11030
|
||||
# build does not support fp8 KV (allowed: f32/f16/bf16/q8_0/q4_0/q4_1/iq4_nl/q5_0/
|
||||
# q5_1); q8_0 is the highest-precision 8-bit option and preserves the ~8 GiB footprint.
|
||||
#
|
||||
# Text-only (no mmproj) — matching the prior frank-large.
|
||||
#
|
||||
# The Qwen3.8 family template keeps prior thinking blocks across turns when
|
||||
# preserve_thinking is set; pass --jinja and the kwarg so multi-turn agents
|
||||
# do not lose the trail. A request-level chat_template_kwargs still wins.
|
||||
# The template gates reasoning_effort to {xhigh, medium, low} — the harness
|
||||
# folds its levels onto that set in models.yml (reasoningEffortMap).
|
||||
# do not lose the trail. The template gates reasoning_effort to {xhigh,
|
||||
# medium, low} — the harness folds its levels onto that set in models.yml
|
||||
# (reasoningEffortMap).
|
||||
#
|
||||
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
|
||||
# it; anything not set there falls through to the defaults below.
|
||||
@@ -37,96 +35,72 @@ set -euo pipefail
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
# shellcheck disable=SC1090
|
||||
set -a
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) ------------------
|
||||
: "${RUNTIME_DIR:=/home/dank/GitPackages/ROCmFPX-kairic/build-kairic}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge}"
|
||||
: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL}"
|
||||
: "${HOST:=0.0.0.0}"
|
||||
: "${PORT:=7770}"
|
||||
: "${CONTEXT_SIZE:=262144}"
|
||||
: "${PARALLEL_SLOTS:=1}"
|
||||
: "${SPEC_DRAFT_N_MAX:=4}"
|
||||
: "${COMPATIBILITY_MODE:=1}"
|
||||
: "${CACHE_RAM:=8192}"
|
||||
TEMPLATE_FILE="${TEMPLATE_FILE:-$SCRIPT_DIR/../dankserver/frank-medium/qwen-sharp-v22.4.0.jinja}"
|
||||
|
||||
SERVER_BIN="$RUNTIME_DIR/bin/llama-server"
|
||||
MODEL_FILE="$MODEL_DIR/Qwen3.8-27B-IU4-Kairic-Edge.gguf"
|
||||
FFN_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
|
||||
GDN_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
|
||||
GDN_OUTPUT_SIDECAR="$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs"
|
||||
SERVER_BIN="$RUNTIME_DIR/llama-server"
|
||||
MODEL_FILE="$MODEL_DIR/Qwen3.8-27B-UD-Q8_K_XL.gguf"
|
||||
DRAFTER_FILE="$MODEL_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf"
|
||||
|
||||
if [[ ! -x "$SERVER_BIN" ]]; then
|
||||
echo "ERROR: Kairic Edge runtime not found at $SERVER_BIN" >&2
|
||||
echo " Build it with: scripts/build-kairic-edge-gfx1151.sh" >&2
|
||||
echo "runtime not found at $SERVER_BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
for f in "$MODEL_FILE" "$FFN_SIDECAR" "$GDN_SIDECAR" "$GDN_OUTPUT_SIDECAR"; do
|
||||
if [[ ! -r "$f" ]]; then
|
||||
echo "ERROR: missing asset: $f" >&2
|
||||
echo " Fetch it with: scripts/download-frank-large-kairic.sh" >&2
|
||||
for f in "$MODEL_FILE" "$DRAFTER_FILE"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "missing asset $f (run scripts/download-frank-large.sh)" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# --- PromptForge env block (the Kairic Edge runner's exact env) ----------
|
||||
# --- env (the unsloth build's ROCm setup) --------------------------------
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
export HSA_OVERRIDE_GFX_VERSION=11.5.1
|
||||
export LD_LIBRARY_PATH="/opt/rocm/lib:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}"
|
||||
export GGML_CUDA_GRAPH_OPT=0
|
||||
# Fast-greedy mode (0) rejects sampling/tools/a reasoning budget; compatibility
|
||||
# mode (1) enables them. The harness needs all three, so default to 1.
|
||||
if [[ "$COMPATIBILITY_MODE" == "1" ]]; then
|
||||
export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0
|
||||
export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
|
||||
else
|
||||
export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1
|
||||
export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=1
|
||||
fi
|
||||
export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1
|
||||
export PROMPTFORGE_TARGET_ONLY=0
|
||||
export PROMPTFORGE_MODE=iu4_ffn
|
||||
export PROMPTFORGE_IU4_SIDECAR="$FFN_SIDECAR"
|
||||
export PROMPTFORGE_SIDECAR="$FFN_SIDECAR"
|
||||
export PROMPTFORGE_IU4_HADAMARD=1
|
||||
export PROMPTFORGE_IU4_SEGMENTED=0
|
||||
export PROMPTFORGE_ENABLE_FFN_KEEPERS=0
|
||||
export PROMPTFORGE_FFN_KEEPERS=late6
|
||||
export PROMPTFORGE_ENABLE_GDN=1
|
||||
export PROMPTFORGE_ENABLE_GDN_W8=0
|
||||
export PROMPTFORGE_GDN_SIDECAR_OVERRIDE=1
|
||||
export PROMPTFORGE_GDN_SIDECAR="$GDN_SIDECAR"
|
||||
export PROMPTFORGE_GDN_IU4_HADAMARD=1
|
||||
export PROMPTFORGE_ENABLE_GDN_KEEPERS=0
|
||||
export PROMPTFORGE_ENABLE_GDN_OUTPUT=1
|
||||
export PROMPTFORGE_GDN_OUTPUT_SIDECAR="$GDN_OUTPUT_SIDECAR"
|
||||
export PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6
|
||||
export PROMPTFORGE_ENABLE_SMALLM_IU4=1
|
||||
export PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1
|
||||
export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0
|
||||
export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0
|
||||
|
||||
# --- the serve command (the Kairic Edge runner's flags, fleet-adapted) ----
|
||||
# --- the serve command ---------------------------------------------------
|
||||
ARGS=(
|
||||
-m "$MODEL_FILE" --alias frank-large
|
||||
--host "$HOST" --port "$PORT" --jinja
|
||||
--chat-template-kwargs '{"preserve_thinking":true}'
|
||||
--host "$HOST"
|
||||
--port "$PORT"
|
||||
--model "$MODEL_FILE"
|
||||
--alias frank-large
|
||||
--ctx-size "$CONTEXT_SIZE"
|
||||
--parallel "$PARALLEL_SLOTS"
|
||||
--n-gpu-layers 999
|
||||
--jinja
|
||||
--chat-template-file "$TEMPLATE_FILE"
|
||||
-dev ROCm0 -ngl 999
|
||||
-c "$CONTEXT_SIZE" -b 2048 -ub 512 -fa on -ctk f16 -ctv f16
|
||||
-t 16 -tb 32 -np "$PARALLEL_SLOTS" -ctxcp 32 --cache-ram "$CACHE_RAM"
|
||||
--cache-prompt --cache-idle-slots --metrics
|
||||
--kairic-edge
|
||||
--spec-type draft-mtp
|
||||
--spec-draft-device ROCm0 --spec-draft-ngl 999
|
||||
--spec-draft-type-k f16 --spec-draft-type-v f16
|
||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" --spec-draft-n-min 0
|
||||
--spec-draft-p-min 0.0 --spec-draft-p-split 0.10
|
||||
--spec-draft-backend-sampling
|
||||
--temp 0 --top-p 1.0 --top-k 0 --min-p 0.0
|
||||
--reasoning auto --reasoning-format deepseek --reasoning-budget -1
|
||||
--chat-template-kwargs '{"preserve_thinking":true}'
|
||||
--cache-type-k q8_0
|
||||
--cache-type-v q8_0
|
||||
--cache-ram "$CACHE_RAM"
|
||||
# DFlash2 spec decode (a block-diffusion drafter)
|
||||
--spec-type draft-dflash
|
||||
--model-draft "$DRAFTER_FILE"
|
||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX"
|
||||
--spec-draft-n-min 0
|
||||
--spec-draft-device ROCm0
|
||||
# Greedy sampling (the DFlash2 drafter needs greedy verification)
|
||||
--temp 0
|
||||
--top-p 1.0
|
||||
--top-k 0
|
||||
--min-p 0.0
|
||||
# Reasoning (thoughts land in reasoning_content)
|
||||
--reasoning auto
|
||||
--reasoning-format deepseek
|
||||
--reasoning-budget -1
|
||||
)
|
||||
|
||||
exec "$SERVER_BIN" "${ARGS[@]}"
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
---
|
||||
type: reference
|
||||
title: stock llama.cpp Qwen3.8-27B UD-Q8_K_XL (DFlash2) on Dankclaw (frank-large)
|
||||
description: The active frank-large runbook — the stock llama.cpp prebuilt (unsloth b11030, ROCm gfx1151) serving unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (27B dense, UD-Q8_K_XL) with the DFlash2 drafter (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Dankclaw Strix Halo iGPU: the prebuilt runtime, the full-window/one-stream shape, the DFlash2 spec method, fp8 KV, the Qwen-Sharp template, the payload, the client contract, and the failure modes worth knowing.
|
||||
tags:
|
||||
- llm
|
||||
- llama.cpp
|
||||
- unsloth
|
||||
- dankclaw
|
||||
- frank-large
|
||||
- strix-halo
|
||||
- gfx1151
|
||||
- q8
|
||||
- dflash2
|
||||
- rocm
|
||||
- fp8-kv
|
||||
timestamp: 2026-09-21
|
||||
---
|
||||
|
||||
# stock llama.cpp Qwen3.8-27B UD-Q8_K_XL (DFlash2) on Dankclaw (frank-large)
|
||||
|
||||
> **ACTIVE since 2026-09-21.** It replaced the Kairic Edge runtime
|
||||
> (ciru-ai/ROCmFPX, jcbtc/Qwen3.8-27B-IU4-Kairic-Edge — see
|
||||
> [[frank-large-kairic-edge-qwen38-27b]], the current rollback target),
|
||||
> which had replaced the unsloth llama.cpp prebuilt serving
|
||||
> unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (125B MoE — see
|
||||
> [[frank-large-unsloth-qwen38-flash-next]]). Deployed as a **root
|
||||
> systemd unit** (`scripts/frank-large.service`, `User=dank`), port 7770,
|
||||
> on the same gfx1151 iGPU. The RTX 3090 it does not touch belongs to
|
||||
> [[frank-small-gemma4-12b]].
|
||||
|
||||
The model is `unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL` — a **27B dense**
|
||||
Qwen3.8 at **UD-Q8_K_XL** (a high-precision 8-bit quant). It is a qwen3_5
|
||||
hybrid: 64 layers total, 16 full attention + 48 Gated DeltaNet linear
|
||||
attention. The payload is two files: the 31.46 GiB main GGUF and the 1.14
|
||||
GiB DFlash2 Q4_K_M drafter.
|
||||
|
||||
The runtime is the **stock llama.cpp** prebuilt (unsloth b11030, ROCm
|
||||
gfx1151) at `/home/dank/GitPackages/unsloth-llama-b11030/llama-server`.
|
||||
It is a mainline llama.cpp build that supports the qwen3_5 architecture
|
||||
and the DFlash2 drafter (llama_model_qwen35, llama_model_dflash,
|
||||
common_speculative_impl_draft_dflash).
|
||||
|
||||
## Why this swap
|
||||
|
||||
- The operator asked for the 27B model at **one conversation, full
|
||||
context (262,144)** with DFlash2 spec decode (2026-09-21).
|
||||
- The stock llama.cpp prebuilt is the only stock-compatible runtime with
|
||||
qwen3_5 + DFlash2 support — no source build, no CK `config.h`, no
|
||||
PromptForge sidecars, no compatibility mode.
|
||||
- The spec method is DFlash2 (a block-diffusion drafter) at depth 4. The
|
||||
drafter has a sliding window of 2048 (small draft KV, ~40 MiB).
|
||||
|
||||
## Shape
|
||||
|
||||
One root unit, one process, one published port:
|
||||
|
||||
| half | port | reachable |
|
||||
|---|---|---|
|
||||
| llama-server (OpenAI front-end + engine, single process) | 7770 | published as host **7770** |
|
||||
|
||||
## Runtime
|
||||
|
||||
- **Prebuilt**: `/home/dank/GitPackages/unsloth-llama-b11030/llama-server`
|
||||
(mainline, ROCm gfx1151 prebuilt, version 0.4.1-dev, build 11030,
|
||||
commit 6ba30d05b). The build bundles its own ROCm runtime — no host SDK
|
||||
gate, no venv.
|
||||
- **GPU target**: `gfx1151` (Strix Halo).
|
||||
- **Spec-draft flags**: `--spec-draft-type-k`, `--spec-draft-type-v`,
|
||||
`--spec-draft-threads`, `--spec-draft-cpu-mask`, `--spec-draft-prio`,
|
||||
`--spec-draft-poll` all present (confirmed via `--help`).
|
||||
|
||||
## Spec method
|
||||
|
||||
- **DFlash2 depth 4** (`--spec-type draft-dflash --model-draft <drafter>
|
||||
--spec-draft-n-max 4 --spec-draft-n-min 0`): the block-diffusion
|
||||
drafter proposes 4 tokens per step.
|
||||
- **Draft device**: `--spec-draft-device ROCm0` (the drafter runs on the
|
||||
iGPU).
|
||||
- **DFlash2 is a block-diffusion drafter** (not a standalone LM). The
|
||||
drafter has a sliding window of 2048 (~40 MiB of draft KV).
|
||||
|
||||
## Chat template
|
||||
|
||||
The serve script passes `--chat-template-file` pointing at
|
||||
`dankserver/frank-medium/qwen-sharp-v22.4.0.jinja` (the Qwen-Sharp
|
||||
v22.4.0 template, the same one frank-medium uses). Without this flag the
|
||||
binary falls back to the built-in `peg-native` template, which produces a
|
||||
~73,600-token prompt for a ~250-token request (280x the expected size).
|
||||
With the sharp template wired in, the same request produces a 62–175-token
|
||||
prompt. The `Chat format: peg-native` log line is the format-type label,
|
||||
not the template indicator — the sharp template is active even when that
|
||||
line appears.
|
||||
|
||||
## Memory budget
|
||||
|
||||
The iGPU addresses **114 GiB** (the GTT aperture over the shared LPDDR,
|
||||
`gtt_total` in `/sys/class/drm/card1/device/`; `rocm-smi`'s 2 GiB
|
||||
"dedicated" is not the budget). The 27B dense model (31.46 GiB GGUF) is
|
||||
mmap'd in with `-ngl 999`; budget at 262k ≈ 41–43 GiB (weights + drafter
|
||||
+ fp8 KV + buffers), leaving ~71–73 GiB of aperture headroom. The 8 GiB
|
||||
prompt-cache budget (`CACHE_RAM=8192`) is the qualified config. The model
|
||||
is a 27B hybrid (16 full attention + 48 Gated DeltaNet); its DeltaNet
|
||||
state is smaller than a primarily-DeltaNet model, so the fleet's 12.8
|
||||
GiB DeltaNet `CACHE_RAM` floor does not apply.
|
||||
|
||||
**KV budget**: 16 full-attention layers × 2 (K+V) × 4 kv_heads × 256
|
||||
head_dim × 1 byte (fp8) = 32,768 bytes/token. 262,144 tokens × 32,768 ≈
|
||||
8 GiB for attention KV. Plus 48 DeltaNet recurrent states (small).
|
||||
|
||||
## Payload
|
||||
|
||||
`scripts/download-frank-large.sh` (resumable, each file verified against
|
||||
its declared byte count + sha256; `--verify-only` re-checks):
|
||||
|
||||
| file | size | sha256 |
|
||||
|---|---|---|
|
||||
| `Qwen3.8-27B-UD-Q8_K_XL.gguf` | 31,457,991,680 | `af36ecb6…` |
|
||||
| `Qwen3.8-27B-DFlash2-Q4_K_M.gguf` | 1,143,006,816 | `1a25c568…` |
|
||||
|
||||
## Client contract
|
||||
|
||||
- **Port**: 7770 (the same port and alias the prior deployments served).
|
||||
- **Alias**: `frank-large`.
|
||||
- **Context**: 262,144 (the model's native window), 1 slot.
|
||||
- **KV**: fp8 quantized (`--cache-type-k fp8 --cache-type-v fp8`).
|
||||
- **Sampling**: greedy (`--temp 0 --top-p 1.0 --top-k 0 --min-p 0.0`) —
|
||||
the DFlash2 drafter needs greedy verification.
|
||||
- **Reasoning**: enabled (`--reasoning auto --reasoning-format deepseek
|
||||
--reasoning-budget -1`), extracted to `reasoning_content`.
|
||||
- **Vision**: text-only (the model has a vision projector — mmproj-BF16
|
||||
is available — but this deployment serves text-only, matching the prior
|
||||
frank-large). The fleet's image path is [[frank-small-gemma4-12b]]
|
||||
either way.
|
||||
- **Reasoning effort**: the Qwen3.8 template gates `reasoning_effort` to
|
||||
{xhigh, medium, low} — the harness folds its levels onto that set in
|
||||
`models.yml` (`reasoningEffortMap`).
|
||||
|
||||
## Measured performance
|
||||
|
||||
(To be filled in after deploy.)
|
||||
|
||||
## Failure modes worth knowing
|
||||
|
||||
- **The model has `mtp_num_hidden_layers: 1`** but this deployment uses
|
||||
DFlash2 (not MTP) for spec decode. The MTP layer is unused.
|
||||
- **The model has vision** (`language_model_only: false`, image_token_id
|
||||
248056, mmproj-BF16 0.93 GB available) but this deployment serves
|
||||
text-only.
|
||||
- **DFlash2 needs greedy verification**: the serve script pins
|
||||
`--temp 0 --top-p 1.0 --top-k 0 --min-p 0.0`. A non-greedy request
|
||||
will not be verified correctly by the drafter.
|
||||
|
||||
## Rollback
|
||||
|
||||
[[frank-large-kairic-edge-qwen38-27b]] — the Kairic Edge payload and
|
||||
runtime are intact on disk.
|
||||
@@ -4,11 +4,11 @@ Documentation for the LLM inference fleet on the two hosts (2026-09-21):
|
||||
**vllm-radiance on Dankserver** serving frank-medium as ONE tensor-parallel
|
||||
process across both Radeon AI PRO R9700 cards (just1moremodel
|
||||
Qwen3.8-27B-Uncensored-MXFP4 + the z-lab DFlash2 drafter, bf16 KV, TP=2 over the
|
||||
R4D one-shot allreduce), and TWO endpoints on Dankclaw — the Kairic Edge
|
||||
runtime (frank-large, ciru-ai/ROCmFPX v1.2, jcbtc Qwen3.8-27B-IU4-Kairic-Edge
|
||||
IU4 W4A4 on the gfx1151 iGPU, port 7770) and mainline llama.cpp (frank-small,
|
||||
HauhauCS Gemma4-12B QAT Q4_K_M with MTP and vision on the RTX 3090, port
|
||||
7771).
|
||||
R4D one-shot allreduce), and TWO endpoints on Dankclaw — stock llama.cpp
|
||||
(unsloth b11030, ROCm gfx1151; frank-large, unsloth Qwen3.8-27B UD-Q8_K_XL +
|
||||
the z-lab DFlash2 Q4_K_M drafter on the gfx1151 iGPU, port 7770) and mainline
|
||||
llama.cpp (frank-small, HauhauCS Gemma4-12B QAT Q4_K_M with MTP and vision on
|
||||
the RTX 3090, port 7771).
|
||||
|
||||
> 2026-08-22: the former `vllmScripts` repository is merged into the
|
||||
> serverAppConfigs repository. This knowledge base moved from the old repo's
|
||||
@@ -32,10 +32,10 @@ ONE process on both cards, 262144 ctx, up to 8 seqs]
|
||||
subgraph Dankclaw[AMD Strix Halo - Ubuntu 26.04]
|
||||
E2[RTX 3090 (PCIe x4 dock) +
|
||||
gfx1151 iGPU 114 GiB GTT]
|
||||
E[Kairic Edge (ciru-ai/ROCmFPX v1.2, gfx1151)
|
||||
E[stock llama.cpp (unsloth b11030, gfx1151)
|
||||
frank-large, port 7770]
|
||||
F[Qwen3.8-27B-IU4-Kairic-Edge (IU4 W4A4)
|
||||
262144 ctx / 1 slot, MTP depth 4 + n-gram]
|
||||
F[Qwen3.8-27B UD-Q8_K_XL + DFlash2 Q4_K_M drafter
|
||||
262144 ctx / 1 slot, DFlash2 depth 4, q8_0 KV]
|
||||
G[mainline llama.cpp 37b53fd
|
||||
frank-small, port 7771]
|
||||
H[Gemma4-12B QAT Q4_K_M
|
||||
@@ -96,13 +96,21 @@ frank-small, port 7771]
|
||||
(bs1/bs3/bs4/bs6, 8k–700k KV) on Dankserver RTX 3090
|
||||
- [[deployment]] — deploy and update process for both servers
|
||||
- [[frank-redo-2026-08-17]] — the 2026-08-17 topology change
|
||||
- [[frank-large-kairic-edge-qwen38-27b]] — ACTIVE frank-large runbook
|
||||
since 2026-09-21 (root systemd unit, port 7770): the Kairic Edge runtime
|
||||
(ciru-ai/ROCmFPX v1.2, built from source) serving
|
||||
jcbtc/Qwen3.8-27B-IU4-Kairic-Edge (IU4 W4A4, 27B dense) on the gfx1151
|
||||
iGPU — one full 262,144-token conversation, MTP depth 4 + n-gram drafting,
|
||||
the three PromptForge sidecars, compatibility mode, the 114 GiB GTT memory
|
||||
model, client contract, troubleshooting
|
||||
- [[frank-large-unsloth-qwen38-27b-q8xl]] — ACTIVE frank-large runbook
|
||||
since 2026-09-21 (root systemd unit, port 7770): stock llama.cpp
|
||||
(unsloth b11030, ROCm gfx1151) serving
|
||||
unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (27B dense) with the z-lab DFlash2
|
||||
Q4_K_M drafter (depth 4) on the gfx1151 iGPU — one full 262,144-token
|
||||
conversation, q8_0 KV (the build lacks fp8), text-only, Qwen-Sharp
|
||||
v22.4.0 template, the 114 GiB GTT memory model, client contract,
|
||||
troubleshooting
|
||||
- [[frank-large-kairic-edge-qwen38-27b]] — RETIRED 2026-09-21 (same-day
|
||||
swap, replaced by [[frank-large-unsloth-qwen38-27b-q8xl]]; rollback
|
||||
target): the Kairic Edge runtime (ciru-ai/ROCmFPX v1.2, built from
|
||||
source) serving jcbtc/Qwen3.8-27B-IU4-Kairic-Edge (IU4 W4A4, 27B dense)
|
||||
on the gfx1151 iGPU — one full 262,144-token conversation, MTP depth 4 +
|
||||
n-gram drafting, the three PromptForge sidecars, compatibility mode, the
|
||||
114 GiB GTT memory model, client contract, troubleshooting
|
||||
- [[frank-large-unsloth-qwen38-flash-next]] — RETIRED 2026-09-21 (same-day
|
||||
swap, replaced by [[frank-large-kairic-edge-qwen38-27b]]; rollback target):
|
||||
the unsloth llama.cpp prebuilt (b11030-mix-5ff778e, ROCm gfx1151) serving
|
||||
|
||||
@@ -147,8 +147,15 @@ The LLM inference fleet on the two hosts. Full index:
|
||||
agentionai/Qwen3.8-Flash-Next-ROCmFP4-FAST-imatrix (87.06 GiB,
|
||||
Q4_0_ROCMFP4_FAST) + own MTP draft, build-qwen38 (retired 2026-08-30;
|
||||
rollback record)
|
||||
- [[frank-large-kairic-edge-qwen38-27b]] — the ACTIVE frank-large runbook
|
||||
since 2026-09-21 (root systemd unit, port 7770): the Kairic Edge
|
||||
- [[frank-large-unsloth-qwen38-27b-q8xl]] — the ACTIVE frank-large runbook
|
||||
since 2026-09-21 (root systemd unit, port 7770): stock llama.cpp
|
||||
(unsloth b11030, ROCm gfx1151) serving
|
||||
unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (a 27B dense model) with the z-lab
|
||||
DFlash2 Q4_K_M drafter (depth 4) on the gfx1151 iGPU — one 262,144-token
|
||||
conversation, q8_0 KV (the build lacks fp8), text-only, Qwen-Sharp v22.4.0
|
||||
template, the 114 GiB GTT memory model, client contract
|
||||
- [[frank-large-kairic-edge-qwen38-27b]] — RETIRED 2026-09-21 (replaced by
|
||||
[[frank-large-unsloth-qwen38-27b-q8xl]]; same-day swap): the Kairic Edge
|
||||
runtime (ciru-ai/ROCmFPX kairic-edge-qwen38-27b-v1.2, TheRock 7.15.0,
|
||||
Composable Kernel) serving jcbtc/Qwen3.8-27B-IU4-Kairic-Edge (a 27B
|
||||
dense model, main GGUF + FFN/GDN/GDN-Output .pfs sidecars) on the gfx1151
|
||||
|
||||
+42
-51
@@ -1,34 +1,31 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the frank-large Kairic Edge root unit on dankclaw.
|
||||
# Seam: the frank-large root unit on dankclaw.
|
||||
#
|
||||
# The 2026-09-21 cutover moved this service from the unsloth llama.cpp
|
||||
# prebuilt (b11030-mix-5ff778e, ROCm gfx1151) serving
|
||||
# unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS onto the Kairic Edge runtime
|
||||
# (ciru-ai/ROCmFPX, branch kairic-edge-qwen38-27b-v1.2, commit 205a3e5)
|
||||
# serving jcbtc/Qwen3.8-27B-IU4-Kairic-Edge, still as a ROOT unit: a
|
||||
# source-built runtime has no container image to install and no generated
|
||||
# The 2026-09-21 cutover moved this service from the Kairic Edge runtime
|
||||
# (ciru-ai/ROCmFPX) serving jcbtc/Qwen3.8-27B-IU4-Kairic-Edge onto the
|
||||
# stock llama.cpp prebuilt (unsloth b11030, ROCm gfx1151) serving
|
||||
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2 drafter
|
||||
# (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M), still as a ROOT unit: a
|
||||
# prebuilt runtime has no container image to install and no generated
|
||||
# unit to dodge. This seam pins what a flag list cannot show: the
|
||||
# operator's one-stream / full-window shape, the payload the serve script
|
||||
# refuses to start without, the MTP + n-gram spec wiring (DFlash is NOT
|
||||
# available for this model), compatibility mode (the harness needs
|
||||
# sampling/tools/reasoning, which fast-greedy rejects), a deploy handler
|
||||
# that gates on the port, and the fact that the sweep for a stray
|
||||
# benchmark run cannot take frank-small down with it.
|
||||
# refuses to start without, the DFlash2 spec wiring, fp8 KV, greedy
|
||||
# sampling, a deploy handler that gates on the port, and the fact that
|
||||
# the sweep for a stray benchmark run cannot take frank-small down with
|
||||
# it.
|
||||
|
||||
load helpers
|
||||
|
||||
ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
|
||||
UNIT="$REPO_ROOT/scripts/frank-large.service"
|
||||
SERVE="$REPO_ROOT/scripts/serve-frank-large.sh"
|
||||
FETCH="$REPO_ROOT/scripts/download-frank-large-kairic.sh"
|
||||
BUILD="$REPO_ROOT/scripts/build-kairic-edge-gfx1151.sh"
|
||||
FETCH="$REPO_ROOT/scripts/download-frank-large.sh"
|
||||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
|
||||
@test "the frank-large unit, env, serve and build files exist where the deployer installs them from" {
|
||||
@test "the frank-large unit, env and serve files exist where the deployer installs them from" {
|
||||
[ -f "$ENV_FILE" ]
|
||||
[ -f "$UNIT" ]
|
||||
[ -x "$SERVE" ]
|
||||
[ -f "$BUILD" ]
|
||||
}
|
||||
|
||||
@test "frank-large is boot-persistent, self-healing and unprivileged" {
|
||||
@@ -44,7 +41,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the operator's shape is one stream of the full 262144 window with MTP depth 4" {
|
||||
@test "the operator's shape is one stream of the full 262144 window with DFlash2 depth 4" {
|
||||
local ctx slots
|
||||
ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
slots="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
@@ -52,70 +49,64 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
[ "$ctx" -eq 262144 ]
|
||||
# One stream.
|
||||
[ "$slots" -eq 1 ]
|
||||
# MTP depth 4 (the model's MTP layers) + n-gram drafting (24/64/64).
|
||||
# DFlash2 depth 4 (the drafter's block-diffusion depth).
|
||||
grep -q '^SPEC_DRAFT_N_MAX=4$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "the harness needs compatibility mode (sampling, tools, reasoning)" {
|
||||
# The fast-greedy path rejects sampling, penalties, probabilities,
|
||||
# grammar-constrained tool calls, logit bias, LoRA and a reasoning
|
||||
# budget. The harness needs all three, so COMPATIBILITY_MODE=1.
|
||||
grep -q '^COMPATIBILITY_MODE=1$' "$ENV_FILE"
|
||||
# The serve script enables the fastpath off in compatibility mode.
|
||||
grep -q 'LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0' "$SERVE"
|
||||
}
|
||||
|
||||
@test "frank-large stays text-only" {
|
||||
# The Kairic Edge model is text-only (no vision projector). The fleet's
|
||||
# The new model is text-only (no vision projector). The fleet's
|
||||
# image path is frank-small; models.yml declares frank-large text-only.
|
||||
! grep -q -- '--mmproj' "$SERVE"
|
||||
grep -q -- '--mmproj' "$REPO_ROOT/scripts/serve-frank-small.sh"
|
||||
}
|
||||
|
||||
@test "the serve script pins the Kairic Edge runtime and refuses a missing build" {
|
||||
grep -q 'ROCmFPX-kairic' "$SERVE"
|
||||
@test "the serve script pins the stock llama.cpp runtime and refuses a missing binary" {
|
||||
grep -q 'unsloth-llama-b11030' "$SERVE"
|
||||
grep -q 'llama-server' "$SERVE"
|
||||
# A missing runtime or payload is a deploy error with a pointer, not a
|
||||
# restart loop.
|
||||
grep -q 'Kairic Edge runtime not found' "$SERVE"
|
||||
grep -q 'runtime not found' "$SERVE"
|
||||
grep -q 'missing asset' "$SERVE"
|
||||
# The Qwen3.8 template keeps prior thinking blocks across turns only
|
||||
# when the kwarg is set.
|
||||
grep -q 'preserve_thinking' "$SERVE"
|
||||
# The chat template file is wired (not just the kwarg) — the 73k-token
|
||||
# fallback incident (2026-09-21) was a missing --chat-template-file.
|
||||
grep -q 'chat-template-file' "$SERVE"
|
||||
}
|
||||
|
||||
@test "the serve script wires the MTP + n-gram spec method and the sidecars" {
|
||||
# The Kairic Edge model uses native MTP depth 4 + n-gram drafting
|
||||
# (24/64/64). DFlash is NOT available for this model (no DRAFT_DFLASH
|
||||
# spec type in the Kairic Edge runtime).
|
||||
grep -q -- '--spec-type draft-mtp' "$SERVE"
|
||||
grep -q -- '--kairic-edge' "$SERVE"
|
||||
@test "the serve script wires the DFlash2 spec method and the drafter" {
|
||||
# The new model uses DFlash2 (block-diffusion drafter) at depth 4.
|
||||
grep -q -- '--spec-type draft-dflash' "$SERVE"
|
||||
grep -q -- '--model-draft' "$SERVE"
|
||||
grep -q -- '--spec-draft-n-max' "$SERVE"
|
||||
# The three PromptForge sidecars (FFN, GDN, GDN-Output) are wired via
|
||||
# the PROMPTFORGE_* env block.
|
||||
grep -q 'PROMPTFORGE_IU4_SIDECAR' "$SERVE"
|
||||
grep -q 'PROMPTFORGE_GDN_SIDECAR' "$SERVE"
|
||||
grep -q 'PROMPTFORGE_GDN_OUTPUT_SIDECAR' "$SERVE"
|
||||
# The drafter runs on the iGPU (ROCm0).
|
||||
grep -q -- '--spec-draft-device ROCm0' "$SERVE"
|
||||
}
|
||||
|
||||
@test "the serve script uses q8_0 KV and greedy sampling" {
|
||||
# q8_0 KV — the unsloth b11030 build does not support fp8 KV (allowed:
|
||||
# f32/f16/bf16/q8_0/q4_0/q4_1/iq4_nl/q5_0/q5_1); q8_0 is the
|
||||
# highest-precision 8-bit option and preserves the ~8 GiB footprint.
|
||||
grep -q -- '--cache-type-k q8_0' "$SERVE"
|
||||
grep -q -- '--cache-type-v q8_0' "$SERVE"
|
||||
# Greedy sampling (DFlash2 needs greedy verification).
|
||||
grep -q -- '--temp 0' "$SERVE"
|
||||
}
|
||||
|
||||
@test "the payload is fetched resumably and verified against declared sizes + sha256" {
|
||||
grep -q 'jcbtc/Qwen3.8-27B-IU4-Kairic-Edge' "$FETCH"
|
||||
grep -q 'unsloth/Qwen3.8-27B-GGUF' "$FETCH"
|
||||
grep -q 'z-lab/Qwen3.8-27B-DFlash2-GGUF' "$FETCH"
|
||||
grep -q -- '-fL -C -' "$FETCH"
|
||||
# Each file's local sha256 is compared to its declared oid and a
|
||||
# mismatch fails the run.
|
||||
grep -q 'SHA256 MISMATCH' "$FETCH"
|
||||
}
|
||||
|
||||
@test "the build script pins the Kairic Edge commit and the CK patch" {
|
||||
grep -q 'fdf4bb7fcc984811cef48ce817d89aac064b984a' "$BUILD"
|
||||
grep -q 'composable-kernel-gfx1151-iu4.patch' "$BUILD"
|
||||
grep -q 'gfx1151' "$BUILD"
|
||||
}
|
||||
|
||||
@test "the fleet endpoint and alias survive the cutover" {
|
||||
grep -q '^HOST=0.0.0.0$' "$ENV_FILE"
|
||||
grep -q '^PORT=7770$' "$ENV_FILE"
|
||||
grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-27B-IU4-Kairic-Edge$' "$ENV_FILE"
|
||||
grep -q '^MODEL_DIR=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL$' "$ENV_FILE"
|
||||
grep -q -- '--alias frank-large' "$SERVE"
|
||||
}
|
||||
|
||||
@@ -138,7 +129,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
# frank-small on this host is ALSO a llama-server. The sweep must match
|
||||
# the model path, not the binary name, or a frank-large deploy restarts
|
||||
# the 3090 endpoint.
|
||||
grep -q "pkill -f 'Qwen3.8-27B-IU4-Kairic-Edge'" "$REMOTE_DEPLOY"
|
||||
grep -q "pkill -f 'Qwen3.8-27B-UD-Q8_K_XL'" "$REMOTE_DEPLOY"
|
||||
# ...and only inside the large handler: extract it and check there is no
|
||||
# bare binary-name pkill in it.
|
||||
local large
|
||||
|
||||
Reference in New Issue
Block a user