frank-medium: add the RDNA4 llama.cpp build and the GGUF fetch
The runtime for the next frank-medium shape is mainline llama.cpp at fb27a525, built for gfx1201 against the ROCm 7.14 nightly the ROCmFPX build already used on this host, with the runtime bundled next to the binary. Two things that pin the ref: DFlash2 (ggml-org/llama.cpp#27342) and qwen35 MTP. Neither is in the ROCmFPX fork's tip (c49ebdb, 2026-08-22), and the DFlash2 drafter cannot run without the first. The fetch script pulls the target at UD-Q8_K_XL plus the DFlash2 draft at Q8_0, resumable and byte-verified -- a truncated 31 GB GGUF is otherwise a load-time crash.
This commit is contained in:
Frank
Executable
+134
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build mainline llama.cpp for Dankserver's two Radeon AI PRO R9700 (gfx1201,
|
||||
# Navi 48) with DFlash2 speculative decoding.
|
||||
#
|
||||
# Run on Dankserver. Produces a self-contained build: the binary carries an
|
||||
# RPATH of $ORIGIN/../lib and the ROCm runtime is copied beside it, so nothing
|
||||
# depends on the host's ROCm packages (Ubuntu's 7.1) or on LD_LIBRARY_PATH.
|
||||
#
|
||||
# bash scripts/build-llama-rdna4.sh
|
||||
#
|
||||
# WHY A PRIVATE TOOLCHAIN (ROCm 7.14.0a20260624, ~17 GiB extracted): the host's
|
||||
# ROCm is Ubuntu's 7.1 packaging, and the pre-existing RDNA4 llama.cpp build on
|
||||
# this box (charlie12345/ROCmFPX) was built against this same TheRock nightly.
|
||||
# Reusing it keeps the toolchain that is known to emit working gfx1201 code
|
||||
# objects; the tarball is only downloaded when the prefix is missing.
|
||||
#
|
||||
# WHY THIS REF: master at fb27a525 carries what this service needs and nothing
|
||||
# older does:
|
||||
# * DFlash2 (`DFlash2DraftModel`): grouped dynamic depthwise convolution plus
|
||||
# the candidate selector, detected from the draft GGUF
|
||||
# (`selector_top_k > 0`) and served through the ordinary `--spec-type
|
||||
# draft-dflash` path. Landed with ggml-org/llama.cpp#27342.
|
||||
# * qwen35 MTP: the `draft-mtp` implementation covers qwen35's single trained
|
||||
# MTP head (common/speculative.cpp), which the target GGUF carries.
|
||||
# Do not "update to latest" casually: both features landed days apart and the
|
||||
# service depends on the pair.
|
||||
#
|
||||
# gfx1201 IS NOT gfx1200. The ROCmFPX fork's docs/BUILD-AMD-ARCHITECTURES.md
|
||||
# records that a gfx1200 build links, loads the model and then segfaults on
|
||||
# these cards, so the arch list here is exactly gfx1201 and no other.
|
||||
set -euo pipefail
|
||||
|
||||
LLAMA_SRC="${LLAMA_SRC:-/home/dank/GitPackages/llama.cpp-dflash2}"
|
||||
LLAMA_REPO="${LLAMA_REPO:-https://github.com/ggml-org/llama.cpp.git}"
|
||||
LLAMA_REF="${LLAMA_REF:-fb27a525d28381a16a4bb038858a10e4927381ca}"
|
||||
BUILD_DIR="${BUILD_DIR:-$LLAMA_SRC/build-rdna4-rocm714}"
|
||||
JOBS="${JOBS:-$(nproc)}"
|
||||
ROCM_VERSION="${ROCM_VERSION:-7.14.0a20260624}"
|
||||
ROCM_LOCAL_PATH="${ROCM_LOCAL_PATH:-${XDG_DATA_HOME:-$HOME/.local/share}/rocm-${ROCM_VERSION}}"
|
||||
HIP_ARCH="${HIP_ARCH:-gfx1201}"
|
||||
|
||||
# ── Source ────────────────────────────────────────────────────────────
|
||||
if [[ ! -d "$LLAMA_SRC/.git" ]]; then
|
||||
echo "cloning $LLAMA_REPO into $LLAMA_SRC"
|
||||
git clone "$LLAMA_REPO" "$LLAMA_SRC"
|
||||
fi
|
||||
git -C "$LLAMA_SRC" fetch --quiet origin "$LLAMA_REF" 2>/dev/null || git -C "$LLAMA_SRC" fetch --quiet origin
|
||||
git -C "$LLAMA_SRC" checkout --detach "$LLAMA_REF"
|
||||
echo "building llama.cpp $(git -C "$LLAMA_SRC" rev-parse --short HEAD) from $LLAMA_SRC"
|
||||
|
||||
# ── ROCm toolchain ────────────────────────────────────────────────────
|
||||
if [[ ! -x "$ROCM_LOCAL_PATH/llvm/bin/clang" ]]; then
|
||||
TARBALL_URL="https://rocm.nightlies.amd.com/tarball-multi-arch/therock-dist-linux-gfx120X-all-${ROCM_VERSION}.tar.gz"
|
||||
TARBALL="$LLAMA_SRC/rocm-${ROCM_VERSION}.tar.gz"
|
||||
if [[ ! -s "$TARBALL" ]]; then
|
||||
echo "downloading ROCm $ROCM_VERSION from TheRock nightlies"
|
||||
curl -fL --retry 5 -o "$TARBALL" "$TARBALL_URL"
|
||||
fi
|
||||
echo "extracting into $ROCM_LOCAL_PATH"
|
||||
mkdir -p "$ROCM_LOCAL_PATH"
|
||||
tar --use-compress-program=gzip -xf "$TARBALL" -C "$ROCM_LOCAL_PATH" --strip-components=1
|
||||
fi
|
||||
CLANG="$ROCM_LOCAL_PATH/llvm/bin/clang"
|
||||
CLANGXX="$ROCM_LOCAL_PATH/llvm/bin/clang++"
|
||||
[[ -x "$CLANG" && -x "$CLANGXX" ]] || { echo "no clang/clang++ under $ROCM_LOCAL_PATH" >&2; exit 1; }
|
||||
|
||||
# ggml-hip's CMake falls back to $ENV{ROCM_PATH} or /opt/rocm when these are
|
||||
# unset, which links a second, mismatched HIP runtime next to the local one.
|
||||
export ROCM_PATH="$ROCM_LOCAL_PATH"
|
||||
export HIP_PATH="$ROCM_LOCAL_PATH"
|
||||
|
||||
# ── Configure ─────────────────────────────────────────────────────────
|
||||
# GGML_HIP_RCCL stays OFF: this host's second card cannot dispatch a RCCL
|
||||
# kernel at all (its PCIe path has no atomic routing), and layer-split serving
|
||||
# needs no collectives. With RCCL off, llama.cpp logs "internal AllReduce init
|
||||
# failed ... falling back to meta-backend butterfly" and only --split-mode
|
||||
# tensor pays for it.
|
||||
cmake -S "$LLAMA_SRC" -B "$BUILD_DIR" \
|
||||
-G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_C_COMPILER="$CLANG" \
|
||||
-DCMAKE_CXX_COMPILER="$CLANGXX" \
|
||||
-DCMAKE_CXX_FLAGS="-I${ROCM_LOCAL_PATH}/include" \
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
|
||||
-DGGML_HIP=ON \
|
||||
-DGGML_HIP_RCCL=OFF \
|
||||
-DGGML_CUDA=OFF \
|
||||
-DGGML_VULKAN=OFF \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DCMAKE_HIP_ARCHITECTURES="$HIP_ARCH" \
|
||||
-DGPU_TARGETS="$HIP_ARCH" \
|
||||
-DLLAMA_BUILD_SERVER=ON \
|
||||
-DLLAMA_BUILD_WEBUI=OFF \
|
||||
-DLLAMA_USE_PREBUILT_WEBUI=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DGGML_BUILD_TESTS=OFF \
|
||||
-DCMAKE_PREFIX_PATH="$ROCM_LOCAL_PATH" \
|
||||
-DROCM_DIR="$ROCM_LOCAL_PATH" \
|
||||
-DBUILD_SHARED_LIBS=OFF
|
||||
|
||||
# ── Build ─────────────────────────────────────────────────────────────
|
||||
cmake --build "$BUILD_DIR" -j "$JOBS" \
|
||||
--target llama-server llama-cli llama-quantize llama-bench
|
||||
|
||||
# ── Bundle the runtime ────────────────────────────────────────────────
|
||||
if command -v patchelf >/dev/null; then
|
||||
LIB_DIR="$BUILD_DIR/lib"
|
||||
mkdir -p "$LIB_DIR"
|
||||
for pat in libamdhip64 librocblas libhipblas libhipblaslt librocsolver \
|
||||
libroctx64 libamd_comgr libhsa-runtime64 librocm_kpack \
|
||||
librocroller librocprofiler-register; do
|
||||
cp -v "$ROCM_LOCAL_PATH/lib/$pat".so* "$LIB_DIR/" 2>/dev/null || true
|
||||
done
|
||||
cp -v "$ROCM_LOCAL_PATH/lib/llvm/lib"/libLLVM.so* \
|
||||
"$ROCM_LOCAL_PATH/lib/llvm/lib"/libclang-cpp.so* \
|
||||
"$ROCM_LOCAL_PATH/lib/llvm/lib"/libomp.so* \
|
||||
"$ROCM_LOCAL_PATH/lib/llvm/lib"/libomptarget.so* "$LIB_DIR/" 2>/dev/null || true
|
||||
cp -v "$ROCM_LOCAL_PATH/lib/rocm_sysdeps/lib"/librocm_sysdeps_*.so* "$LIB_DIR/" 2>/dev/null || true
|
||||
[[ -d "$ROCM_LOCAL_PATH/lib/rocblas/library" ]] && \
|
||||
cp -r "$ROCM_LOCAL_PATH/lib/rocblas/library" "$LIB_DIR/rocblas/" 2>/dev/null || true
|
||||
[[ -d "$ROCM_LOCAL_PATH/lib/hipblaslt/library" ]] && \
|
||||
cp -r "$ROCM_LOCAL_PATH/lib/hipblaslt/library" "$LIB_DIR/hipblaslt/" 2>/dev/null || true
|
||||
for bin in "$BUILD_DIR"/bin/llama-*; do
|
||||
[[ -f "$bin" ]] && patchelf --set-rpath '$ORIGIN/../lib' "$bin"
|
||||
done
|
||||
else
|
||||
echo "patchelf missing; run with LD_LIBRARY_PATH=$ROCM_LOCAL_PATH/lib:$ROCM_LOCAL_PATH/lib/llvm/lib" >&2
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "built for $HIP_ARCH:"
|
||||
"$BUILD_DIR/bin/llama-server" --version 2>&1 | head -3
|
||||
ldd "$BUILD_DIR/bin/llama-server" 2>/dev/null | grep -E "not found" && exit 1 || true
|
||||
echo " $BUILD_DIR/bin/llama-server"
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-medium payload for Dankserver (2x Radeon AI PRO R9700):
|
||||
# the target GGUF and the DFlash2 draft model, into /home/dank/models.
|
||||
#
|
||||
# bash scripts/download-frank-medium-gguf.sh
|
||||
# bash scripts/download-frank-medium-gguf.sh --verify-only
|
||||
#
|
||||
# Both repositories are public (anonymous download; no token needed). If a
|
||||
# token exists at ~/.cache/huggingface/token it is used, which keeps a 31 GB
|
||||
# transfer clear of the anonymous rate limit. The script is resumable
|
||||
# (curl -C -) and checks every artifact's exact byte count afterwards, so a
|
||||
# truncated file fails here instead of load-crashing the server.
|
||||
#
|
||||
# WHY THESE TWO FILES:
|
||||
# * Target: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at
|
||||
# UD-Q8_K_XL -- Unsloth Dynamic 3.0 quant (per-tensor types from Unsloth's
|
||||
# own quant of the same size, on Unsloth's importance matrix). The file
|
||||
# carries the model's MTP layer (`blk.64.nextn.*`), so llama.cpp can also
|
||||
# draft from the model itself with no second copy in VRAM.
|
||||
# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q8_0 -- the block-diffusion
|
||||
# drafter with the candidate selector (5 layers, hidden 5120). Q8_0 is
|
||||
# 2.0 GiB against BF16's 3.9 GiB and the drafter's job is throughput, not
|
||||
# output quality (verification is lossless either way).
|
||||
set -euo pipefail
|
||||
|
||||
MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
||||
MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
||||
MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf"
|
||||
MODEL_SIZE=31457990112
|
||||
|
||||
DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF"
|
||||
DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF"
|
||||
DRAFT_FILE="Qwen3.8-27B-DFlash2-Q8_0.gguf"
|
||||
DRAFT_SIZE=2056414816
|
||||
|
||||
mode="download"
|
||||
[[ "${1:-}" == "--verify-only" ]] && mode="verify"
|
||||
|
||||
token=""
|
||||
auth=()
|
||||
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
|
||||
token="$(cat "$HOME/.cache/huggingface/token")"
|
||||
auth=(-H "Authorization: Bearer $token")
|
||||
else
|
||||
echo "no HF token at ~/.cache/huggingface/token; downloading anonymously"
|
||||
fi
|
||||
|
||||
dl() { # dl <repo> <dir> <file>
|
||||
mkdir -p "$2"
|
||||
echo " fetching $3"
|
||||
curl -fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600 \
|
||||
"${auth[@]}" "https://huggingface.co/$1/resolve/main/$3"
|
||||
}
|
||||
|
||||
if [[ "$mode" == "download" ]]; then
|
||||
echo "target: $MODEL_REPO"
|
||||
dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE"
|
||||
echo "draft: $DRAFT_REPO"
|
||||
dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE"
|
||||
fi
|
||||
|
||||
echo "verifying sizes"
|
||||
ok=1
|
||||
check() { # check <path> <expected>
|
||||
local act
|
||||
act="$(stat -c%s "$1" 2>/dev/null || echo 0)"
|
||||
if [[ "$act" -eq "$2" ]]; then
|
||||
echo " OK $1 ($act bytes)"
|
||||
else
|
||||
echo " BAD $1 ($act bytes, want $2)" >&2
|
||||
ok=0
|
||||
fi
|
||||
}
|
||||
check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE"
|
||||
check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE"
|
||||
[[ -z "$token" ]] && echo "note: no token used; the byte counts above are the integrity check"
|
||||
[[ "$ok" -eq 1 ]] || exit 1
|
||||
echo "all artifacts present and sized correctly"
|
||||
Reference in New Issue
Block a user