diff --git a/scripts/build-llama-rdna4.sh b/scripts/build-llama-rdna4.sh new file mode 100755 index 0000000..0bb1203 --- /dev/null +++ b/scripts/build-llama-rdna4.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# Build mainline llama.cpp for Dankserver's two Radeon AI PRO R9700 (gfx1201, +# Navi 48) with DFlash2 speculative decoding. +# +# Run on Dankserver. Produces a self-contained build: the binary carries an +# RPATH of $ORIGIN/../lib and the ROCm runtime is copied beside it, so nothing +# depends on the host's ROCm packages (Ubuntu's 7.1) or on LD_LIBRARY_PATH. +# +# bash scripts/build-llama-rdna4.sh +# +# WHY A PRIVATE TOOLCHAIN (ROCm 7.14.0a20260624, ~17 GiB extracted): the host's +# ROCm is Ubuntu's 7.1 packaging, and the pre-existing RDNA4 llama.cpp build on +# this box (charlie12345/ROCmFPX) was built against this same TheRock nightly. +# Reusing it keeps the toolchain that is known to emit working gfx1201 code +# objects; the tarball is only downloaded when the prefix is missing. +# +# WHY THIS REF: master at fb27a525 carries what this service needs and nothing +# older does: +# * DFlash2 (`DFlash2DraftModel`): grouped dynamic depthwise convolution plus +# the candidate selector, detected from the draft GGUF +# (`selector_top_k > 0`) and served through the ordinary `--spec-type +# draft-dflash` path. Landed with ggml-org/llama.cpp#27342. +# * qwen35 MTP: the `draft-mtp` implementation covers qwen35's single trained +# MTP head (common/speculative.cpp), which the target GGUF carries. +# Do not "update to latest" casually: both features landed days apart and the +# service depends on the pair. +# +# gfx1201 IS NOT gfx1200. The ROCmFPX fork's docs/BUILD-AMD-ARCHITECTURES.md +# records that a gfx1200 build links, loads the model and then segfaults on +# these cards, so the arch list here is exactly gfx1201 and no other. +set -euo pipefail + +LLAMA_SRC="${LLAMA_SRC:-/home/dank/GitPackages/llama.cpp-dflash2}" +LLAMA_REPO="${LLAMA_REPO:-https://github.com/ggml-org/llama.cpp.git}" +LLAMA_REF="${LLAMA_REF:-fb27a525d28381a16a4bb038858a10e4927381ca}" +BUILD_DIR="${BUILD_DIR:-$LLAMA_SRC/build-rdna4-rocm714}" +JOBS="${JOBS:-$(nproc)}" +ROCM_VERSION="${ROCM_VERSION:-7.14.0a20260624}" +ROCM_LOCAL_PATH="${ROCM_LOCAL_PATH:-${XDG_DATA_HOME:-$HOME/.local/share}/rocm-${ROCM_VERSION}}" +HIP_ARCH="${HIP_ARCH:-gfx1201}" + +# ── Source ──────────────────────────────────────────────────────────── +if [[ ! -d "$LLAMA_SRC/.git" ]]; then + echo "cloning $LLAMA_REPO into $LLAMA_SRC" + git clone "$LLAMA_REPO" "$LLAMA_SRC" +fi +git -C "$LLAMA_SRC" fetch --quiet origin "$LLAMA_REF" 2>/dev/null || git -C "$LLAMA_SRC" fetch --quiet origin +git -C "$LLAMA_SRC" checkout --detach "$LLAMA_REF" +echo "building llama.cpp $(git -C "$LLAMA_SRC" rev-parse --short HEAD) from $LLAMA_SRC" + +# ── ROCm toolchain ──────────────────────────────────────────────────── +if [[ ! -x "$ROCM_LOCAL_PATH/llvm/bin/clang" ]]; then + TARBALL_URL="https://rocm.nightlies.amd.com/tarball-multi-arch/therock-dist-linux-gfx120X-all-${ROCM_VERSION}.tar.gz" + TARBALL="$LLAMA_SRC/rocm-${ROCM_VERSION}.tar.gz" + if [[ ! -s "$TARBALL" ]]; then + echo "downloading ROCm $ROCM_VERSION from TheRock nightlies" + curl -fL --retry 5 -o "$TARBALL" "$TARBALL_URL" + fi + echo "extracting into $ROCM_LOCAL_PATH" + mkdir -p "$ROCM_LOCAL_PATH" + tar --use-compress-program=gzip -xf "$TARBALL" -C "$ROCM_LOCAL_PATH" --strip-components=1 +fi +CLANG="$ROCM_LOCAL_PATH/llvm/bin/clang" +CLANGXX="$ROCM_LOCAL_PATH/llvm/bin/clang++" +[[ -x "$CLANG" && -x "$CLANGXX" ]] || { echo "no clang/clang++ under $ROCM_LOCAL_PATH" >&2; exit 1; } + +# ggml-hip's CMake falls back to $ENV{ROCM_PATH} or /opt/rocm when these are +# unset, which links a second, mismatched HIP runtime next to the local one. +export ROCM_PATH="$ROCM_LOCAL_PATH" +export HIP_PATH="$ROCM_LOCAL_PATH" + +# ── Configure ───────────────────────────────────────────────────────── +# GGML_HIP_RCCL stays OFF: this host's second card cannot dispatch a RCCL +# kernel at all (its PCIe path has no atomic routing), and layer-split serving +# needs no collectives. With RCCL off, llama.cpp logs "internal AllReduce init +# failed ... falling back to meta-backend butterfly" and only --split-mode +# tensor pays for it. +cmake -S "$LLAMA_SRC" -B "$BUILD_DIR" \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER="$CLANG" \ + -DCMAKE_CXX_COMPILER="$CLANGXX" \ + -DCMAKE_CXX_FLAGS="-I${ROCM_LOCAL_PATH}/include" \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DGGML_HIP=ON \ + -DGGML_HIP_RCCL=OFF \ + -DGGML_CUDA=OFF \ + -DGGML_VULKAN=OFF \ + -DGGML_NATIVE=OFF \ + -DCMAKE_HIP_ARCHITECTURES="$HIP_ARCH" \ + -DGPU_TARGETS="$HIP_ARCH" \ + -DLLAMA_BUILD_SERVER=ON \ + -DLLAMA_BUILD_WEBUI=OFF \ + -DLLAMA_USE_PREBUILT_WEBUI=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DGGML_BUILD_TESTS=OFF \ + -DCMAKE_PREFIX_PATH="$ROCM_LOCAL_PATH" \ + -DROCM_DIR="$ROCM_LOCAL_PATH" \ + -DBUILD_SHARED_LIBS=OFF + +# ── Build ───────────────────────────────────────────────────────────── +cmake --build "$BUILD_DIR" -j "$JOBS" \ + --target llama-server llama-cli llama-quantize llama-bench + +# ── Bundle the runtime ──────────────────────────────────────────────── +if command -v patchelf >/dev/null; then + LIB_DIR="$BUILD_DIR/lib" + mkdir -p "$LIB_DIR" + for pat in libamdhip64 librocblas libhipblas libhipblaslt librocsolver \ + libroctx64 libamd_comgr libhsa-runtime64 librocm_kpack \ + librocroller librocprofiler-register; do + cp -v "$ROCM_LOCAL_PATH/lib/$pat".so* "$LIB_DIR/" 2>/dev/null || true + done + cp -v "$ROCM_LOCAL_PATH/lib/llvm/lib"/libLLVM.so* \ + "$ROCM_LOCAL_PATH/lib/llvm/lib"/libclang-cpp.so* \ + "$ROCM_LOCAL_PATH/lib/llvm/lib"/libomp.so* \ + "$ROCM_LOCAL_PATH/lib/llvm/lib"/libomptarget.so* "$LIB_DIR/" 2>/dev/null || true + cp -v "$ROCM_LOCAL_PATH/lib/rocm_sysdeps/lib"/librocm_sysdeps_*.so* "$LIB_DIR/" 2>/dev/null || true + [[ -d "$ROCM_LOCAL_PATH/lib/rocblas/library" ]] && \ + cp -r "$ROCM_LOCAL_PATH/lib/rocblas/library" "$LIB_DIR/rocblas/" 2>/dev/null || true + [[ -d "$ROCM_LOCAL_PATH/lib/hipblaslt/library" ]] && \ + cp -r "$ROCM_LOCAL_PATH/lib/hipblaslt/library" "$LIB_DIR/hipblaslt/" 2>/dev/null || true + for bin in "$BUILD_DIR"/bin/llama-*; do + [[ -f "$bin" ]] && patchelf --set-rpath '$ORIGIN/../lib' "$bin" + done +else + echo "patchelf missing; run with LD_LIBRARY_PATH=$ROCM_LOCAL_PATH/lib:$ROCM_LOCAL_PATH/lib/llvm/lib" >&2 +fi + +echo +echo "built for $HIP_ARCH:" +"$BUILD_DIR/bin/llama-server" --version 2>&1 | head -3 +ldd "$BUILD_DIR/bin/llama-server" 2>/dev/null | grep -E "not found" && exit 1 || true +echo " $BUILD_DIR/bin/llama-server" diff --git a/scripts/download-frank-medium-gguf.sh b/scripts/download-frank-medium-gguf.sh new file mode 100755 index 0000000..34aeebf --- /dev/null +++ b/scripts/download-frank-medium-gguf.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Download the frank-medium payload for Dankserver (2x Radeon AI PRO R9700): +# the target GGUF and the DFlash2 draft model, into /home/dank/models. +# +# bash scripts/download-frank-medium-gguf.sh +# bash scripts/download-frank-medium-gguf.sh --verify-only +# +# Both repositories are public (anonymous download; no token needed). If a +# token exists at ~/.cache/huggingface/token it is used, which keeps a 31 GB +# transfer clear of the anonymous rate limit. The script is resumable +# (curl -C -) and checks every artifact's exact byte count afterwards, so a +# truncated file fails here instead of load-crashing the server. +# +# WHY THESE TWO FILES: +# * Target: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at +# UD-Q8_K_XL -- Unsloth Dynamic 3.0 quant (per-tensor types from Unsloth's +# own quant of the same size, on Unsloth's importance matrix). The file +# carries the model's MTP layer (`blk.64.nextn.*`), so llama.cpp can also +# draft from the model itself with no second copy in VRAM. +# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q8_0 -- the block-diffusion +# drafter with the candidate selector (5 layers, hidden 5120). Q8_0 is +# 2.0 GiB against BF16's 3.9 GiB and the drafter's job is throughput, not +# output quality (verification is lossless either way). +set -euo pipefail + +MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF" +MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF" +MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf" +MODEL_SIZE=31457990112 + +DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF" +DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF" +DRAFT_FILE="Qwen3.8-27B-DFlash2-Q8_0.gguf" +DRAFT_SIZE=2056414816 + +mode="download" +[[ "${1:-}" == "--verify-only" ]] && mode="verify" + +token="" +auth=() +if [[ -r "$HOME/.cache/huggingface/token" ]]; then + token="$(cat "$HOME/.cache/huggingface/token")" + auth=(-H "Authorization: Bearer $token") +else + echo "no HF token at ~/.cache/huggingface/token; downloading anonymously" +fi + +dl() { # dl + mkdir -p "$2" + echo " fetching $3" + curl -fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600 \ + "${auth[@]}" "https://huggingface.co/$1/resolve/main/$3" +} + +if [[ "$mode" == "download" ]]; then + echo "target: $MODEL_REPO" + dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE" + echo "draft: $DRAFT_REPO" + dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE" +fi + +echo "verifying sizes" +ok=1 +check() { # check + local act + act="$(stat -c%s "$1" 2>/dev/null || echo 0)" + if [[ "$act" -eq "$2" ]]; then + echo " OK $1 ($act bytes)" + else + echo " BAD $1 ($act bytes, want $2)" >&2 + ok=0 + fi +} +check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE" +check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE" +[[ -z "$token" ]] && echo "note: no token used; the byte counts above are the integrity check" +[[ "$ok" -eq 1 ]] || exit 1 +echo "all artifacts present and sized correctly"