Frank
91546dbb95
DSpark came off after a day. It was the better-looking file (1.27 GiB against 1.92) but it declares no sliding window, and the draft context is built with n_ctx = llama_n_ctx(ctx_tgt) -- so its KV spanned the whole 262144-token window (5.0 GiB) and cost ROCm1 four layers of split to carry. DFlash2's window bounds its draft KV at 2048 positions (~40 MiB), which frees ~4.3 GiB on the draft-carrying card. The balance point moved with it, so it was re-measured at the deployed batch (6144 x 1536, 2 slots, 262144) rather than interpolated: 39,26 2.42 / 7.20 37,28 4.28 / 5.34 35,30 5.14 / 4.48 38,27 3.85 / 5.77 36,29 4.71 / 4.91 34,31 6.57 / 3.05 36:29 is the minimum (0.20 GiB apart) and leaves 3.2 GiB above the operator's 1.5 GiB floor on each card, so the full window stays. One -ts step is one layer, and qwen35's layers are not the same size -- 0.43 GiB on a DeltaNet boundary, 1.43 GiB on a full-attention one -- which is why the table exists. Also in here: - the fetch script pointed at the wrong file (DSpark) AND declared the wrong size for it: 1359284737 against the Hub's 1455376576, 96 MB short, which made --verify-only fail on a payload that was in fact correct. Repointed at z-lab DFlash2 Q8_0, 2056414816, and both files' sha256 were checked against the Hub's LFS oids (host == Hub for both). - the seam's fit was recalibrated for a window-bounded draft KV and for the new point; it now passes 36:29 and fails every other split measured, including 1:1. Its ubatch term is an extrapolation and it cannot model contiguity, which the comment says out loud. - the serve script's dead dspark-only tunables (n-min, p-min, p-split, poll) are gone; the build's defaults apply.
79 lines
3.0 KiB
Bash
Executable File
79 lines
3.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Download the frank-medium payload for Dankserver (2x Radeon AI PRO R9700):
|
|
# the target GGUF and the DFlash2 draft model, into /home/dank/models.
|
|
#
|
|
# bash scripts/download-frank-medium-gguf.sh
|
|
# bash scripts/download-frank-medium-gguf.sh --verify-only
|
|
#
|
|
# Both repositories are public (anonymous download; no token needed). If a
|
|
# token exists at ~/.cache/huggingface/token it is used, which keeps a 31 GB
|
|
# transfer clear of the anonymous rate limit. The script is resumable
|
|
# (curl -C -) and checks every artifact's exact byte count afterwards, so a
|
|
# truncated file fails here instead of load-crashing the server.
|
|
#
|
|
# WHY THESE TWO FILES:
|
|
# * Target: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at
|
|
# UD-Q8_K_XL -- Unsloth Dynamic 3.0 quant (per-tensor types from Unsloth's
|
|
# own quant of the same size, on Unsloth's importance matrix). The file
|
|
# carries the model's MTP layer (`blk.64.nextn.*`), so llama.cpp can also
|
|
# draft from the model itself with no second copy in VRAM.
|
|
# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q8_0 -- the block-diffusion
|
|
# drafter (5 layers, hidden 5120, block_size 8, a 2048-token sliding
|
|
# window). Q8_0 is 1.92 GiB against BF16's 3.6 GiB and the drafter's job is
|
|
# throughput, not output quality (the target verifies every drafted token).
|
|
set -euo pipefail
|
|
|
|
MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
|
MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
|
MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf"
|
|
MODEL_SIZE=31457990112
|
|
|
|
DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF"
|
|
DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF"
|
|
DRAFT_FILE="Qwen3.8-27B-DFlash2-Q8_0.gguf"
|
|
DRAFT_SIZE=2056414816
|
|
|
|
mode="download"
|
|
[[ "${1:-}" == "--verify-only" ]] && mode="verify"
|
|
|
|
token=""
|
|
auth=()
|
|
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
|
|
token="$(cat "$HOME/.cache/huggingface/token")"
|
|
auth=(-H "Authorization: Bearer $token")
|
|
else
|
|
echo "no HF token at ~/.cache/huggingface/token; downloading anonymously"
|
|
fi
|
|
|
|
dl() { # dl <repo> <dir> <file>
|
|
mkdir -p "$2"
|
|
echo " fetching $3"
|
|
curl -fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600 \
|
|
"${auth[@]}" "https://huggingface.co/$1/resolve/main/$3"
|
|
}
|
|
|
|
if [[ "$mode" == "download" ]]; then
|
|
echo "target: $MODEL_REPO"
|
|
dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE"
|
|
echo "draft: $DRAFT_REPO"
|
|
dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE"
|
|
fi
|
|
|
|
echo "verifying sizes"
|
|
ok=1
|
|
check() { # check <path> <expected>
|
|
local act
|
|
act="$(stat -c%s "$1" 2>/dev/null || echo 0)"
|
|
if [[ "$act" -eq "$2" ]]; then
|
|
echo " OK $1 ($act bytes)"
|
|
else
|
|
echo " BAD $1 ($act bytes, want $2)" >&2
|
|
ok=0
|
|
fi
|
|
}
|
|
check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE"
|
|
check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE"
|
|
[[ -z "$token" ]] && echo "note: no token used; the byte counts above are the integrity check"
|
|
[[ "$ok" -eq 1 ]] || exit 1
|
|
echo "all artifacts present and sized correctly"
|