frank-medium: swap the drafter to magnitudedev/Qwen3.8-27B-DSpark
DSpark (Q8_0, 1.27 GiB) replaces z-lab DFlash2 (1.92 GiB): same 5-layer block-diffusion shape, Markov head instead of the candidate selector, and block_size 7, so the draft depth drops 7 -> 6 (llama.cpp asserts n_max <= block_size - 1). The serve script's dspark branch never passed --spec-draft-model (only the dflash branch did), so dspark was not deployable as written; the model file gate now covers both types. Download script, env and the unit seam track the new file and its exact byte count.
This commit is contained in:
1 parent
96062b627e
commit
7e11d247a8
4 files changed
+31
-33
No files matched your search
@@ -1,6 +1,6 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Download the frank-medium payload for Dankserver (2x Radeon AI PRO R9700):
|
# Download the frank-medium payload for Dankserver (2x Radeon AI PRO R9700):
|
||||||
# the target GGUF and the DFlash2 draft model, into /home/dank/models.
|
# the target GGUF and the DSpark draft model, into /home/dank/models.
|
||||||
#
|
#
|
||||||
# bash scripts/download-frank-medium-gguf.sh
|
# bash scripts/download-frank-medium-gguf.sh
|
||||||
# bash scripts/download-frank-medium-gguf.sh --verify-only
|
# bash scripts/download-frank-medium-gguf.sh --verify-only
|
||||||
@@ -17,10 +17,10 @@
|
|||||||
# own quant of the same size, on Unsloth's importance matrix). The file
|
# own quant of the same size, on Unsloth's importance matrix). The file
|
||||||
# carries the model's MTP layer (`blk.64.nextn.*`), so llama.cpp can also
|
# carries the model's MTP layer (`blk.64.nextn.*`), so llama.cpp can also
|
||||||
# draft from the model itself with no second copy in VRAM.
|
# draft from the model itself with no second copy in VRAM.
|
||||||
# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q8_0 -- the block-diffusion
|
# * Draft: magnitudedev/Qwen3.8-27B-DSpark-GGUF at Q8_0 -- the
|
||||||
# drafter with the candidate selector (5 layers, hidden 5120). Q8_0 is
|
# block-diffusion drafter with the Markov head (5 layers, hidden 5120,
|
||||||
# 2.0 GiB against BF16's 3.9 GiB and the drafter's job is throughput, not
|
# block_size 7). Q8_0 is 1.27 GiB against BF16's ~2.5 GiB and the
|
||||||
# output quality (verification is lossless either way).
|
# drafter's job is throughput, not output quality (lossless verification).
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
||||||
@@ -28,10 +28,10 @@ MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
|||||||
MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf"
|
MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf"
|
||||||
MODEL_SIZE=31457990112
|
MODEL_SIZE=31457990112
|
||||||
|
|
||||||
DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF"
|
DRAFT_REPO="magnitudedev/Qwen3.8-27B-DSpark-GGUF"
|
||||||
DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF"
|
DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DSpark-GGUF"
|
||||||
DRAFT_FILE="Qwen3.8-27B-DFlash2-Q8_0.gguf"
|
DRAFT_FILE="Qwen3.8-27B-DSpark-Q8_0.gguf"
|
||||||
DRAFT_SIZE=2056414816
|
DRAFT_SIZE=1359284737
|
||||||
|
|
||||||
mode="download"
|
mode="download"
|
||||||
[[ "${1:-}" == "--verify-only" ]] && mode="verify"
|
[[ "${1:-}" == "--verify-only" ]] && mode="verify"
|
||||||
|
|||||||
+13
-15
@@ -1,5 +1,5 @@
|
|||||||
# Tunables for frank-medium (ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP
|
# Tunables for frank-medium (ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP
|
||||||
# UD-Q8_K_XL + z-lab/Qwen3.8-27B-DFlash2-GGUF Q8_0 drafter), mainline llama.cpp
|
# UD-Q8_K_XL + magnitudedev/Qwen3.8-27B-DSpark-GGUF Q8_0 drafter), mainline llama.cpp
|
||||||
# on Dankserver, 2x AMD Radeon AI PRO R9700 (gfx1201). Since 2026-09-16 this
|
# on Dankserver, 2x AMD Radeon AI PRO R9700 (gfx1201). Since 2026-09-16 this
|
||||||
# is the SOLE LLM runtime on the box: the vllm-radiance replica pair and its
|
# is the SOLE LLM runtime on the box: the vllm-radiance replica pair and its
|
||||||
# routing proxy are retired.
|
# routing proxy are retired.
|
||||||
@@ -39,9 +39,9 @@
|
|||||||
# carries the model's MTP head (blk.64.nextn.*), so a file-level switch to
|
# carries the model's MTP head (blk.64.nextn.*), so a file-level switch to
|
||||||
# --spec-type draft-mtp is available without a second copy in VRAM.
|
# --spec-type draft-mtp is available without a second copy in VRAM.
|
||||||
#
|
#
|
||||||
# Drafter: z-lab/Qwen3.8-27B-DFlash2-GGUF @ Q8_0, 1.92 GiB: 5 layers, hidden
|
# Drafter: magnitudedev/Qwen3.8-27B-DSpark-GGUF @ Q8_0, 1.27 GiB: 5 layers,
|
||||||
# 5120, block-diffusion proposal with the candidate selector. Decoding stays
|
# hidden 5120, block-diffusion proposal with the Markov head (dspark).
|
||||||
# lossless -- the target verifies every drafted token.
|
# Decoding stays lossless -- the target verifies every drafted token.
|
||||||
#
|
#
|
||||||
# Architecture (GGUF arch qwen35): 64 layers as 16 x (3 x (Gated DeltaNet ->
|
# Architecture (GGUF arch qwen35): 64 layers as 16 x (3 x (Gated DeltaNet ->
|
||||||
# FFN) + 1 x (Gated Attention -> FFN)) -- 48 linear-attention layers whose
|
# FFN) + 1 x (Gated Attention -> FFN)) -- 48 linear-attention layers whose
|
||||||
@@ -74,8 +74,8 @@ LLAMA_BIN=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama
|
|||||||
# Payload, fetched with scripts/download-frank-medium-gguf.sh.
|
# Payload, fetched with scripts/download-frank-medium-gguf.sh.
|
||||||
MODEL_DIR=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF
|
MODEL_DIR=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF
|
||||||
MODEL_FILE=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf
|
MODEL_FILE=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf
|
||||||
DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF
|
DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DSpark-GGUF
|
||||||
DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q8_0.gguf
|
DRAFT_MODEL_FILE=Qwen3.8-27B-DSpark-Q8_0.gguf
|
||||||
|
|
||||||
# Fleet identity. 7770 is the fleet's dankserver endpoint; the alias is the id
|
# Fleet identity. 7770 is the fleet's dankserver endpoint; the alias is the id
|
||||||
# at /v1/models and what every client asks for.
|
# at /v1/models and what every client asks for.
|
||||||
@@ -137,15 +137,13 @@ REASONING_BUDGET=-1
|
|||||||
CHAT_TEMPLATE_FILE=
|
CHAT_TEMPLATE_FILE=
|
||||||
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}'
|
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}'
|
||||||
|
|
||||||
# Speculative decoding: the DFlash2 block-diffusion drafter PLUS the draftless
|
# Speculative decoding: the DSpark block-diffusion drafter PLUS the draftless
|
||||||
# n-gram matcher. Measured against the drafter alone on 2026-09-16 (tables
|
# n-gram matcher. DSpark's GGUF carries block_size 7, so at most 6 draft
|
||||||
# below): the chain is byte-identical to DFlash2 by itself where n-grams cannot
|
# tokens per verification step (llama.cpp asserts n_max <= block_size - 1).
|
||||||
# match, and ~2.6x it where they can. DFlash2's GGUF carries block_size 8, so at
|
# The drafter rides ROCm1, the card that ends up holding the target's last
|
||||||
# most 7 draft tokens per verification step (llama.cpp asserts
|
# layers.
|
||||||
# n_max <= block_size - 1); 7 is also what the drafter was published with. The
|
SPEC_TYPE=draft-dspark,ngram-mod
|
||||||
# drafter rides ROCm1, the card that ends up holding the target's last layers.
|
SPEC_DRAFT_N_MAX=6
|
||||||
SPEC_TYPE=draft-dflash,ngram-mod
|
|
||||||
SPEC_DRAFT_N_MAX=7
|
|
||||||
SPEC_DRAFT_NGL=999
|
SPEC_DRAFT_NGL=999
|
||||||
SPEC_DRAFT_DEVICE=ROCm1
|
SPEC_DRAFT_DEVICE=ROCm1
|
||||||
SPEC_DRAFT_TYPE_K=f16
|
SPEC_DRAFT_TYPE_K=f16
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Serve ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF @ UD-Q8_K_XL as
|
# Serve ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF @ UD-Q8_K_XL as
|
||||||
# frank-medium on Dankserver's two Radeon AI PRO R9700 (gfx1201), with the
|
# frank-medium on Dankserver's two Radeon AI PRO R9700 (gfx1201), with the
|
||||||
# z-lab DFlash2 block-diffusion drafter, using mainline llama.cpp.
|
# magnitudedev DSpark block-diffusion drafter, using mainline llama.cpp.
|
||||||
#
|
#
|
||||||
# Runtime: llama.cpp master fb27a525 (2026-09-16), a gfx1201 HIP build that
|
# Runtime: llama.cpp master fb27a525 (2026-09-16), a gfx1201 HIP build that
|
||||||
# bundles its own ROCm 7.14 runtime (RPATH $ORIGIN/../lib), produced by
|
# bundles its own ROCm 7.14 runtime (RPATH $ORIGIN/../lib), produced by
|
||||||
@@ -117,7 +117,7 @@ fi
|
|||||||
SPEC_ARGS=()
|
SPEC_ARGS=()
|
||||||
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
|
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
|
||||||
SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
||||||
if [[ "$SPEC_TYPE" == *dflash* ]]; then
|
if [[ "$SPEC_TYPE" == *dflash* || "$SPEC_TYPE" == *dspark* ]]; then
|
||||||
if [[ ! -r "$DRAFT_MODEL" ]]; then
|
if [[ ! -r "$DRAFT_MODEL" ]]; then
|
||||||
echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
|
echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
# day. What this seam pins is what the flags alone do not say: boot persistence
|
# day. What this seam pins is what the flags alone do not say: boot persistence
|
||||||
# for a NATIVE runtime (no generated quadlet is involved), that BOTH cards work
|
# for a NATIVE runtime (no generated quadlet is involved), that BOTH cards work
|
||||||
# on ONE model through a layer split with no collective in the path, that the
|
# on ONE model through a layer split with no collective in the path, that the
|
||||||
# DFlash2 drafter is wired with a depth its block size actually allows, that the
|
# DSpark drafter is wired with a depth its block size actually allows, that the
|
||||||
# runtime and payload are pinned rather than floated, and that one slot's KV
|
# runtime and payload are pinned rather than floated, and that one slot's KV
|
||||||
# still fits a 32 GiB card beside the weights it holds.
|
# still fits a 32 GiB card beside the weights it holds.
|
||||||
|
|
||||||
@@ -21,7 +21,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|||||||
# Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic.
|
# Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic.
|
||||||
CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it
|
CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it
|
||||||
MODEL_BYTES=31457990112 # UD-Q8_K_XL target
|
MODEL_BYTES=31457990112 # UD-Q8_K_XL target
|
||||||
DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
|
DRAFT_BYTES=1359284737 # DSpark Q8_0 drafter
|
||||||
|
|
||||||
@test "frank-medium's unit, serve script, env and its build/fetch scripts exist" {
|
@test "frank-medium's unit, serve script, env and its build/fetch scripts exist" {
|
||||||
[ -f "$UNIT" ]
|
[ -f "$UNIT" ]
|
||||||
@@ -65,8 +65,8 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
|
|||||||
# The chain is the measured winner (see the runbook's A/B): free where the
|
# The chain is the measured winner (see the runbook's A/B): free where the
|
||||||
# matcher cannot match, ~2.6x the drafter alone on prompt-echo traffic. A
|
# matcher cannot match, ~2.6x the drafter alone on prompt-echo traffic. A
|
||||||
# bare `draft-dflash` here would silently give that up.
|
# bare `draft-dflash` here would silently give that up.
|
||||||
grep -qE '^SPEC_TYPE=draft-dflash,ngram-mod$' "$ENV_FILE"
|
grep -qE '^SPEC_TYPE=draft-dspark,ngram-mod$' "$ENV_FILE"
|
||||||
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE"
|
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DSpark-Q8_0\.gguf$' "$ENV_FILE"
|
||||||
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
|
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
|
||||||
grep -q -- '--spec-draft-model "$DRAFT_MODEL"' "$SERVE"
|
grep -q -- '--spec-draft-model "$DRAFT_MODEL"' "$SERVE"
|
||||||
# ngram-mod's own window -- and only when the type asks for it, so a pure
|
# ngram-mod's own window -- and only when the type asks for it, so a pure
|
||||||
@@ -85,11 +85,11 @@ end = next(i for i in range(guard + 1, len(lines))
|
|||||||
if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent)
|
if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent)
|
||||||
assert '--spec-ngram-mod-n-match' in chr(10).join(lines[guard:end]), 'ngram flags must sit in the ngram branch'
|
assert '--spec-ngram-mod-n-match' in chr(10).join(lines[guard:end]), 'ngram flags must sit in the ngram branch'
|
||||||
"
|
"
|
||||||
# DFlash2 drafts into a block of 8, so at most 7 draft tokens per
|
# DSpark drafts into a block of 7, so at most 6 draft tokens per
|
||||||
# verification step -- llama.cpp asserts n_max <= block_size - 1.
|
# verification step -- llama.cpp asserts n_max <= block_size - 1.
|
||||||
local n_max
|
local n_max
|
||||||
n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")"
|
n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||||
[ -n "$n_max" ] && [ "$n_max" -ge 1 ] && [ "$n_max" -le 7 ]
|
[ -n "$n_max" ] && [ "$n_max" -ge 1 ] && [ "$n_max" -le 6 ]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "a draft MODEL is passed only for DFlash types, never for MTP" {
|
@test "a draft MODEL is passed only for DFlash types, never for MTP" {
|
||||||
@@ -124,7 +124,7 @@ print('draft-model wiring ok')
|
|||||||
|
|
||||||
@test "the payload is fetched resumably and verified by size" {
|
@test "the payload is fetched resumably and verified by size" {
|
||||||
grep -q 'UD-Q8_K_XL\.gguf' "$FETCH"
|
grep -q 'UD-Q8_K_XL\.gguf' "$FETCH"
|
||||||
grep -q 'Qwen3\.8-27B-DFlash2-Q8_0\.gguf' "$FETCH"
|
grep -q 'Qwen3\.8-27B-DSpark-Q8_0\.gguf' "$FETCH"
|
||||||
grep -q 'curl -fL -C -' "$FETCH"
|
grep -q 'curl -fL -C -' "$FETCH"
|
||||||
grep -q 'verifying sizes' "$FETCH"
|
grep -q 'verifying sizes' "$FETCH"
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user