frank-medium: orcarouter Q8_0 + DFlash2 Q4_K_M, 2x160k, balanced 32:33
Operator call 2026-09-19: switch the target to orcarouter/Qwen3.8-27B-Uncensored-GGUF Q8_0 (gated repo; the fetch script now requires an HF token and verifies sha256, not just size), take the drafter to the Q4_K_M rung of z-lab/Qwen3.8-27B-DFlash2-GGUF, cut the pool to two 163840-token conversations (327680, entirely inside the trained 262144 window), and balance the tensors across the cards (32:33) in place of the 35:30 x16-lean. The in-file MTP head stays one SPEC_TYPE change away.
This commit is contained in:
Frank committed
1 parent
374041cd7a
commit
d374ab963f
4 files changed
+210
-236
No files matched your search
@@ -1,84 +1,109 @@
|
||||
#!/usr/bin/env bash
|
||||
# the target GGUF, the BF16 vision projector and the DFlash2 draft model.
|
||||
# Fetch the frank-medium payload: the target GGUF, the f16 vision projector and
|
||||
# the DFlash2 draft model.
|
||||
#
|
||||
# bash scripts/download-frank-medium-gguf.sh
|
||||
# bash scripts/download-frank-medium-gguf.sh --verify-only
|
||||
#
|
||||
# Both repositories are public (anonymous download; no token needed). If a
|
||||
# token exists at ~/.cache/huggingface/token it is used, which keeps a 31 GB
|
||||
# transfer clear of the anonymous rate limit. The script is resumable
|
||||
# (curl -C -) and checks every artifact's exact byte count afterwards, so a
|
||||
# truncated file fails here instead of load-crashing the server.
|
||||
# The target repo (orcarouter) is GATED: anonymous requests get a 401, so a
|
||||
# token is REQUIRED for it. The script reads ~/.cache/huggingface/token
|
||||
# (mode 600) or the HF_TOKEN environment variable, and passes it to curl
|
||||
# out-of-band via a mode-600 -K file (a -H header is world-readable in
|
||||
# /proc/<pid>/cmdline -- the 2026-09-16 HF-token leak). The drafter repo
|
||||
# (z-lab) is public; the same token is used for it when present.
|
||||
#
|
||||
# WHY THESE TWO FILES:
|
||||
# * Target: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at
|
||||
# UD-Q8_K_XL -- Unsloth Dynamic 3.0 quant (per-tensor types from Unsloth's
|
||||
# own quant of the same size, on Unsloth's importance matrix). The file
|
||||
# carries the model's MTP layer (`blk.64.nextn.*`), so llama.cpp can also
|
||||
# draft from the model itself with no second copy in VRAM.
|
||||
# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q8_0 -- the block-diffusion
|
||||
# WHY THESE FILES:
|
||||
# * Target: orcarouter/Qwen3.8-27B-Uncensored-GGUF at Q8_0 -- the
|
||||
# operator's requested quant (2026-09-19). Arch qwen35, 262144 trained
|
||||
# window, and the file carries the model's MTP layer (blk.64.nextn.*), so
|
||||
# llama.cpp can also draft from the model itself with no second copy in
|
||||
# VRAM.
|
||||
# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q4_K_M -- the block-diffusion
|
||||
# drafter (5 layers, hidden 5120, block_size 8, a 2048-token sliding
|
||||
# window). Q8_0 is 1.92 GiB against BF16's 3.6 GiB and the drafter's job is
|
||||
# throughput, not output quality (the target verifies every drafted token).
|
||||
# window). Q4_K_M is 1.06 GiB against Q8_0's 1.92 GiB and the drafter's
|
||||
# job is throughput, not output quality (the target verifies every drafted
|
||||
# token).
|
||||
#
|
||||
# The script is resumable (curl -C -) and checks every artifact's exact byte
|
||||
# count AND sha256, so a truncated or mis-fetched file fails here instead of
|
||||
# load-crashing the server.
|
||||
set -euo pipefail
|
||||
|
||||
MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
||||
MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
||||
MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf"
|
||||
MODEL_SIZE=31457990112
|
||||
# The BF16 vision projector from the same repo; llama-server autoloads any
|
||||
# mmproj*.gguf beside the target, so it must land in MODEL_DIR.
|
||||
MM_PROJ_FILE="mmproj-BF16.gguf"
|
||||
MM_PROJ_SIZE=931145888
|
||||
MODEL_REPO="orcarouter/Qwen3.8-27B-Uncensored-GGUF"
|
||||
MODEL_DIR="/home/dank/models/Qwen3.8-27B-Uncensored-GGUF"
|
||||
MODEL_FILE="Qwen3.8-27B-Uncensored-Q8_0.gguf"
|
||||
MODEL_SIZE=29047084416
|
||||
MODEL_SHA256=31756fca94beca71ea4b8706d6fdc896dab2a3c6376ab0c1863b98512a24f8d6
|
||||
# The f16 vision projector from the same repo; the serve script names it via
|
||||
# --mmproj (no autoload in this build), so it must land in MODEL_DIR.
|
||||
MM_PROJ_FILE="mmproj-Qwen3.8-27B-Uncensored-f16.gguf"
|
||||
MM_PROJ_SIZE=931145984
|
||||
MM_PROJ_SHA256=add205b7bfdb3f71f6da36b0a82aa20928dd829a920878c602628cdfbebc5288
|
||||
|
||||
DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF"
|
||||
DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF"
|
||||
DRAFT_FILE="Qwen3.8-27B-DFlash2-Q8_0.gguf"
|
||||
DRAFT_SIZE=2056414816
|
||||
DRAFT_FILE="Qwen3.8-27B-DFlash2-Q4_K_M.gguf"
|
||||
DRAFT_SIZE=1143006816
|
||||
DRAFT_SHA256=1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd
|
||||
|
||||
mode="download"
|
||||
[[ "${1:-}" == "--verify-only" ]] && mode="verify"
|
||||
|
||||
token=""
|
||||
auth=()
|
||||
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
|
||||
token="${HF_TOKEN:-}"
|
||||
if [[ -z "$token" && -r "$HOME/.cache/huggingface/token" ]]; then
|
||||
token="$(cat "$HOME/.cache/huggingface/token")"
|
||||
auth=(-H "Authorization: Bearer $token")
|
||||
else
|
||||
echo "no HF token at ~/.cache/huggingface/token; downloading anonymously"
|
||||
fi
|
||||
|
||||
auth_file=""
|
||||
cleanup() { [[ -n "$auth_file" ]] && rm -f "$auth_file"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
dl() { # dl <repo> <dir> <file>
|
||||
mkdir -p "$2"
|
||||
echo " fetching $3"
|
||||
curl -fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600 \
|
||||
"${auth[@]}" "https://huggingface.co/$1/resolve/main/$3"
|
||||
local args=(-fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600)
|
||||
[[ -n "$auth_file" ]] && args+=(-K "$auth_file")
|
||||
curl "${args[@]}" "https://huggingface.co/$1/resolve/main/$3"
|
||||
}
|
||||
|
||||
if [[ "$mode" == "download" ]]; then
|
||||
echo "target: $MODEL_REPO"
|
||||
if [[ -z "$token" ]]; then
|
||||
echo "ERROR: the target repo is gated and no HF token is available" >&2
|
||||
echo " (set HF_TOKEN or write it to ~/.cache/huggingface/token, mode 600)" >&2
|
||||
exit 1
|
||||
fi
|
||||
auth_file="$(mktemp)"
|
||||
printf 'header = "Authorization: Bearer %s"\n' "$token" > "$auth_file"
|
||||
chmod 600 "$auth_file"
|
||||
|
||||
echo "target: $MODEL_REPO (gated)"
|
||||
dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE"
|
||||
echo "projector: $MODEL_REPO"
|
||||
echo "projector: $MODEL_REPO (gated)"
|
||||
dl "$MODEL_REPO" "$MODEL_DIR" "$MM_PROJ_FILE"
|
||||
echo "draft: $DRAFT_REPO"
|
||||
dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE"
|
||||
fi
|
||||
|
||||
echo "verifying sizes"
|
||||
echo "verifying sizes and sha256"
|
||||
ok=1
|
||||
check() { # check <path> <expected>
|
||||
local act
|
||||
check() { # check <path> <expected-size> <expected-sha256>
|
||||
local act act_sha
|
||||
act="$(stat -c%s "$1" 2>/dev/null || echo 0)"
|
||||
if [[ "$act" -eq "$2" ]]; then
|
||||
echo " OK $1 ($act bytes)"
|
||||
else
|
||||
echo " BAD $1 ($act bytes, want $2)" >&2
|
||||
if [[ "$act" -ne "$2" ]]; then
|
||||
echo " BAD-SIZE $1 ($act bytes, want $2)" >&2
|
||||
ok=0
|
||||
return
|
||||
fi
|
||||
act_sha="$(sha256sum "$1" | cut -d' ' -f1)"
|
||||
if [[ "$act_sha" != "$3" ]]; then
|
||||
echo " BAD-SHA $1 ($act_sha)" >&2
|
||||
ok=0
|
||||
return
|
||||
fi
|
||||
echo " OK $1 ($act bytes)"
|
||||
}
|
||||
check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE"
|
||||
check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"
|
||||
check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE"
|
||||
[[ -z "$token" ]] && echo "note: no token used; the byte counts above are the integrity check"
|
||||
check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE" "$MODEL_SHA256"
|
||||
check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE" "$MM_PROJ_SHA256"
|
||||
check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE" "$DRAFT_SHA256"
|
||||
[[ "$ok" -eq 1 ]] || exit 1
|
||||
echo "all artifacts present and sized correctly"
|
||||
echo "all artifacts present, sized and hashed correctly"
|
||||
Reference in new issue
Block a user