frank-medium: orcarouter Q8_0 + DFlash2 Q4_K_M, 2x160k, balanced 32:33
Operator call 2026-09-19: switch the target to orcarouter/Qwen3.8-27B-Uncensored-GGUF Q8_0 (gated repo; the fetch script now requires an HF token and verifies sha256, not just size), take the drafter to the Q4_K_M rung of z-lab/Qwen3.8-27B-DFlash2-GGUF, cut the pool to two 163840-token conversations (327680, entirely inside the trained 262144 window), and balance the tensors across the cards (32:33) in place of the 35:30 x16-lean. The in-file MTP head stays one SPEC_TYPE change away.
This commit is contained in:
Frank
@@ -1175,9 +1175,9 @@ deploy_llm_small() {
|
||||
# PCIe, once per forward pass. That is the "pipeline parallelism" the endpoint
|
||||
# wanted, with no collective anywhere in it.
|
||||
LLM_MEDIUM_BIN="${LLM_MEDIUM_BIN:-/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}"
|
||||
LLM_MEDIUM_MODEL="${LLM_MEDIUM_MODEL:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}"
|
||||
LLM_MEDIUM_DRAFT="${LLM_MEDIUM_DRAFT:-/home/dank/models/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q8_0.gguf}"
|
||||
LLM_MEDIUM_MMPROJ="${LLM_MEDIUM_MMPROJ:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/mmproj-BF16.gguf}"
|
||||
LLM_MEDIUM_MODEL="${LLM_MEDIUM_MODEL:-/home/dank/models/Qwen3.8-27B-Uncensored-GGUF/Qwen3.8-27B-Uncensored-Q8_0.gguf}"
|
||||
LLM_MEDIUM_DRAFT="${LLM_MEDIUM_DRAFT:-/home/dank/models/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
|
||||
LLM_MEDIUM_MMPROJ="${LLM_MEDIUM_MMPROJ:-/home/dank/models/Qwen3.8-27B-Uncensored-GGUF/mmproj-Qwen3.8-27B-Uncensored-f16.gguf}"
|
||||
|
||||
# The three vLLM user units are retired HERE, not left to rot: the proxy
|
||||
# published 7770 on the host network, so the llama-server below cannot bind
|
||||
|
||||
@@ -1,84 +1,109 @@
|
||||
#!/usr/bin/env bash
|
||||
# the target GGUF, the BF16 vision projector and the DFlash2 draft model.
|
||||
# Fetch the frank-medium payload: the target GGUF, the f16 vision projector and
|
||||
# the DFlash2 draft model.
|
||||
#
|
||||
# bash scripts/download-frank-medium-gguf.sh
|
||||
# bash scripts/download-frank-medium-gguf.sh --verify-only
|
||||
#
|
||||
# Both repositories are public (anonymous download; no token needed). If a
|
||||
# token exists at ~/.cache/huggingface/token it is used, which keeps a 31 GB
|
||||
# transfer clear of the anonymous rate limit. The script is resumable
|
||||
# (curl -C -) and checks every artifact's exact byte count afterwards, so a
|
||||
# truncated file fails here instead of load-crashing the server.
|
||||
# The target repo (orcarouter) is GATED: anonymous requests get a 401, so a
|
||||
# token is REQUIRED for it. The script reads ~/.cache/huggingface/token
|
||||
# (mode 600) or the HF_TOKEN environment variable, and passes it to curl
|
||||
# out-of-band via a mode-600 -K file (a -H header is world-readable in
|
||||
# /proc/<pid>/cmdline -- the 2026-09-16 HF-token leak). The drafter repo
|
||||
# (z-lab) is public; the same token is used for it when present.
|
||||
#
|
||||
# WHY THESE TWO FILES:
|
||||
# * Target: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at
|
||||
# UD-Q8_K_XL -- Unsloth Dynamic 3.0 quant (per-tensor types from Unsloth's
|
||||
# own quant of the same size, on Unsloth's importance matrix). The file
|
||||
# carries the model's MTP layer (`blk.64.nextn.*`), so llama.cpp can also
|
||||
# draft from the model itself with no second copy in VRAM.
|
||||
# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q8_0 -- the block-diffusion
|
||||
# WHY THESE FILES:
|
||||
# * Target: orcarouter/Qwen3.8-27B-Uncensored-GGUF at Q8_0 -- the
|
||||
# operator's requested quant (2026-09-19). Arch qwen35, 262144 trained
|
||||
# window, and the file carries the model's MTP layer (blk.64.nextn.*), so
|
||||
# llama.cpp can also draft from the model itself with no second copy in
|
||||
# VRAM.
|
||||
# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q4_K_M -- the block-diffusion
|
||||
# drafter (5 layers, hidden 5120, block_size 8, a 2048-token sliding
|
||||
# window). Q8_0 is 1.92 GiB against BF16's 3.6 GiB and the drafter's job is
|
||||
# throughput, not output quality (the target verifies every drafted token).
|
||||
# window). Q4_K_M is 1.06 GiB against Q8_0's 1.92 GiB and the drafter's
|
||||
# job is throughput, not output quality (the target verifies every drafted
|
||||
# token).
|
||||
#
|
||||
# The script is resumable (curl -C -) and checks every artifact's exact byte
|
||||
# count AND sha256, so a truncated or mis-fetched file fails here instead of
|
||||
# load-crashing the server.
|
||||
set -euo pipefail
|
||||
|
||||
MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
||||
MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
||||
MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf"
|
||||
MODEL_SIZE=31457990112
|
||||
# The BF16 vision projector from the same repo; llama-server autoloads any
|
||||
# mmproj*.gguf beside the target, so it must land in MODEL_DIR.
|
||||
MM_PROJ_FILE="mmproj-BF16.gguf"
|
||||
MM_PROJ_SIZE=931145888
|
||||
MODEL_REPO="orcarouter/Qwen3.8-27B-Uncensored-GGUF"
|
||||
MODEL_DIR="/home/dank/models/Qwen3.8-27B-Uncensored-GGUF"
|
||||
MODEL_FILE="Qwen3.8-27B-Uncensored-Q8_0.gguf"
|
||||
MODEL_SIZE=29047084416
|
||||
MODEL_SHA256=31756fca94beca71ea4b8706d6fdc896dab2a3c6376ab0c1863b98512a24f8d6
|
||||
# The f16 vision projector from the same repo; the serve script names it via
|
||||
# --mmproj (no autoload in this build), so it must land in MODEL_DIR.
|
||||
MM_PROJ_FILE="mmproj-Qwen3.8-27B-Uncensored-f16.gguf"
|
||||
MM_PROJ_SIZE=931145984
|
||||
MM_PROJ_SHA256=add205b7bfdb3f71f6da36b0a82aa20928dd829a920878c602628cdfbebc5288
|
||||
|
||||
DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF"
|
||||
DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF"
|
||||
DRAFT_FILE="Qwen3.8-27B-DFlash2-Q8_0.gguf"
|
||||
DRAFT_SIZE=2056414816
|
||||
DRAFT_FILE="Qwen3.8-27B-DFlash2-Q4_K_M.gguf"
|
||||
DRAFT_SIZE=1143006816
|
||||
DRAFT_SHA256=1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd
|
||||
|
||||
mode="download"
|
||||
[[ "${1:-}" == "--verify-only" ]] && mode="verify"
|
||||
|
||||
token=""
|
||||
auth=()
|
||||
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
|
||||
token="${HF_TOKEN:-}"
|
||||
if [[ -z "$token" && -r "$HOME/.cache/huggingface/token" ]]; then
|
||||
token="$(cat "$HOME/.cache/huggingface/token")"
|
||||
auth=(-H "Authorization: Bearer $token")
|
||||
else
|
||||
echo "no HF token at ~/.cache/huggingface/token; downloading anonymously"
|
||||
fi
|
||||
|
||||
auth_file=""
|
||||
cleanup() { [[ -n "$auth_file" ]] && rm -f "$auth_file"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
dl() { # dl <repo> <dir> <file>
|
||||
mkdir -p "$2"
|
||||
echo " fetching $3"
|
||||
curl -fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600 \
|
||||
"${auth[@]}" "https://huggingface.co/$1/resolve/main/$3"
|
||||
local args=(-fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600)
|
||||
[[ -n "$auth_file" ]] && args+=(-K "$auth_file")
|
||||
curl "${args[@]}" "https://huggingface.co/$1/resolve/main/$3"
|
||||
}
|
||||
|
||||
if [[ "$mode" == "download" ]]; then
|
||||
echo "target: $MODEL_REPO"
|
||||
if [[ -z "$token" ]]; then
|
||||
echo "ERROR: the target repo is gated and no HF token is available" >&2
|
||||
echo " (set HF_TOKEN or write it to ~/.cache/huggingface/token, mode 600)" >&2
|
||||
exit 1
|
||||
fi
|
||||
auth_file="$(mktemp)"
|
||||
printf 'header = "Authorization: Bearer %s"\n' "$token" > "$auth_file"
|
||||
chmod 600 "$auth_file"
|
||||
|
||||
echo "target: $MODEL_REPO (gated)"
|
||||
dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE"
|
||||
echo "projector: $MODEL_REPO"
|
||||
echo "projector: $MODEL_REPO (gated)"
|
||||
dl "$MODEL_REPO" "$MODEL_DIR" "$MM_PROJ_FILE"
|
||||
echo "draft: $DRAFT_REPO"
|
||||
dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE"
|
||||
fi
|
||||
|
||||
echo "verifying sizes"
|
||||
echo "verifying sizes and sha256"
|
||||
ok=1
|
||||
check() { # check <path> <expected>
|
||||
local act
|
||||
check() { # check <path> <expected-size> <expected-sha256>
|
||||
local act act_sha
|
||||
act="$(stat -c%s "$1" 2>/dev/null || echo 0)"
|
||||
if [[ "$act" -eq "$2" ]]; then
|
||||
echo " OK $1 ($act bytes)"
|
||||
else
|
||||
echo " BAD $1 ($act bytes, want $2)" >&2
|
||||
if [[ "$act" -ne "$2" ]]; then
|
||||
echo " BAD-SIZE $1 ($act bytes, want $2)" >&2
|
||||
ok=0
|
||||
return
|
||||
fi
|
||||
act_sha="$(sha256sum "$1" | cut -d' ' -f1)"
|
||||
if [[ "$act_sha" != "$3" ]]; then
|
||||
echo " BAD-SHA $1 ($act_sha)" >&2
|
||||
ok=0
|
||||
return
|
||||
fi
|
||||
echo " OK $1 ($act bytes)"
|
||||
}
|
||||
check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE"
|
||||
check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"
|
||||
check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE"
|
||||
[[ -z "$token" ]] && echo "note: no token used; the byte counts above are the integrity check"
|
||||
check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE" "$MODEL_SHA256"
|
||||
check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE" "$MM_PROJ_SHA256"
|
||||
check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE" "$DRAFT_SHA256"
|
||||
[[ "$ok" -eq 1 ]] || exit 1
|
||||
echo "all artifacts present and sized correctly"
|
||||
echo "all artifacts present, sized and hashed correctly"
|
||||
|
||||
+80
-139
@@ -1,7 +1,7 @@
|
||||
# Tunables for frank-medium (ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP
|
||||
# UD-Q8_K_XL + z-lab/Qwen3.8-27B-DFlash2-GGUF Q8_0 drafter), mainline llama.cpp
|
||||
# on Dankserver, 2x AMD Radeon AI PRO R9700 (gfx1201). Since 2026-09-16 this
|
||||
# is the SOLE LLM runtime on the box: the vllm-radiance replica pair and its
|
||||
# Tunables for frank-medium (orcarouter/Qwen3.8-27B-Uncensored-GGUF Q8_0 +
|
||||
# z-lab/Qwen3.8-27B-DFlash2-GGUF Q4_K_M drafter), mainline llama.cpp on
|
||||
# Dankserver, 2x AMD Radeon AI PRO R9700 (gfx1201). Since 2026-09-16 this is
|
||||
# the SOLE LLM runtime on the box: the vllm-radiance replica pair and its
|
||||
# routing proxy are retired.
|
||||
#
|
||||
# Hardware: two R9700 (Navi 48, RDNA4, 32,624 MiB each), no XGMI, different
|
||||
@@ -31,17 +31,17 @@
|
||||
# The ref is pinned by TWO features: DFlash2 (ggml-org/llama.cpp#27342 -- the
|
||||
# drafter is detected from its GGUF) and qwen35 MTP. Do not float it.
|
||||
#
|
||||
# Model: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF @ UD-Q8_K_XL,
|
||||
# 29.29 GiB, arch qwen35, Unsloth Dynamic 3.0 (per-tensor types from Unsloth's
|
||||
# own quant of the same size, on Unsloth's importance matrix). Abliteration is
|
||||
# orcarouter's single refusal direction projected out of 131 tensors; the vision
|
||||
# tower loads from mmproj-BF16.gguf (see NO_MMPROJ). The file also
|
||||
# carries the model's MTP head (blk.64.nextn.*), so a file-level switch to
|
||||
# --spec-type draft-mtp is available without a second copy in VRAM.
|
||||
# Model: orcarouter/Qwen3.8-27B-Uncensored-GGUF @ Q8_0, 27.0 GiB, arch qwen35
|
||||
# (verified from the GGUF header 2026-09-19: 262144 trained window, 24 heads /
|
||||
# 4 KV heads, nextn_predict_layers=1). The file ALSO carries the model's MTP
|
||||
# head (blk.64.nextn.*), so --spec-type draft-mtp is available without a second
|
||||
# copy in VRAM -- one SPEC_TYPE change, the drafter below stays on disk.
|
||||
#
|
||||
# Drafter: z-lab/Qwen3.8-27B-DFlash2-GGUF @ Q8_0, 1.92 GiB: 5 layers, hidden
|
||||
# Drafter: z-lab/Qwen3.8-27B-DFlash2-GGUF @ Q4_K_M, 1.06 GiB (operator call
|
||||
# 2026-09-19: the Q4 rung -- the drafter's job is throughput, not output
|
||||
# quality, and the target verifies every drafted token). 5 layers, hidden
|
||||
# 5120, a block-diffusion proposal (block_size 8) under a 2048-token sliding
|
||||
# window. Decoding stays lossless -- the target verifies every drafted token.
|
||||
# window (dflash.attention.sliding_window=2048, verified from the header).
|
||||
# The window is what makes it cheap HERE: its KV is bounded at 2048 positions
|
||||
# (~40 MiB) instead of spanning the context (see SIZING).
|
||||
#
|
||||
@@ -52,10 +52,14 @@
|
||||
# in the attention layers only, plus ~0.16 GiB per sequence of f32 DeltaNet
|
||||
# state at any depth.
|
||||
#
|
||||
# SIZING, measured (rocm-smi, 2026-09-16). Weights split across the two cards;
|
||||
# the drafter AND its KV ride the second. The loader's compute buffers are part
|
||||
# of what is seen below -- they are allocated at load, not per request, and they
|
||||
# scale with --ubatch-size:
|
||||
# SIZING, the deployed shape (32:33 / ctx 327680 / 6144x512 / two slots /
|
||||
# DFlash2 Q4_K_M), measured 2026-09-19 after the model swap (rocm-smi):
|
||||
#
|
||||
# card 0 (x16) 26.9 GiB used, 5.7 GiB free
|
||||
# card 1 (x1) 27.8 GiB used, 4.8 GiB free
|
||||
#
|
||||
# The predecessor's measured table (rocm-smi, 2026-09-16, UD-Q8_K_XL target)
|
||||
# stays as the calibration history:
|
||||
#
|
||||
# CTX split batch slots drafter GPU 0 free GPU 1 free note (ubatch 1536)
|
||||
# 262144 1,1 2048 1 DFlash2 8.64 GiB 4.43 GiB day's start
|
||||
@@ -71,7 +75,7 @@
|
||||
# batch block below):
|
||||
#
|
||||
# 262144 36,29 6144 512 2 DFlash2 6.01 GiB 7.32 GiB ubatch only
|
||||
# 262144 35,30 6144 512 2 DFlash2 6.44 GiB 6.90 GiB <- deployed
|
||||
# 262144 35,30 6144 512 2 DFlash2 6.44 GiB 6.90 GiB
|
||||
#
|
||||
# ONE -ts STEP IS ONE LAYER, AND LAYERS ARE NOT THE SAME SIZE. This is a hybrid
|
||||
# stack -- 16 x (3 Gated DeltaNet + 1 Gated Attention) blocks -- so moving a
|
||||
@@ -80,36 +84,27 @@
|
||||
# were measured, which is why the balance is found by measurement rather than by
|
||||
# interpolating between splits.
|
||||
#
|
||||
# 262144 is the window the model ships (n_ctx_train) and it fits with room, so
|
||||
# the fleet gets the whole thing; the arithmetic that used to cap this endpoint
|
||||
# at 114688 was the vLLM predecessor's, not this runtime's. If a future change
|
||||
# adds VRAM (a longer drafter, quantized KV, more slots) re-read the numbers
|
||||
# rather than assuming, and prefer CTX_SIZE over --gpu-layers.
|
||||
# WHY THE SPLIT IS 32:33 (operator call, 2026-09-19). The operator asked for
|
||||
# the tensors BALANCED across the two cards, which for 65 layers is 32:33 --
|
||||
# the most even split that exists. The predecessor's 35:30 skew existed to lean
|
||||
# work onto the x16 card (ROCm0), because ROCm1's Gen4 x1 chipset path made it
|
||||
# the 98 % utilization bottleneck; the measured trade of the balanced point is
|
||||
# that the x1 card runs hotter while the x16 card idles a little more, and the
|
||||
# operator accepted that for symmetry. Do not re-skew without asking.
|
||||
#
|
||||
# WHY THE SPLIT IS 35:30. Three effects decide it, and all three are measured:
|
||||
# * THE DRAFTER AND ITS OUTPUT LAYERS RIDE ROCm1. It carries the drafter's
|
||||
# weights, the draft graph and the last target layers, so it takes fewer
|
||||
# than half the layer count.
|
||||
# * THE UBATCH BUFFERS COST ROCm1 MORE THAN ROCm0, and shrinking them pays
|
||||
# ROCm1 back more: 1536 -> 512 freed 1.30 GiB on ROCm0 but 2.41 GiB on
|
||||
# ROCm1 at this shape. Read the other way, that is exactly why the smaller
|
||||
# batch moved the balance toward ROCm1 -- and cost ROCm0 one layer.
|
||||
# * THE LAYER GRANULARITY IS THE FLOOR ON HOW WELL ANY SPLIT CAN BALANCE. One
|
||||
# -ts step is one layer -- ~0.43 GiB DeltaNet, ~1.43 GiB attention -- so the
|
||||
# reachable minimum gap is comparable to one step rather than to zero. 35:30
|
||||
# sits at 0.46 GiB, i.e. one DeltaNet layer's worth, against 1.31 GiB at
|
||||
# 36:29.
|
||||
# It also leaves the most room: 6.44 / 6.90 GiB free clears the operator's 1.5
|
||||
# GiB-per-card floor by ~5 GiB, so no headroom has to be bought out of CTX_SIZE
|
||||
# and the full window stays.
|
||||
# THE TWO CARDS' PATHS ARE NOT SYMMETRIC (measured, 2026-09-17): ROCm0
|
||||
# (03:00.0) hangs off CPU root port 00:01.1 at Gen4 x16 end to end (~31.5 GB/s);
|
||||
# ROCm1 (0b:00.0) rides the B650 chipset through bridges that each negotiate
|
||||
# Gen4 x1 (~1.97 GB/s). The layer-split boundary itself is not the cost
|
||||
# (~5.2 MB of activation per 512-token ubatch rides the wire); the x1 card is
|
||||
# COMPUTE-bound, not PCIe-bound.
|
||||
#
|
||||
# A FAULT TO WATCH. A 39:26 split, at 1x batch / 1 slot, died on its eighth
|
||||
# request: the amdgpu driver logged SDMA0 UTCL2 page faults and the engine
|
||||
# surfaced `ROCm error: an illegal memory access was encountered` -> ggml_abort,
|
||||
# i.e. a fault in the cross-card boundary copy, this layer split's hot path. It
|
||||
# has not reproduced (36:29 served this file's verification battery and the e2e
|
||||
# gate, and 35:30 is the deployed shape), but the signature is worth recognising
|
||||
# -- it is NOT an OOM, and no amount of headroom prevents it.
|
||||
# has not reproduced since, but the signature is worth recognising -- it is NOT
|
||||
# an OOM, and no amount of headroom prevents it.
|
||||
#
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP reads
|
||||
# those bare vars; 2026-08-13 incident). llama.cpp takes --host/--port/--temp
|
||||
@@ -119,15 +114,15 @@
|
||||
LLAMA_BIN=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server
|
||||
|
||||
# Payload, fetched with scripts/download-frank-medium-gguf.sh.
|
||||
MODEL_DIR=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF
|
||||
MODEL_FILE=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf
|
||||
# The BF16 vision projector, same repo as the target. llama-server autoloads
|
||||
# any mmproj*.gguf beside the model; the serve script refuses to start without
|
||||
# it when vision is enabled, so a missing file is a deploy error, not a
|
||||
# text-only server that pretends to be multimodal.
|
||||
MM_PROJ_FILE=mmproj-BF16.gguf
|
||||
MODEL_DIR=/home/dank/models/Qwen3.8-27B-Uncensored-GGUF
|
||||
MODEL_FILE=Qwen3.8-27B-Uncensored-Q8_0.gguf
|
||||
# The f16 vision projector, same repo as the target. llama-server loads it via
|
||||
# the explicit --mmproj flag (no autoload in this build); the serve script
|
||||
# refuses to start without it when vision is enabled, so a missing file is a
|
||||
# deploy error, not a text-only server that pretends to be multimodal.
|
||||
MM_PROJ_FILE=mmproj-Qwen3.8-27B-Uncensored-f16.gguf
|
||||
DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF
|
||||
DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q8_0.gguf
|
||||
DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q4_K_M.gguf
|
||||
|
||||
# Fleet identity. 7770 is the fleet's dankserver endpoint; the alias is the id
|
||||
# at /v1/models and what every client asks for.
|
||||
@@ -141,64 +136,23 @@ HIP_VISIBLE_DEVICES=0,1
|
||||
DEVICES=ROCm0,ROCm1
|
||||
SPLIT_MODE=layer
|
||||
|
||||
# 35:30 -- operator call 2026-09-17: lean the work onto the x16 card. The
|
||||
# two R9700s do NOT have symmetric PCIe paths. ROCm0 (03:00.0) hangs off
|
||||
# CPU root port 00:01.1 at Gen4 x16 end to end (~31.5 GB/s); ROCm1
|
||||
# (0b:00.0) rides the B650 chipset through bridges that each negotiate
|
||||
# Gen4 x1 (~1.97 GB/s). Live rocm-smi showed the x1 card at 98% busy
|
||||
# against 40% on the x16 card: the MTP drafter, the output layers and the
|
||||
# heavier compute buffers are structurally pinned to the card holding the
|
||||
# LAST layers (the draft context shares the target's weights there), so
|
||||
# the bottlenecked card was doing most of the work. Moving one layer to
|
||||
# ROCm0 shifts ~0.43 GiB of DeltaNet weights (layer 34 is a DeltaNet --
|
||||
# the 4k..4k+2 position of block 8) off the x1 card and gives the healthy
|
||||
# card the bigger share. The layer-split boundary itself is not the cost
|
||||
# (~5.2 MB of activation per 512-token ubatch rides the wire), but the
|
||||
# card behind the x1 link should not be the saturated one while its
|
||||
# x16-sibling idles.
|
||||
#
|
||||
# WHY NOT FURTHER. One -ts step is one LAYER: ~0.43 GiB (DeltaNet) or
|
||||
# ~1.43 GiB (attention). The next step, 36:29, would cross the
|
||||
# full-attention layer at 35: the seam fit puts card 0 at 31.41 GiB
|
||||
# (free 0.45 GiB, under the 1.0 GiB floor) with a 2.22 GiB gap (over the
|
||||
# 1.483 GiB slack) -- both gates fail. 35:30 (fit: 30.59 / 30.01, gap
|
||||
# 0.58) is the farthest safe shift toward the healthy card at this pool.
|
||||
# Do not move it without re-measuring.
|
||||
|
||||
# MEASURED 2026-09-18 (isolated :7790, 2k-token decode, AVG rocm-smi util over
|
||||
# the request -- peak util lies, a layer split touches both cards every token):
|
||||
#
|
||||
# split draft_dev decode AVG util0(x16) AVG util1(x1)
|
||||
# 35,30 ROCm1 31.52 41 % 98 % <- deployed
|
||||
# 36,29 ROCm1 31.49 43 % 98 %
|
||||
# 35,30 ROCm0 31.52 42 % 98 %
|
||||
# 36,29 ROCm0 31.48 43 % 98 %
|
||||
#
|
||||
# The x1 card stays the 98 % bottleneck in EVERY configuration. The lever that
|
||||
# was tried is moving the MTP drafter to the x16 card (SPEC_DRAFT_DEVICE=ROCm0):
|
||||
# it does NOT flip the balance, because the drafter is a small nextn head and the
|
||||
# per-step cost is dominated by the VERIFY phase -- both cards run the full target
|
||||
# -- plus the LOGITS, which are pinned to the card holding the last layers. A
|
||||
# target-layer move (36:29) buys 2 points on the x16 card and erodes its VRAM
|
||||
# margin (fit: free 0.45 GiB) for nothing. The x1 card is COMPUTE-bound, not
|
||||
# PCIe-bound (~5.2 MB of activation per 512-token ubatch crosses the link);
|
||||
# saturating it costs no tokens beyond what any R9700 would. 35:30 is the
|
||||
# balance; do not chase the x16 card past it.
|
||||
TENSOR_SPLIT=35,30
|
||||
# 32:33 -- balanced (operator call 2026-09-19): 65 layers split as evenly as
|
||||
# they can be, ROCm1 (the last-layers card, which also carries the drafter and
|
||||
# the output layers) holding the extra one. See the WHY THE SPLIT IS 32:33
|
||||
# block above for the trade against the predecessor's 35:30 x16-lean.
|
||||
TENSOR_SPLIT=32,33
|
||||
|
||||
NGL=999
|
||||
# Auto-fit has no business resizing a context we set explicitly.
|
||||
FIT=off
|
||||
FLASH_ATTN=on
|
||||
|
||||
# 393216 positions over TWO slots, so each conversation gets 196608 --
|
||||
# 327680 positions over TWO slots, so each conversation gets 163840 --
|
||||
# --parallel DIVIDES --ctx-size, so more slots would shrink every
|
||||
# conversation rather than grow the cache. The checkpoint's trained window is
|
||||
# 262144 and the GGUF carries NO rope scaling (freq_base 1e7, no YaRN keys),
|
||||
# so positions beyond 262144 extrapolate on the base rope: it runs, and 1.5x
|
||||
# is a modest ask of a rope, but the top 131k of the pool is untrained
|
||||
# territory.
|
||||
CTX_SIZE=393216
|
||||
# conversation rather than grow the cache. 163840 is inside the checkpoint's
|
||||
# trained 262144 window, so the whole pool is trained territory (no rope
|
||||
# extrapolation, unlike the predecessor's 393216).
|
||||
CTX_SIZE=327680
|
||||
PARALLEL=2
|
||||
|
||||
# MEASURED 2026-09-16 with scripts/bench-batch-ubatch.sh on this box: llama-bench,
|
||||
@@ -258,20 +212,13 @@ CTX_CHECKPOINTS=0
|
||||
# (server_prompt_cache, tools/server/server-task.cpp): update() evicts
|
||||
# oldest-first past the cap, and alloc() REFUSES outright any single state
|
||||
# bigger than the whole cap. A state is the serialized KV of a whole sequence
|
||||
# plus its DeltaNet state plus the MTP draft context's own KV, so at this shape
|
||||
# it costs 68 KiB per token --
|
||||
# plus its DeltaNet state plus the draft context's own KV, so at this shape it
|
||||
# costs ~68 KiB per token --
|
||||
# 16 full-attention layers x 4 kv heads x 256 dim x 2 B x 2 (K,V) = 64 KiB
|
||||
# plus 4096 B for the draft context (one nextn layer)
|
||||
# -- which makes the cap a TOKEN budget in disguise: 8192 MiB held ~120k
|
||||
# tokens, 16384 MiB holds ~240k.
|
||||
#
|
||||
# DOUBLED 8192 -> 16384 on 2026-09-18 because the old cap was BINDING, not
|
||||
# theoretical: alloc() refused 137 states in the two days from 2026-09-16 with
|
||||
# - prompt state size 11388.990 MiB exceeds cache size limit 8192.000 MiB, skipping
|
||||
# i.e. 11.0-11.4 GiB states -- conversations of ~170k tokens at the law above --
|
||||
# so every deep conversation was silently uncacheable and re-prefilled from
|
||||
# scratch on its next turn. At 16384 a FULL 196608-token slot (~12.8 GiB) fits
|
||||
# with ~3 GiB of the budget spare for a second entry.
|
||||
# plus the draft context's window-bounded KV (a ~40 MiB constant for DFlash2)
|
||||
# -- which makes the cap a TOKEN budget in disguise: 16384 MiB holds ~240k
|
||||
# tokens, i.e. a FULL 163840-token slot (~10.6 GiB) with ~5 GiB of the budget
|
||||
# spare for a second entry.
|
||||
#
|
||||
# The cost is host RAM, allocated on demand and never pinned (plain heap
|
||||
# vectors), so no VRAM number above moves. dankserver has 62 GiB beside the
|
||||
@@ -300,24 +247,21 @@ REASONING_BUDGET=-1
|
||||
CHAT_TEMPLATE_FILE=
|
||||
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}'
|
||||
|
||||
# Speculative decoding: the model's OWN MTP head -- the blk.<n>.nextn.*
|
||||
# tensors inside the target file. No sidecar file, no second copy of the
|
||||
# weights: the draft context runs on the main model (shares_model), so the
|
||||
# cost is one nextn layer's KV (4096 B/position) plus small buffers on the
|
||||
# card that holds the last layers. This is what makes the 393216 pool fit:
|
||||
# the DFlash2 sidecar's 1.92 GiB file and its buffers are off ROCm1.
|
||||
# n_max: 3 is the measured MTP winner (see the table below -- n=7 accepts 33 %
|
||||
# of 147, n=3 accepts 58 % of 72). The drafter rides ROCm1.
|
||||
SPEC_TYPE=draft-mtp
|
||||
SPEC_DRAFT_N_MAX=3
|
||||
# Speculative decoding: the DFlash2 sidecar (operator call 2026-09-19). The
|
||||
# target file also carries its own MTP head (nextn_predict_layers=1), so
|
||||
# draft-mtp is one SPEC_TYPE change away -- the drafter file stays on disk.
|
||||
# n_max 7 is the measured DFlash2 winner (see the table below); the drafter
|
||||
# rides ROCm1, the card that holds the last layers.
|
||||
SPEC_TYPE=draft-dflash
|
||||
SPEC_DRAFT_N_MAX=7
|
||||
SPEC_DRAFT_NGL=999
|
||||
SPEC_DRAFT_DEVICE=ROCm1
|
||||
SPEC_DRAFT_TYPE_K=f16
|
||||
SPEC_DRAFT_TYPE_V=f16
|
||||
|
||||
# MTP (the model's own head) vs the DFlash2 drafter -- MEASURED 2026-09-16
|
||||
# (temperature 0, 1k prompt / 512 out; the autoregressive baseline and both
|
||||
# MTP depths were run in one session, same binary, same prompts):
|
||||
# DFlash2 vs the model's own MTP head -- MEASURED 2026-09-16 on the predecessor
|
||||
# payload (temperature 0, 1k prompt / 512 out; the autoregressive baseline and
|
||||
# both MTP depths were run in one session, same binary, same prompts):
|
||||
#
|
||||
# none 17.7 tok/s card1 26.07 GB
|
||||
# draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB
|
||||
@@ -326,11 +270,10 @@ SPEC_DRAFT_TYPE_V=f16
|
||||
#
|
||||
# All four produced IDENTICAL greedy text, so verification is lossless
|
||||
# whichever proposer runs. DFlash2 is the faster proposer (2.46x over
|
||||
# autoregressive, 34 % over the best MTP); the operator switched to MTP on
|
||||
# 2026-09-17 for the VRAM, not the speed: the MTP head adds ~3.3 GB to the
|
||||
# card that holds it, against 2.06 GB for the drafter's file, and it is the
|
||||
# only drafter that fits the 393216 pool. Do not switch this without
|
||||
# re-measuring.
|
||||
# autoregressive, 34 % over the best MTP). The MTP head adds ~3.3 GB to the
|
||||
# card that holds it, against 1.06 GB for the Q4_K_M drafter's file -- at the
|
||||
# 327680 pool the drafter's window-bounded KV (~40 MiB) leaves headroom on
|
||||
# both cards, so the speed wins.
|
||||
#
|
||||
# THE TWO CANNOT BE CHAINED. `--spec-type draft-mtp,draft-dflash` dies at load:
|
||||
# context type MTP requested but model doesn't contain MTP layers
|
||||
@@ -344,7 +287,7 @@ SPEC_DRAFT_TYPE_V=f16
|
||||
# ngram-mod: the draftless n-gram matcher (CPU hash table in host RAM, no VRAM,
|
||||
# no model file, no draft context). It joins the chain unconditionally --
|
||||
# unlike MTP, which needs a draft context and therefore cannot share one with
|
||||
# DFlash2. NOT CURRENTLY ACTIVE: SPEC_TYPE is draft-mtp alone, and the serve
|
||||
# DFlash2. NOT CURRENTLY ACTIVE: SPEC_TYPE is draft-dflash alone, and the serve
|
||||
# script only passes --spec-ngram-mod-* when the type contains "ngram". The
|
||||
# window below is retained because the DFlash2 + ngram chain was measured free
|
||||
# on prose and 2.6x on prompt-echo -- one SPEC_TYPE change brings it back.
|
||||
@@ -352,7 +295,7 @@ SPEC_DRAFT_TYPE_V=f16
|
||||
# WINDOW MEASURED 2026-09-16 (chain at each window, decode tok/s):
|
||||
#
|
||||
# n_min/n_max prose 1k/512 prompt-echo 1.5k/400
|
||||
# 48/64 43.9 200.6 <- deployed
|
||||
# 48/64 43.9 200.6
|
||||
# 16/32 36.0 161.0
|
||||
# 8/16 36.2 113.8
|
||||
#
|
||||
@@ -369,9 +312,7 @@ SPEC_NGRAM_N_MATCH=24
|
||||
SPEC_NGRAM_N_MIN=48
|
||||
SPEC_NGRAM_N_MAX=64
|
||||
|
||||
# Vision enabled: llama-server autoloads mmproj-BF16.gguf from MODEL_DIR
|
||||
# (888 MiB, lands on ROCm0 -- the fit above charges it there). frank-small
|
||||
# (Dankclaw, RTX 3090) remains the fleet's dedicated image endpoint; this
|
||||
# makes frank-medium multimodal too, which is why the seam flips from
|
||||
# "text only" to "vision loads with the model".
|
||||
# Vision enabled: llama-server loads mmproj-Qwen3.8-27B-Uncensored-f16.gguf
|
||||
# (931 MiB, lands on ROCm0 -- the fit charges it there) via the explicit
|
||||
# --mmproj flag. This keeps frank-medium multimodal across the model swap.
|
||||
NO_MMPROJ=0
|
||||
|
||||
@@ -20,8 +20,8 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
|
||||
# Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic.
|
||||
CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it
|
||||
MODEL_BYTES=31457990112 # UD-Q8_K_XL target
|
||||
DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
|
||||
MODEL_BYTES=29047084416 # orcarouter Q8_0 target
|
||||
DRAFT_BYTES=1143006816 # DFlash2 Q4_K_M drafter
|
||||
|
||||
@test "frank-medium's unit, serve script, env and its build/fetch scripts exist" {
|
||||
[ -f "$UNIT" ]
|
||||
@@ -48,17 +48,19 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
|
||||
grep -q '^SPLIT_MODE=layer$' "$ENV_FILE"
|
||||
grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE"
|
||||
# The split is a layer PAIR: the shares add up to the model's 65 blocks (one
|
||||
# -ts unit per layer) and ROCm1 -- the card that also carries the drafter and
|
||||
# the output layers -- takes the FEWER of the two. WHICH pair balances the
|
||||
# cards is the seam test's job, which recomputes it from this file; pinning a
|
||||
# literal here would only mean that re-measuring the balance breaks a test
|
||||
# about parallelism.
|
||||
# -ts unit per layer). The operator asked for the tensors BALANCED across
|
||||
# the two cards (2026-09-19), which for 65 layers is 32:33 -- the most even
|
||||
# split that exists, with ROCm1 (the last-layers card, which also carries
|
||||
# the drafter and the output layers) holding the extra one. WHICH pair
|
||||
# balances the cards is the fit test's job, which recomputes it from this
|
||||
# file; pinning a literal here would only mean that re-measuring the
|
||||
# balance breaks a test about parallelism.
|
||||
local split_hi split_lo
|
||||
split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")"
|
||||
[ -n "$split_hi" ] && [ -n "$split_lo" ]
|
||||
[ $(( split_hi + split_lo )) -eq 65 ]
|
||||
[ "$split_lo" -lt "$split_hi" ]
|
||||
[ $(( split_lo - split_hi )) -le 1 ]
|
||||
grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE"
|
||||
grep -q -- '--device "$DEVICES"' "$SERVE"
|
||||
grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE"
|
||||
@@ -72,24 +74,26 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
|
||||
grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE"
|
||||
}
|
||||
|
||||
@test "the drafter is the model's own MTP head, at the measured depth" {
|
||||
# MTP's head lives in the target file (blk.<n>.nextn.*): no sidecar, no
|
||||
# second copy of the weights, and it is the only drafter that fits the
|
||||
# 393216 pool. n_max 3 is the measured MTP winner -- n=7 accepts 33 % of
|
||||
# 147, n=3 accepts 58 % of 72 (the env's table; identical greedy text at
|
||||
# every depth, so this is pure speed).
|
||||
grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE"
|
||||
@test "the drafter is the DFlash2 sidecar at the measured depth" {
|
||||
# The DFlash2 sidecar is the deployed drafter (operator call 2026-09-19).
|
||||
# n_max 7 is the measured DFlash2 winner -- 43.5 tok/s at 41 % acceptance
|
||||
# against 32.4 for the in-file MTP head at n=3 (the env's table; identical
|
||||
# greedy text at every depth, so this is pure speed). The target file
|
||||
# also carries its own MTP head (nextn_predict_layers=1), so draft-mtp is
|
||||
# one SPEC_TYPE change away; the drafter file stays on disk for that.
|
||||
grep -qE '^SPEC_TYPE=draft-dflash$' "$ENV_FILE"
|
||||
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
|
||||
local n_max
|
||||
n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
[ -n "$n_max" ] && [ "$n_max" -eq 3 ]
|
||||
# The DFlash2 sidecar stays on disk and in the fetch script -- one
|
||||
# SPEC_TYPE change brings the faster chain back -- and ngram-mod's window
|
||||
# is retained with it, so neither may be dropped from the env.
|
||||
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE"
|
||||
[ -n "$n_max" ] && [ "$n_max" -eq 7 ]
|
||||
# The Q4_K_M rung is the operator's requested drafter quant -- the
|
||||
# drafter's job is throughput, not output quality, and the target
|
||||
# verifies every drafted token. ngram-mod's window is retained in the env
|
||||
# for the chain, so it may not be dropped.
|
||||
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q4_K_M\.gguf$' "$ENV_FILE"
|
||||
grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE"
|
||||
# ...and neither is loaded: ngram-mod's flags only go out when the type
|
||||
# contains ngram, so a pure MTP configuration is not handed them.
|
||||
# contains ngram, so a pure dflash configuration is not handed them.
|
||||
python3 -c "
|
||||
lines = open('$SERVE').read().splitlines()
|
||||
guard = next(i for i, l in enumerate(lines) if '*ngram*' in l)
|
||||
@@ -130,11 +134,14 @@ print('draft-model wiring ok')
|
||||
grep -q 'frank-medium.env' "$SERVE"
|
||||
}
|
||||
|
||||
@test "the payload is fetched resumably and verified by size" {
|
||||
grep -q 'UD-Q8_K_XL\.gguf' "$FETCH"
|
||||
grep -q 'Qwen3\.8-27B-DFlash2-Q8_0\.gguf' "$FETCH"
|
||||
grep -q 'curl -fL -C -' "$FETCH"
|
||||
grep -q 'verifying sizes' "$FETCH"
|
||||
@test "the payload is fetched resumably and verified by size and sha256" {
|
||||
grep -q 'Qwen3\.8-27B-Uncensored-Q8_0\.gguf' "$FETCH"
|
||||
grep -q 'Qwen3\.8-27B-DFlash2-Q4_K_M\.gguf' "$FETCH"
|
||||
grep -q -- '-fL -C -' "$FETCH"
|
||||
grep -q 'verifying sizes and sha256' "$FETCH"
|
||||
# The target repo is gated: the script must fail loud without a token
|
||||
# rather than write a 401 body to disk and report success.
|
||||
grep -q 'no HF token is available' "$FETCH"
|
||||
}
|
||||
|
||||
@test "both cards stay balanced and keep the operator's 1.5 GiB floor" {
|
||||
@@ -158,9 +165,9 @@ print('draft-model wiring ok')
|
||||
# measured 6.44 / 6.90 GiB free. The fit reproduces card 1 to 0.00 GiB
|
||||
# and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the
|
||||
# deployed point is re-measured on the host after every split/ctx/batch
|
||||
# move (the current one is 35:30 / 393216 / MTP, moved from 34:31 on
|
||||
# 2026-09-17 to lean work off ROCm1's Gen4-x1 chipset path -- see the
|
||||
# TENSOR_SPLIT header in frank-medium.env).
|
||||
# move (the current one is 32:33 / 327680 / DFlash2 Q4_K_M, the balanced
|
||||
# split the operator asked for on 2026-09-19 -- see the TENSOR_SPLIT
|
||||
# header in frank-medium.env).
|
||||
#
|
||||
# card 0 = its share of the weights and of the KV (qwen35 spends KV on
|
||||
# its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim
|
||||
@@ -197,22 +204,22 @@ print('draft-model wiring ok')
|
||||
# allocation.
|
||||
local kv_pos=65536 draft_kv_pos=20480 mtp_kv_pos=4096 floor slack
|
||||
local card_hi card_lo diff
|
||||
local mmproj=931145888 # mmproj-BF16.gguf, autoloaded onto the primary device
|
||||
local mmproj=931145984 # mmproj-Qwen3.8-27B-Uncensored-f16.gguf, on the primary device
|
||||
# 1.5 GiB was the floor until the 2x196k shape: the 34:31 / 393216 /
|
||||
# MTP point measured 28.91 / 30.78 GiB, leaving 2.95 / 1.08 GiB, so the
|
||||
# floor is the measured 1.08 GiB rounded down. The margin for one big
|
||||
# allocation is thinner than it used to be. The layer-toward-ROCm0
|
||||
# lever has since been spent (35:30, 2026-09-17): if a large allocation
|
||||
# fails at load now, the next lever is a ctx cut, not another layer.
|
||||
# floor is the measured 1.08 GiB rounded down. The 32:33 / 327680 /
|
||||
# DFlash2 point is roomier than that (fit: 6.97 / 4.89 GiB free), so the
|
||||
# floor stays at the measured 1.0 GiB -- if a large allocation fails at
|
||||
# load, the next lever is a ctx cut, not a floor change.
|
||||
floor=$(( 1024 * 1024 * 1024 )) # 1.0 GiB per card
|
||||
# The deployed point (35:30 at ctx 393216 with MTP) has a fit gap of
|
||||
# 0.58 GiB -- closer to balanced than its 34:31 predecessor (1.06 GiB).
|
||||
# The slack sits at 1.483 GiB: it accepts the deployed split and still
|
||||
# rejects the next step toward ROCm0, 36:29, which fails BOTH gates
|
||||
# (gap 2.22 GiB, card 0 free 0.45 GiB under the floor). (The 36:29 /
|
||||
# 327680 / DFlash2 configuration's slack was 3225 MiB -- its 3.15 GiB
|
||||
# gap was by construction, and is no longer the deployed shape.)
|
||||
slack=$(( 1483 * 1024 * 1024 ))
|
||||
# The deployed point (32:33 at ctx 327680 with DFlash2) has a fit gap of
|
||||
# 2082 MiB -- the balanced split the operator asked for, not the
|
||||
# tightest reachable one. The slack sits at 2150 MiB: it accepts the
|
||||
# deployed split and rejects a two-layer re-skew (34:31 / 31:34 both
|
||||
# move the fit gap well past it). The x16-lean 35:30 that served the
|
||||
# predecessor payload (gap 0.58 GiB) no longer applies: the operator
|
||||
# chose symmetry over the x1-card bottleneck.
|
||||
slack=$(( 2150 * 1024 * 1024 ))
|
||||
card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \
|
||||
+ kv_pos * ctx * split_hi / (split_hi + split_lo) \
|
||||
+ 381898377 \
|
||||
@@ -243,8 +250,9 @@ print('draft-model wiring ok')
|
||||
diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff ))
|
||||
[ "$diff" -le "$slack" ]
|
||||
# ...and the window may not wander far past what the checkpoint was
|
||||
# trained for (262144; the GGUF carries no rope scaling, so the top 131k
|
||||
# of the 393216 pool extrapolates on the base rope).
|
||||
# trained for (262144; the GGUF carries no rope scaling, so positions
|
||||
# past that extrapolate on the base rope). The deployed 327680 pool is
|
||||
# two 163840 slots -- entirely trained territory.
|
||||
[ "$ctx" -le 393216 ]
|
||||
}
|
||||
|
||||
@@ -280,16 +288,16 @@ print('draft-model wiring ok')
|
||||
grep -q -- '--cache-prompt' "$SERVE"
|
||||
}
|
||||
|
||||
@test "vision is enabled: the BF16 projector loads with the model" {
|
||||
@test "vision is enabled: the f16 projector loads with the model" {
|
||||
grep -q '^NO_MMPROJ=0$' "$ENV_FILE"
|
||||
grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE"
|
||||
grep -q '^MM_PROJ_FILE=mmproj-Qwen3\.8-27B-Uncensored-f16\.gguf$' "$ENV_FILE"
|
||||
# Explicit flag, not autoload: this build starts text-only without it,
|
||||
# and refuses to start without the projector file, so a missing file is
|
||||
# a deploy error, not a silent text-only server.
|
||||
grep -q -- '--mmproj "$MM_PROJ"' "$SERVE"
|
||||
grep -q 'vision is enabled but the projector is missing' "$SERVE"
|
||||
# And the download script fetches and byte-verifies it.
|
||||
grep -q '^MM_PROJ_SIZE=931145888$' "$FETCH"
|
||||
grep -q '^MM_PROJ_SIZE=931145984$' "$FETCH"
|
||||
grep -q -- 'check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"' "$FETCH"
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user