dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-medium: orcarouter Q8_0 + DFlash2 Q4_K_M, 2x160k, balanced 32:33

Browse Source 
Operator call 2026-09-19: switch the target to
orcarouter/Qwen3.8-27B-Uncensored-GGUF Q8_0 (gated repo; the fetch script now
requires an HF token and verifies sha256, not just size), take the drafter to
the Q4_K_M rung of z-lab/Qwen3.8-27B-DFlash2-GGUF, cut the pool to two
163840-token conversations (327680, entirely inside the trained 262144
window), and balance the tensors across the cards (32:33) in place of the
35:30 x16-lean.  The in-file MTP head stays one SPEC_TYPE change away.
This commit is contained in:
Frank
2026-09-19 02:10:38 -07:00
parent 374041cd7a
commit d374ab963f
4 changed files with 210 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/deploy-remote.sh
+3 -3
View File
@@ -1175,9 +1175,9 @@ deploy_llm_small() {
# PCIe, once per forward pass. That is the "pipeline parallelism" the endpoint # PCIe, once per forward pass. That is the "pipeline parallelism" the endpoint
# wanted, with no collective anywhere in it. # wanted, with no collective anywhere in it.
LLM_MEDIUM_BIN="${LLM_MEDIUM_BIN:-/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}" LLM_MEDIUM_BIN="${LLM_MEDIUM_BIN:-/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}"
LLM_MEDIUM_MODEL="${LLM_MEDIUM_MODEL:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}" LLM_MEDIUM_MODEL="${LLM_MEDIUM_MODEL:-/home/dank/models/Qwen3.8-27B-Uncensored-GGUF/Qwen3.8-27B-Uncensored-Q8_0.gguf}"
LLM_MEDIUM_DRAFT="${LLM_MEDIUM_DRAFT:-/home/dank/models/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q8_0.gguf}" LLM_MEDIUM_DRAFT="${LLM_MEDIUM_DRAFT:-/home/dank/models/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
LLM_MEDIUM_MMPROJ="${LLM_MEDIUM_MMPROJ:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/mmproj-BF16.gguf}" LLM_MEDIUM_MMPROJ="${LLM_MEDIUM_MMPROJ:-/home/dank/models/Qwen3.8-27B-Uncensored-GGUF/mmproj-Qwen3.8-27B-Uncensored-f16.gguf}"
# The three vLLM user units are retired HERE, not left to rot: the proxy # The three vLLM user units are retired HERE, not left to rot: the proxy
# published 7770 on the host network, so the llama-server below cannot bind # published 7770 on the host network, so the llama-server below cannot bind
scripts/download-frank-medium-gguf.sh
+72 -47
View File
@@ -1,84 +1,109 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# the target GGUF, the BF16 vision projector and the DFlash2 draft model. # Fetch the frank-medium payload: the target GGUF, the f16 vision projector and
# the DFlash2 draft model.
# #
# bash scripts/download-frank-medium-gguf.sh # bash scripts/download-frank-medium-gguf.sh
# bash scripts/download-frank-medium-gguf.sh --verify-only # bash scripts/download-frank-medium-gguf.sh --verify-only
# #
# Both repositories are public (anonymous download; no token needed). If a # The target repo (orcarouter) is GATED: anonymous requests get a 401, so a
# token exists at ~/.cache/huggingface/token it is used, which keeps a 31 GB # token is REQUIRED for it. The script reads ~/.cache/huggingface/token
# transfer clear of the anonymous rate limit. The script is resumable # (mode 600) or the HF_TOKEN environment variable, and passes it to curl
# (curl -C -) and checks every artifact's exact byte count afterwards, so a # out-of-band via a mode-600 -K file (a -H header is world-readable in
# truncated file fails here instead of load-crashing the server. # /proc/<pid>/cmdline -- the 2026-09-16 HF-token leak). The drafter repo
# (z-lab) is public; the same token is used for it when present.
# #
# WHY THESE TWO FILES: # WHY THESE FILES:
# * Target: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at # * Target: orcarouter/Qwen3.8-27B-Uncensored-GGUF at Q8_0 -- the
# UD-Q8_K_XL -- Unsloth Dynamic 3.0 quant (per-tensor types from Unsloth's # operator's requested quant (2026-09-19). Arch qwen35, 262144 trained
# own quant of the same size, on Unsloth's importance matrix). The file # window, and the file carries the model's MTP layer (blk.64.nextn.*), so
# carries the model's MTP layer (`blk.64.nextn.*`), so llama.cpp can also # llama.cpp can also draft from the model itself with no second copy in
# draft from the model itself with no second copy in VRAM. # VRAM.
# * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q8_0 -- the block-diffusion # * Draft: z-lab/Qwen3.8-27B-DFlash2-GGUF at Q4_K_M -- the block-diffusion
# drafter (5 layers, hidden 5120, block_size 8, a 2048-token sliding # drafter (5 layers, hidden 5120, block_size 8, a 2048-token sliding
# window). Q8_0 is 1.92 GiB against BF16's 3.6 GiB and the drafter's job is # window). Q4_K_M is 1.06 GiB against Q8_0's 1.92 GiB and the drafter's
# throughput, not output quality (the target verifies every drafted token). # job is throughput, not output quality (the target verifies every drafted
# token).
#
# The script is resumable (curl -C -) and checks every artifact's exact byte
# count AND sha256, so a truncated or mis-fetched file fails here instead of
# load-crashing the server.
set -euo pipefail set -euo pipefail
MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF" MODEL_REPO="orcarouter/Qwen3.8-27B-Uncensored-GGUF"
MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF" MODEL_DIR="/home/dank/models/Qwen3.8-27B-Uncensored-GGUF"
MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf" MODEL_FILE="Qwen3.8-27B-Uncensored-Q8_0.gguf"
MODEL_SIZE=31457990112 MODEL_SIZE=29047084416
# The BF16 vision projector from the same repo; llama-server autoloads any MODEL_SHA256=31756fca94beca71ea4b8706d6fdc896dab2a3c6376ab0c1863b98512a24f8d6
# mmproj*.gguf beside the target, so it must land in MODEL_DIR. # The f16 vision projector from the same repo; the serve script names it via
MM_PROJ_FILE="mmproj-BF16.gguf" # --mmproj (no autoload in this build), so it must land in MODEL_DIR.
MM_PROJ_SIZE=931145888 MM_PROJ_FILE="mmproj-Qwen3.8-27B-Uncensored-f16.gguf"
MM_PROJ_SIZE=931145984
MM_PROJ_SHA256=add205b7bfdb3f71f6da36b0a82aa20928dd829a920878c602628cdfbebc5288
DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF" DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF"
DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF" DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF"
DRAFT_FILE="Qwen3.8-27B-DFlash2-Q8_0.gguf" DRAFT_FILE="Qwen3.8-27B-DFlash2-Q4_K_M.gguf"
DRAFT_SIZE=2056414816 DRAFT_SIZE=1143006816
DRAFT_SHA256=1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd
mode="download" mode="download"
[[ "${1:-}" == "--verify-only" ]] && mode="verify" [[ "${1:-}" == "--verify-only" ]] && mode="verify"
token="" token="${HF_TOKEN:-}"
auth=() if [[ -z "$token" && -r "$HOME/.cache/huggingface/token" ]]; then
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
token="$(cat "$HOME/.cache/huggingface/token")" token="$(cat "$HOME/.cache/huggingface/token")"
auth=(-H "Authorization: Bearer $token")
else
echo "no HF token at ~/.cache/huggingface/token; downloading anonymously"
fi fi
auth_file=""
cleanup() { [[ -n "$auth_file" ]] && rm -f "$auth_file"; }
trap cleanup EXIT
dl() { # dl <repo> <dir> <file> dl() { # dl <repo> <dir> <file>
mkdir -p "$2" mkdir -p "$2"
echo " fetching $3" echo " fetching $3"
curl -fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600 \ local args=(-fL -C - -o "$2/$3" --retry 10 --retry-delay 5 --retry-max-time 21600)
"${auth[@]}" "https://huggingface.co/$1/resolve/main/$3" [[ -n "$auth_file" ]] && args+=(-K "$auth_file")
curl "${args[@]}" "https://huggingface.co/$1/resolve/main/$3"
} }
if [[ "$mode" == "download" ]]; then if [[ "$mode" == "download" ]]; then
echo "target: $MODEL_REPO" if [[ -z "$token" ]]; then
echo "ERROR: the target repo is gated and no HF token is available" >&2
echo " (set HF_TOKEN or write it to ~/.cache/huggingface/token, mode 600)" >&2
exit 1
fi
auth_file="$(mktemp)"
printf 'header = "Authorization: Bearer %s"\n' "$token" > "$auth_file"
chmod 600 "$auth_file"
echo "target: $MODEL_REPO (gated)"
dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE" dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE"
echo "projector: $MODEL_REPO" echo "projector: $MODEL_REPO (gated)"
dl "$MODEL_REPO" "$MODEL_DIR" "$MM_PROJ_FILE" dl "$MODEL_REPO" "$MODEL_DIR" "$MM_PROJ_FILE"
echo "draft: $DRAFT_REPO" echo "draft: $DRAFT_REPO"
dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE" dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE"
fi fi
echo "verifying sizes" echo "verifying sizes and sha256"
ok=1 ok=1
check() { # check <path> <expected> check() { # check <path> <expected-size> <expected-sha256>
local act local act act_sha
act="$(stat -c%s "$1" 2>/dev/null || echo 0)" act="$(stat -c%s "$1" 2>/dev/null || echo 0)"
if [[ "$act" -eq "$2" ]]; then if [[ "$act" -ne "$2" ]]; then
echo " OK $1 ($act bytes)" echo " BAD-SIZE $1 ($act bytes, want $2)" >&2
else
echo " BAD $1 ($act bytes, want $2)" >&2
ok=0 ok=0
return
fi fi
act_sha="$(sha256sum "$1" | cut -d' ' -f1)"
if [[ "$act_sha" != "$3" ]]; then
echo " BAD-SHA $1 ($act_sha)" >&2
ok=0
return
fi
echo " OK $1 ($act bytes)"
} }
check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE" check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE" "$MODEL_SHA256"
check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE" check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE" "$MM_PROJ_SHA256"
check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE" check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE" "$DRAFT_SHA256"
[[ -z "$token" ]] && echo "note: no token used; the byte counts above are the integrity check"
[[ "$ok" -eq 1 ]] || exit 1 [[ "$ok" -eq 1 ]] || exit 1
echo "all artifacts present and sized correctly" echo "all artifacts present, sized and hashed correctly"
scripts/frank-medium.env
+80 -139
View File
@@ -1,7 +1,7 @@
# Tunables for frank-medium (ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP # Tunables for frank-medium (orcarouter/Qwen3.8-27B-Uncensored-GGUF Q8_0 +
# UD-Q8_K_XL + z-lab/Qwen3.8-27B-DFlash2-GGUF Q8_0 drafter), mainline llama.cpp # z-lab/Qwen3.8-27B-DFlash2-GGUF Q4_K_M drafter), mainline llama.cpp on
# on Dankserver, 2x AMD Radeon AI PRO R9700 (gfx1201). Since 2026-09-16 this # Dankserver, 2x AMD Radeon AI PRO R9700 (gfx1201). Since 2026-09-16 this is
# is the SOLE LLM runtime on the box: the vllm-radiance replica pair and its # the SOLE LLM runtime on the box: the vllm-radiance replica pair and its
# routing proxy are retired. # routing proxy are retired.
# #
# Hardware: two R9700 (Navi 48, RDNA4, 32,624 MiB each), no XGMI, different # Hardware: two R9700 (Navi 48, RDNA4, 32,624 MiB each), no XGMI, different
@@ -31,17 +31,17 @@
# The ref is pinned by TWO features: DFlash2 (ggml-org/llama.cpp#27342 -- the # The ref is pinned by TWO features: DFlash2 (ggml-org/llama.cpp#27342 -- the
# drafter is detected from its GGUF) and qwen35 MTP. Do not float it. # drafter is detected from its GGUF) and qwen35 MTP. Do not float it.
# #
# Model: ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF @ UD-Q8_K_XL, # Model: orcarouter/Qwen3.8-27B-Uncensored-GGUF @ Q8_0, 27.0 GiB, arch qwen35
# 29.29 GiB, arch qwen35, Unsloth Dynamic 3.0 (per-tensor types from Unsloth's # (verified from the GGUF header 2026-09-19: 262144 trained window, 24 heads /
# own quant of the same size, on Unsloth's importance matrix). Abliteration is # 4 KV heads, nextn_predict_layers=1). The file ALSO carries the model's MTP
# orcarouter's single refusal direction projected out of 131 tensors; the vision # head (blk.64.nextn.*), so --spec-type draft-mtp is available without a second
# tower loads from mmproj-BF16.gguf (see NO_MMPROJ). The file also # copy in VRAM -- one SPEC_TYPE change, the drafter below stays on disk.
# carries the model's MTP head (blk.64.nextn.*), so a file-level switch to
# --spec-type draft-mtp is available without a second copy in VRAM.
# #
# Drafter: z-lab/Qwen3.8-27B-DFlash2-GGUF @ Q8_0, 1.92 GiB: 5 layers, hidden # Drafter: z-lab/Qwen3.8-27B-DFlash2-GGUF @ Q4_K_M, 1.06 GiB (operator call
# 2026-09-19: the Q4 rung -- the drafter's job is throughput, not output
# quality, and the target verifies every drafted token). 5 layers, hidden
# 5120, a block-diffusion proposal (block_size 8) under a 2048-token sliding # 5120, a block-diffusion proposal (block_size 8) under a 2048-token sliding
# window. Decoding stays lossless -- the target verifies every drafted token. # window (dflash.attention.sliding_window=2048, verified from the header).
# The window is what makes it cheap HERE: its KV is bounded at 2048 positions # The window is what makes it cheap HERE: its KV is bounded at 2048 positions
# (~40 MiB) instead of spanning the context (see SIZING). # (~40 MiB) instead of spanning the context (see SIZING).
# #
@@ -52,10 +52,14 @@
# in the attention layers only, plus ~0.16 GiB per sequence of f32 DeltaNet # in the attention layers only, plus ~0.16 GiB per sequence of f32 DeltaNet
# state at any depth. # state at any depth.
# #
# SIZING, measured (rocm-smi, 2026-09-16). Weights split across the two cards; # SIZING, the deployed shape (32:33 / ctx 327680 / 6144x512 / two slots /
# the drafter AND its KV ride the second. The loader's compute buffers are part # DFlash2 Q4_K_M), measured 2026-09-19 after the model swap (rocm-smi):
# of what is seen below -- they are allocated at load, not per request, and they #
# scale with --ubatch-size: # card 0 (x16) 26.9 GiB used, 5.7 GiB free
# card 1 (x1) 27.8 GiB used, 4.8 GiB free
#
# The predecessor's measured table (rocm-smi, 2026-09-16, UD-Q8_K_XL target)
# stays as the calibration history:
# #
# CTX split batch slots drafter GPU 0 free GPU 1 free note (ubatch 1536) # CTX split batch slots drafter GPU 0 free GPU 1 free note (ubatch 1536)
# 262144 1,1 2048 1 DFlash2 8.64 GiB 4.43 GiB day's start # 262144 1,1 2048 1 DFlash2 8.64 GiB 4.43 GiB day's start
@@ -71,7 +75,7 @@
# batch block below): # batch block below):
# #
# 262144 36,29 6144 512 2 DFlash2 6.01 GiB 7.32 GiB ubatch only # 262144 36,29 6144 512 2 DFlash2 6.01 GiB 7.32 GiB ubatch only
# 262144 35,30 6144 512 2 DFlash2 6.44 GiB 6.90 GiB <- deployed # 262144 35,30 6144 512 2 DFlash2 6.44 GiB 6.90 GiB
# #
# ONE -ts STEP IS ONE LAYER, AND LAYERS ARE NOT THE SAME SIZE. This is a hybrid # ONE -ts STEP IS ONE LAYER, AND LAYERS ARE NOT THE SAME SIZE. This is a hybrid
# stack -- 16 x (3 Gated DeltaNet + 1 Gated Attention) blocks -- so moving a # stack -- 16 x (3 Gated DeltaNet + 1 Gated Attention) blocks -- so moving a
@@ -80,36 +84,27 @@
# were measured, which is why the balance is found by measurement rather than by # were measured, which is why the balance is found by measurement rather than by
# interpolating between splits. # interpolating between splits.
# #
# 262144 is the window the model ships (n_ctx_train) and it fits with room, so # WHY THE SPLIT IS 32:33 (operator call, 2026-09-19). The operator asked for
# the fleet gets the whole thing; the arithmetic that used to cap this endpoint # the tensors BALANCED across the two cards, which for 65 layers is 32:33 --
# at 114688 was the vLLM predecessor's, not this runtime's. If a future change # the most even split that exists. The predecessor's 35:30 skew existed to lean
# adds VRAM (a longer drafter, quantized KV, more slots) re-read the numbers # work onto the x16 card (ROCm0), because ROCm1's Gen4 x1 chipset path made it
# rather than assuming, and prefer CTX_SIZE over --gpu-layers. # the 98 % utilization bottleneck; the measured trade of the balanced point is
# that the x1 card runs hotter while the x16 card idles a little more, and the
# operator accepted that for symmetry. Do not re-skew without asking.
# #
# WHY THE SPLIT IS 35:30. Three effects decide it, and all three are measured: # THE TWO CARDS' PATHS ARE NOT SYMMETRIC (measured, 2026-09-17): ROCm0
# * THE DRAFTER AND ITS OUTPUT LAYERS RIDE ROCm1. It carries the drafter's # (03:00.0) hangs off CPU root port 00:01.1 at Gen4 x16 end to end (~31.5 GB/s);
# weights, the draft graph and the last target layers, so it takes fewer # ROCm1 (0b:00.0) rides the B650 chipset through bridges that each negotiate
# than half the layer count. # Gen4 x1 (~1.97 GB/s). The layer-split boundary itself is not the cost
# * THE UBATCH BUFFERS COST ROCm1 MORE THAN ROCm0, and shrinking them pays # (~5.2 MB of activation per 512-token ubatch rides the wire); the x1 card is
# ROCm1 back more: 1536 -> 512 freed 1.30 GiB on ROCm0 but 2.41 GiB on # COMPUTE-bound, not PCIe-bound.
# ROCm1 at this shape. Read the other way, that is exactly why the smaller
# batch moved the balance toward ROCm1 -- and cost ROCm0 one layer.
# * THE LAYER GRANULARITY IS THE FLOOR ON HOW WELL ANY SPLIT CAN BALANCE. One
# -ts step is one layer -- ~0.43 GiB DeltaNet, ~1.43 GiB attention -- so the
# reachable minimum gap is comparable to one step rather than to zero. 35:30
# sits at 0.46 GiB, i.e. one DeltaNet layer's worth, against 1.31 GiB at
# 36:29.
# It also leaves the most room: 6.44 / 6.90 GiB free clears the operator's 1.5
# GiB-per-card floor by ~5 GiB, so no headroom has to be bought out of CTX_SIZE
# and the full window stays.
# #
# A FAULT TO WATCH. A 39:26 split, at 1x batch / 1 slot, died on its eighth # A FAULT TO WATCH. A 39:26 split, at 1x batch / 1 slot, died on its eighth
# request: the amdgpu driver logged SDMA0 UTCL2 page faults and the engine # request: the amdgpu driver logged SDMA0 UTCL2 page faults and the engine
# surfaced `ROCm error: an illegal memory access was encountered` -> ggml_abort, # surfaced `ROCm error: an illegal memory access was encountered` -> ggml_abort,
# i.e. a fault in the cross-card boundary copy, this layer split's hot path. It # i.e. a fault in the cross-card boundary copy, this layer split's hot path. It
# has not reproduced (36:29 served this file's verification battery and the e2e # has not reproduced since, but the signature is worth recognising -- it is NOT
# gate, and 35:30 is the deployed shape), but the signature is worth recognising # an OOM, and no amount of headroom prevents it.
# -- it is NOT an OOM, and no amount of headroom prevents it.
# #
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP reads # WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP reads
# those bare vars; 2026-08-13 incident). llama.cpp takes --host/--port/--temp # those bare vars; 2026-08-13 incident). llama.cpp takes --host/--port/--temp
@@ -119,15 +114,15 @@
LLAMA_BIN=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server LLAMA_BIN=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server
# Payload, fetched with scripts/download-frank-medium-gguf.sh. # Payload, fetched with scripts/download-frank-medium-gguf.sh.
MODEL_DIR=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF MODEL_DIR=/home/dank/models/Qwen3.8-27B-Uncensored-GGUF
MODEL_FILE=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf MODEL_FILE=Qwen3.8-27B-Uncensored-Q8_0.gguf
# The BF16 vision projector, same repo as the target. llama-server autoloads # The f16 vision projector, same repo as the target. llama-server loads it via
# any mmproj*.gguf beside the model; the serve script refuses to start without # the explicit --mmproj flag (no autoload in this build); the serve script
# it when vision is enabled, so a missing file is a deploy error, not a # refuses to start without it when vision is enabled, so a missing file is a
# text-only server that pretends to be multimodal. # deploy error, not a text-only server that pretends to be multimodal.
MM_PROJ_FILE=mmproj-BF16.gguf MM_PROJ_FILE=mmproj-Qwen3.8-27B-Uncensored-f16.gguf
DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF
DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q8_0.gguf DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q4_K_M.gguf
# Fleet identity. 7770 is the fleet's dankserver endpoint; the alias is the id # Fleet identity. 7770 is the fleet's dankserver endpoint; the alias is the id
# at /v1/models and what every client asks for. # at /v1/models and what every client asks for.
@@ -141,64 +136,23 @@ HIP_VISIBLE_DEVICES=0,1
DEVICES=ROCm0,ROCm1 DEVICES=ROCm0,ROCm1
SPLIT_MODE=layer SPLIT_MODE=layer
# 35:30 -- operator call 2026-09-17: lean the work onto the x16 card. The # 32:33 -- balanced (operator call 2026-09-19): 65 layers split as evenly as
# two R9700s do NOT have symmetric PCIe paths. ROCm0 (03:00.0) hangs off # they can be, ROCm1 (the last-layers card, which also carries the drafter and
# CPU root port 00:01.1 at Gen4 x16 end to end (~31.5 GB/s); ROCm1 # the output layers) holding the extra one. See the WHY THE SPLIT IS 32:33
# (0b:00.0) rides the B650 chipset through bridges that each negotiate # block above for the trade against the predecessor's 35:30 x16-lean.
# Gen4 x1 (~1.97 GB/s). Live rocm-smi showed the x1 card at 98% busy TENSOR_SPLIT=32,33
# against 40% on the x16 card: the MTP drafter, the output layers and the
# heavier compute buffers are structurally pinned to the card holding the
# LAST layers (the draft context shares the target's weights there), so
# the bottlenecked card was doing most of the work. Moving one layer to
# ROCm0 shifts ~0.43 GiB of DeltaNet weights (layer 34 is a DeltaNet --
# the 4k..4k+2 position of block 8) off the x1 card and gives the healthy
# card the bigger share. The layer-split boundary itself is not the cost
# (~5.2 MB of activation per 512-token ubatch rides the wire), but the
# card behind the x1 link should not be the saturated one while its
# x16-sibling idles.
#
# WHY NOT FURTHER. One -ts step is one LAYER: ~0.43 GiB (DeltaNet) or
# ~1.43 GiB (attention). The next step, 36:29, would cross the
# full-attention layer at 35: the seam fit puts card 0 at 31.41 GiB
# (free 0.45 GiB, under the 1.0 GiB floor) with a 2.22 GiB gap (over the
# 1.483 GiB slack) -- both gates fail. 35:30 (fit: 30.59 / 30.01, gap
# 0.58) is the farthest safe shift toward the healthy card at this pool.
# Do not move it without re-measuring.
# MEASURED 2026-09-18 (isolated :7790, 2k-token decode, AVG rocm-smi util over
# the request -- peak util lies, a layer split touches both cards every token):
#
# split draft_dev decode AVG util0(x16) AVG util1(x1)
# 35,30 ROCm1 31.52 41 % 98 % <- deployed
# 36,29 ROCm1 31.49 43 % 98 %
# 35,30 ROCm0 31.52 42 % 98 %
# 36,29 ROCm0 31.48 43 % 98 %
#
# The x1 card stays the 98 % bottleneck in EVERY configuration. The lever that
# was tried is moving the MTP drafter to the x16 card (SPEC_DRAFT_DEVICE=ROCm0):
# it does NOT flip the balance, because the drafter is a small nextn head and the
# per-step cost is dominated by the VERIFY phase -- both cards run the full target
# -- plus the LOGITS, which are pinned to the card holding the last layers. A
# target-layer move (36:29) buys 2 points on the x16 card and erodes its VRAM
# margin (fit: free 0.45 GiB) for nothing. The x1 card is COMPUTE-bound, not
# PCIe-bound (~5.2 MB of activation per 512-token ubatch crosses the link);
# saturating it costs no tokens beyond what any R9700 would. 35:30 is the
# balance; do not chase the x16 card past it.
TENSOR_SPLIT=35,30
NGL=999 NGL=999
# Auto-fit has no business resizing a context we set explicitly. # Auto-fit has no business resizing a context we set explicitly.
FIT=off FIT=off
FLASH_ATTN=on FLASH_ATTN=on
# 393216 positions over TWO slots, so each conversation gets 196608 -- # 327680 positions over TWO slots, so each conversation gets 163840 --
# --parallel DIVIDES --ctx-size, so more slots would shrink every # --parallel DIVIDES --ctx-size, so more slots would shrink every
# conversation rather than grow the cache. The checkpoint's trained window is # conversation rather than grow the cache. 163840 is inside the checkpoint's
# 262144 and the GGUF carries NO rope scaling (freq_base 1e7, no YaRN keys), # trained 262144 window, so the whole pool is trained territory (no rope
# so positions beyond 262144 extrapolate on the base rope: it runs, and 1.5x # extrapolation, unlike the predecessor's 393216).
# is a modest ask of a rope, but the top 131k of the pool is untrained CTX_SIZE=327680
# territory.
CTX_SIZE=393216
PARALLEL=2 PARALLEL=2
# MEASURED 2026-09-16 with scripts/bench-batch-ubatch.sh on this box: llama-bench, # MEASURED 2026-09-16 with scripts/bench-batch-ubatch.sh on this box: llama-bench,
@@ -258,20 +212,13 @@ CTX_CHECKPOINTS=0
# (server_prompt_cache, tools/server/server-task.cpp): update() evicts # (server_prompt_cache, tools/server/server-task.cpp): update() evicts
# oldest-first past the cap, and alloc() REFUSES outright any single state # oldest-first past the cap, and alloc() REFUSES outright any single state
# bigger than the whole cap. A state is the serialized KV of a whole sequence # bigger than the whole cap. A state is the serialized KV of a whole sequence
# plus its DeltaNet state plus the MTP draft context's own KV, so at this shape # plus its DeltaNet state plus the draft context's own KV, so at this shape it
# it costs 68 KiB per token -- # costs ~68 KiB per token --
# 16 full-attention layers x 4 kv heads x 256 dim x 2 B x 2 (K,V) = 64 KiB # 16 full-attention layers x 4 kv heads x 256 dim x 2 B x 2 (K,V) = 64 KiB
# plus 4096 B for the draft context (one nextn layer) # plus the draft context's window-bounded KV (a ~40 MiB constant for DFlash2)
# -- which makes the cap a TOKEN budget in disguise: 8192 MiB held ~120k # -- which makes the cap a TOKEN budget in disguise: 16384 MiB holds ~240k
# tokens, 16384 MiB holds ~240k. # tokens, i.e. a FULL 163840-token slot (~10.6 GiB) with ~5 GiB of the budget
# # spare for a second entry.
# DOUBLED 8192 -> 16384 on 2026-09-18 because the old cap was BINDING, not
# theoretical: alloc() refused 137 states in the two days from 2026-09-16 with
# - prompt state size 11388.990 MiB exceeds cache size limit 8192.000 MiB, skipping
# i.e. 11.0-11.4 GiB states -- conversations of ~170k tokens at the law above --
# so every deep conversation was silently uncacheable and re-prefilled from
# scratch on its next turn. At 16384 a FULL 196608-token slot (~12.8 GiB) fits
# with ~3 GiB of the budget spare for a second entry.
# #
# The cost is host RAM, allocated on demand and never pinned (plain heap # The cost is host RAM, allocated on demand and never pinned (plain heap
# vectors), so no VRAM number above moves. dankserver has 62 GiB beside the # vectors), so no VRAM number above moves. dankserver has 62 GiB beside the
@@ -300,24 +247,21 @@ REASONING_BUDGET=-1
CHAT_TEMPLATE_FILE= CHAT_TEMPLATE_FILE=
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}' CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}'
# Speculative decoding: the model's OWN MTP head -- the blk.<n>.nextn.* # Speculative decoding: the DFlash2 sidecar (operator call 2026-09-19). The
# tensors inside the target file. No sidecar file, no second copy of the # target file also carries its own MTP head (nextn_predict_layers=1), so
# weights: the draft context runs on the main model (shares_model), so the # draft-mtp is one SPEC_TYPE change away -- the drafter file stays on disk.
# cost is one nextn layer's KV (4096 B/position) plus small buffers on the # n_max 7 is the measured DFlash2 winner (see the table below); the drafter
# card that holds the last layers. This is what makes the 393216 pool fit: # rides ROCm1, the card that holds the last layers.
# the DFlash2 sidecar's 1.92 GiB file and its buffers are off ROCm1. SPEC_TYPE=draft-dflash
# n_max: 3 is the measured MTP winner (see the table below -- n=7 accepts 33 % SPEC_DRAFT_N_MAX=7
# of 147, n=3 accepts 58 % of 72). The drafter rides ROCm1.
SPEC_TYPE=draft-mtp
SPEC_DRAFT_N_MAX=3
SPEC_DRAFT_NGL=999 SPEC_DRAFT_NGL=999
SPEC_DRAFT_DEVICE=ROCm1 SPEC_DRAFT_DEVICE=ROCm1
SPEC_DRAFT_TYPE_K=f16 SPEC_DRAFT_TYPE_K=f16
SPEC_DRAFT_TYPE_V=f16 SPEC_DRAFT_TYPE_V=f16
# MTP (the model's own head) vs the DFlash2 drafter -- MEASURED 2026-09-16 # DFlash2 vs the model's own MTP head -- MEASURED 2026-09-16 on the predecessor
# (temperature 0, 1k prompt / 512 out; the autoregressive baseline and both # payload (temperature 0, 1k prompt / 512 out; the autoregressive baseline and
# MTP depths were run in one session, same binary, same prompts): # both MTP depths were run in one session, same binary, same prompts):
# #
# none 17.7 tok/s card1 26.07 GB # none 17.7 tok/s card1 26.07 GB
# draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB # draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB
@@ -326,11 +270,10 @@ SPEC_DRAFT_TYPE_V=f16
# #
# All four produced IDENTICAL greedy text, so verification is lossless # All four produced IDENTICAL greedy text, so verification is lossless
# whichever proposer runs. DFlash2 is the faster proposer (2.46x over # whichever proposer runs. DFlash2 is the faster proposer (2.46x over
# autoregressive, 34 % over the best MTP); the operator switched to MTP on # autoregressive, 34 % over the best MTP). The MTP head adds ~3.3 GB to the
# 2026-09-17 for the VRAM, not the speed: the MTP head adds ~3.3 GB to the # card that holds it, against 1.06 GB for the Q4_K_M drafter's file -- at the
# card that holds it, against 2.06 GB for the drafter's file, and it is the # 327680 pool the drafter's window-bounded KV (~40 MiB) leaves headroom on
# only drafter that fits the 393216 pool. Do not switch this without # both cards, so the speed wins.
# re-measuring.
# #
# THE TWO CANNOT BE CHAINED. `--spec-type draft-mtp,draft-dflash` dies at load: # THE TWO CANNOT BE CHAINED. `--spec-type draft-mtp,draft-dflash` dies at load:
# context type MTP requested but model doesn't contain MTP layers # context type MTP requested but model doesn't contain MTP layers
@@ -344,7 +287,7 @@ SPEC_DRAFT_TYPE_V=f16
# ngram-mod: the draftless n-gram matcher (CPU hash table in host RAM, no VRAM, # ngram-mod: the draftless n-gram matcher (CPU hash table in host RAM, no VRAM,
# no model file, no draft context). It joins the chain unconditionally -- # no model file, no draft context). It joins the chain unconditionally --
# unlike MTP, which needs a draft context and therefore cannot share one with # unlike MTP, which needs a draft context and therefore cannot share one with
# DFlash2. NOT CURRENTLY ACTIVE: SPEC_TYPE is draft-mtp alone, and the serve # DFlash2. NOT CURRENTLY ACTIVE: SPEC_TYPE is draft-dflash alone, and the serve
# script only passes --spec-ngram-mod-* when the type contains "ngram". The # script only passes --spec-ngram-mod-* when the type contains "ngram". The
# window below is retained because the DFlash2 + ngram chain was measured free # window below is retained because the DFlash2 + ngram chain was measured free
# on prose and 2.6x on prompt-echo -- one SPEC_TYPE change brings it back. # on prose and 2.6x on prompt-echo -- one SPEC_TYPE change brings it back.
@@ -352,7 +295,7 @@ SPEC_DRAFT_TYPE_V=f16
# WINDOW MEASURED 2026-09-16 (chain at each window, decode tok/s): # WINDOW MEASURED 2026-09-16 (chain at each window, decode tok/s):
# #
# n_min/n_max prose 1k/512 prompt-echo 1.5k/400 # n_min/n_max prose 1k/512 prompt-echo 1.5k/400
# 48/64 43.9 200.6 <- deployed # 48/64 43.9 200.6
# 16/32 36.0 161.0 # 16/32 36.0 161.0
# 8/16 36.2 113.8 # 8/16 36.2 113.8
# #
@@ -369,9 +312,7 @@ SPEC_NGRAM_N_MATCH=24
SPEC_NGRAM_N_MIN=48 SPEC_NGRAM_N_MIN=48
SPEC_NGRAM_N_MAX=64 SPEC_NGRAM_N_MAX=64
# Vision enabled: llama-server autoloads mmproj-BF16.gguf from MODEL_DIR # Vision enabled: llama-server loads mmproj-Qwen3.8-27B-Uncensored-f16.gguf
# (888 MiB, lands on ROCm0 -- the fit above charges it there). frank-small # (931 MiB, lands on ROCm0 -- the fit charges it there) via the explicit
# (Dankclaw, RTX 3090) remains the fleet's dedicated image endpoint; this # --mmproj flag. This keeps frank-medium multimodal across the model swap.
# makes frank-medium multimodal too, which is why the seam flips from
# "text only" to "vision loads with the model".
NO_MMPROJ=0 NO_MMPROJ=0
tests/frank_medium_unit.bats
+55 -47
View File
@@ -20,8 +20,8 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
# Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic. # Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic.
CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it
MODEL_BYTES=31457990112 # UD-Q8_K_XL target MODEL_BYTES=29047084416 # orcarouter Q8_0 target
DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter DRAFT_BYTES=1143006816 # DFlash2 Q4_K_M drafter
@test "frank-medium's unit, serve script, env and its build/fetch scripts exist" { @test "frank-medium's unit, serve script, env and its build/fetch scripts exist" {
[ -f "$UNIT" ] [ -f "$UNIT" ]
@@ -48,17 +48,19 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
grep -q '^SPLIT_MODE=layer$' "$ENV_FILE" grep -q '^SPLIT_MODE=layer$' "$ENV_FILE"
grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE" grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE"
# The split is a layer PAIR: the shares add up to the model's 65 blocks (one # The split is a layer PAIR: the shares add up to the model's 65 blocks (one
# -ts unit per layer) and ROCm1 -- the card that also carries the drafter and # -ts unit per layer). The operator asked for the tensors BALANCED across
# the output layers -- takes the FEWER of the two. WHICH pair balances the # the two cards (2026-09-19), which for 65 layers is 32:33 -- the most even
# cards is the seam test's job, which recomputes it from this file; pinning a # split that exists, with ROCm1 (the last-layers card, which also carries
# literal here would only mean that re-measuring the balance breaks a test # the drafter and the output layers) holding the extra one. WHICH pair
# about parallelism. # balances the cards is the fit test's job, which recomputes it from this
# file; pinning a literal here would only mean that re-measuring the
# balance breaks a test about parallelism.
local split_hi split_lo local split_hi split_lo
split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")" split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")"
split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")" split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")"
[ -n "$split_hi" ] && [ -n "$split_lo" ] [ -n "$split_hi" ] && [ -n "$split_lo" ]
[ $(( split_hi + split_lo )) -eq 65 ] [ $(( split_hi + split_lo )) -eq 65 ]
[ "$split_lo" -lt "$split_hi" ] [ $(( split_lo - split_hi )) -le 1 ]
grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE" grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE"
grep -q -- '--device "$DEVICES"' "$SERVE" grep -q -- '--device "$DEVICES"' "$SERVE"
grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE" grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE"
@@ -72,24 +74,26 @@ DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE" grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE"
} }
@test "the drafter is the model's own MTP head, at the measured depth" { @test "the drafter is the DFlash2 sidecar at the measured depth" {
# MTP's head lives in the target file (blk.<n>.nextn.*): no sidecar, no # The DFlash2 sidecar is the deployed drafter (operator call 2026-09-19).
# second copy of the weights, and it is the only drafter that fits the # n_max 7 is the measured DFlash2 winner -- 43.5 tok/s at 41 % acceptance
# 393216 pool. n_max 3 is the measured MTP winner -- n=7 accepts 33 % of # against 32.4 for the in-file MTP head at n=3 (the env's table; identical
# 147, n=3 accepts 58 % of 72 (the env's table; identical greedy text at # greedy text at every depth, so this is pure speed). The target file
# every depth, so this is pure speed). # also carries its own MTP head (nextn_predict_layers=1), so draft-mtp is
grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE" # one SPEC_TYPE change away; the drafter file stays on disk for that.
grep -qE '^SPEC_TYPE=draft-dflash$' "$ENV_FILE"
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE" grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
local n_max local n_max
n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")" n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")"
[ -n "$n_max" ] && [ "$n_max" -eq 3 ] [ -n "$n_max" ] && [ "$n_max" -eq 7 ]
# The DFlash2 sidecar stays on disk and in the fetch script -- one # The Q4_K_M rung is the operator's requested drafter quant -- the
# SPEC_TYPE change brings the faster chain back -- and ngram-mod's window # drafter's job is throughput, not output quality, and the target
# is retained with it, so neither may be dropped from the env. # verifies every drafted token. ngram-mod's window is retained in the env
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE" # for the chain, so it may not be dropped.
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q4_K_M\.gguf$' "$ENV_FILE"
grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE" grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE"
# ...and neither is loaded: ngram-mod's flags only go out when the type # ...and neither is loaded: ngram-mod's flags only go out when the type
# contains ngram, so a pure MTP configuration is not handed them. # contains ngram, so a pure dflash configuration is not handed them.
python3 -c " python3 -c "
lines = open('$SERVE').read().splitlines() lines = open('$SERVE').read().splitlines()
guard = next(i for i, l in enumerate(lines) if '*ngram*' in l) guard = next(i for i, l in enumerate(lines) if '*ngram*' in l)
@@ -130,11 +134,14 @@ print('draft-model wiring ok')
grep -q 'frank-medium.env' "$SERVE" grep -q 'frank-medium.env' "$SERVE"
} }
@test "the payload is fetched resumably and verified by size" { @test "the payload is fetched resumably and verified by size and sha256" {
grep -q 'UD-Q8_K_XL\.gguf' "$FETCH" grep -q 'Qwen3\.8-27B-Uncensored-Q8_0\.gguf' "$FETCH"
grep -q 'Qwen3\.8-27B-DFlash2-Q8_0\.gguf' "$FETCH" grep -q 'Qwen3\.8-27B-DFlash2-Q4_K_M\.gguf' "$FETCH"
grep -q 'curl -fL -C -' "$FETCH" grep -q -- '-fL -C -' "$FETCH"
grep -q 'verifying sizes' "$FETCH" grep -q 'verifying sizes and sha256' "$FETCH"
# The target repo is gated: the script must fail loud without a token
# rather than write a 401 body to disk and report success.
grep -q 'no HF token is available' "$FETCH"
} }
@test "both cards stay balanced and keep the operator's 1.5 GiB floor" { @test "both cards stay balanced and keep the operator's 1.5 GiB floor" {
@@ -158,9 +165,9 @@ print('draft-model wiring ok')
# measured 6.44 / 6.90 GiB free. The fit reproduces card 1 to 0.00 GiB # measured 6.44 / 6.90 GiB free. The fit reproduces card 1 to 0.00 GiB
# and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the # and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the
# deployed point is re-measured on the host after every split/ctx/batch # deployed point is re-measured on the host after every split/ctx/batch
# move (the current one is 35:30 / 393216 / MTP, moved from 34:31 on # move (the current one is 32:33 / 327680 / DFlash2 Q4_K_M, the balanced
# 2026-09-17 to lean work off ROCm1's Gen4-x1 chipset path -- see the # split the operator asked for on 2026-09-19 -- see the TENSOR_SPLIT
# TENSOR_SPLIT header in frank-medium.env). # header in frank-medium.env).
# #
# card 0 = its share of the weights and of the KV (qwen35 spends KV on # card 0 = its share of the weights and of the KV (qwen35 spends KV on
# its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim # its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim
@@ -197,22 +204,22 @@ print('draft-model wiring ok')
# allocation. # allocation.
local kv_pos=65536 draft_kv_pos=20480 mtp_kv_pos=4096 floor slack local kv_pos=65536 draft_kv_pos=20480 mtp_kv_pos=4096 floor slack
local card_hi card_lo diff local card_hi card_lo diff
local mmproj=931145888 # mmproj-BF16.gguf, autoloaded onto the primary device local mmproj=931145984 # mmproj-Qwen3.8-27B-Uncensored-f16.gguf, on the primary device
# 1.5 GiB was the floor until the 2x196k shape: the 34:31 / 393216 / # 1.5 GiB was the floor until the 2x196k shape: the 34:31 / 393216 /
# MTP point measured 28.91 / 30.78 GiB, leaving 2.95 / 1.08 GiB, so the # MTP point measured 28.91 / 30.78 GiB, leaving 2.95 / 1.08 GiB, so the
# floor is the measured 1.08 GiB rounded down. The margin for one big # floor is the measured 1.08 GiB rounded down. The 32:33 / 327680 /
# allocation is thinner than it used to be. The layer-toward-ROCm0 # DFlash2 point is roomier than that (fit: 6.97 / 4.89 GiB free), so the
# lever has since been spent (35:30, 2026-09-17): if a large allocation # floor stays at the measured 1.0 GiB -- if a large allocation fails at
# fails at load now, the next lever is a ctx cut, not another layer. # load, the next lever is a ctx cut, not a floor change.
floor=$(( 1024 * 1024 * 1024 )) # 1.0 GiB per card floor=$(( 1024 * 1024 * 1024 )) # 1.0 GiB per card
# The deployed point (35:30 at ctx 393216 with MTP) has a fit gap of # The deployed point (32:33 at ctx 327680 with DFlash2) has a fit gap of
# 0.58 GiB -- closer to balanced than its 34:31 predecessor (1.06 GiB). # 2082 MiB -- the balanced split the operator asked for, not the
# The slack sits at 1.483 GiB: it accepts the deployed split and still # tightest reachable one. The slack sits at 2150 MiB: it accepts the
# rejects the next step toward ROCm0, 36:29, which fails BOTH gates # deployed split and rejects a two-layer re-skew (34:31 / 31:34 both
# (gap 2.22 GiB, card 0 free 0.45 GiB under the floor). (The 36:29 / # move the fit gap well past it). The x16-lean 35:30 that served the
# 327680 / DFlash2 configuration's slack was 3225 MiB -- its 3.15 GiB # predecessor payload (gap 0.58 GiB) no longer applies: the operator
# gap was by construction, and is no longer the deployed shape.) # chose symmetry over the x1-card bottleneck.
slack=$(( 1483 * 1024 * 1024 )) slack=$(( 2150 * 1024 * 1024 ))
card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \ card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \
+ kv_pos * ctx * split_hi / (split_hi + split_lo) \ + kv_pos * ctx * split_hi / (split_hi + split_lo) \
+ 381898377 \ + 381898377 \
@@ -243,8 +250,9 @@ print('draft-model wiring ok')
diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff )) diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff ))
[ "$diff" -le "$slack" ] [ "$diff" -le "$slack" ]
# ...and the window may not wander far past what the checkpoint was # ...and the window may not wander far past what the checkpoint was
# trained for (262144; the GGUF carries no rope scaling, so the top 131k # trained for (262144; the GGUF carries no rope scaling, so positions
# of the 393216 pool extrapolates on the base rope). # past that extrapolate on the base rope). The deployed 327680 pool is
# two 163840 slots -- entirely trained territory.
[ "$ctx" -le 393216 ] [ "$ctx" -le 393216 ]
} }
@@ -280,16 +288,16 @@ print('draft-model wiring ok')
grep -q -- '--cache-prompt' "$SERVE" grep -q -- '--cache-prompt' "$SERVE"
} }
@test "vision is enabled: the BF16 projector loads with the model" { @test "vision is enabled: the f16 projector loads with the model" {
grep -q '^NO_MMPROJ=0$' "$ENV_FILE" grep -q '^NO_MMPROJ=0$' "$ENV_FILE"
grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE" grep -q '^MM_PROJ_FILE=mmproj-Qwen3\.8-27B-Uncensored-f16\.gguf$' "$ENV_FILE"
# Explicit flag, not autoload: this build starts text-only without it, # Explicit flag, not autoload: this build starts text-only without it,
# and refuses to start without the projector file, so a missing file is # and refuses to start without the projector file, so a missing file is
# a deploy error, not a silent text-only server. # a deploy error, not a silent text-only server.
grep -q -- '--mmproj "$MM_PROJ"' "$SERVE" grep -q -- '--mmproj "$MM_PROJ"' "$SERVE"
grep -q 'vision is enabled but the projector is missing' "$SERVE" grep -q 'vision is enabled but the projector is missing' "$SERVE"
# And the download script fetches and byte-verifies it. # And the download script fetches and byte-verifies it.
grep -q '^MM_PROJ_SIZE=931145888$' "$FETCH" grep -q '^MM_PROJ_SIZE=931145984$' "$FETCH"
grep -q -- 'check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"' "$FETCH" grep -q -- 'check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"' "$FETCH"
} }