frank-medium: 36:29 split, 327680 context, vision enabled
The 320k pool (163840 per conversation over two slots) costs ~4.3 GiB of KV, and the BF16 projector (888 MiB) lands on ROCm0. Fit at the deployed point, computed, not estimated: ROCm0 29.19 / ROCm1 26.04 GiB of 31.86 -- 2.67 / 5.82 GiB free, both above the 1.5 GiB floor. 36:29 is one layer over the 35:30 balance on purpose: at this context the balance point is 34:31 (0.11 GiB gap), and the operator moved a layer to ROCm0 to buy headroom on the drafter card. The gap is 3.15 GiB by construction, so the seam's slack is that gap rounded up (3225 MiB) with the trade written down, and the ctx cap moves 262144 -> 327680 with the rope caveat (the GGUF carries no scaling; positions past the trained 262144 extrapolate on the base rope). Vision: NO_MMPROJ=0, the serve script now refuses to start without the projector when vision is on (a missing file is a deploy error, not a silent text-only server), the download script fetches and byte-verifies it, and the deploy handler's preflight checks for it. The projector is already on the host, byte- and sha256-verified against the Hub.
This commit is contained in:
Frank
@@ -1114,6 +1114,7 @@ deploy_llm_small() {
|
||||
LLM_MEDIUM_BIN="${LLM_MEDIUM_BIN:-/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}"
|
||||
LLM_MEDIUM_MODEL="${LLM_MEDIUM_MODEL:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}"
|
||||
LLM_MEDIUM_DRAFT="${LLM_MEDIUM_DRAFT:-/home/dank/models/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q8_0.gguf}"
|
||||
LLM_MEDIUM_MMPROJ="${LLM_MEDIUM_MMPROJ:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/mmproj-BF16.gguf}"
|
||||
|
||||
# The three vLLM user units are retired HERE, not left to rot: the proxy
|
||||
# published 7770 on the host network, so the llama-server below cannot bind
|
||||
@@ -1153,6 +1154,11 @@ deploy_llm_medium() {
|
||||
echo "frank-medium: fetch it: bash scripts/download-frank-medium-gguf.sh" >&2
|
||||
missing=1
|
||||
fi
|
||||
if [[ ! -f "$LLM_MEDIUM_MMPROJ" ]]; then
|
||||
echo "ERROR: vision projector absent at $LLM_MEDIUM_MMPROJ" >&2
|
||||
echo "frank-medium: fetch it: bash scripts/download-frank-medium-gguf.sh" >&2
|
||||
missing=1
|
||||
fi
|
||||
[[ "$missing" -eq 0 ]] || return 1
|
||||
|
||||
retire_vllm_medium_quadlets
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-medium payload for Dankserver (2x Radeon AI PRO R9700):
|
||||
# the target GGUF and the DFlash2 draft model, into /home/dank/models.
|
||||
# the target GGUF, the BF16 vision projector and the DFlash2 draft model.
|
||||
#
|
||||
# bash scripts/download-frank-medium-gguf.sh
|
||||
# bash scripts/download-frank-medium-gguf.sh --verify-only
|
||||
@@ -27,6 +26,10 @@ MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
||||
MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF"
|
||||
MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf"
|
||||
MODEL_SIZE=31457990112
|
||||
# The BF16 vision projector from the same repo; llama-server autoloads any
|
||||
# mmproj*.gguf beside the target, so it must land in MODEL_DIR.
|
||||
MM_PROJ_FILE="mmproj-BF16.gguf"
|
||||
MM_PROJ_SIZE=931145888
|
||||
|
||||
DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF"
|
||||
DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF"
|
||||
@@ -55,6 +58,8 @@ dl() { # dl <repo> <dir> <file>
|
||||
if [[ "$mode" == "download" ]]; then
|
||||
echo "target: $MODEL_REPO"
|
||||
dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE"
|
||||
echo "projector: $MODEL_REPO"
|
||||
dl "$MODEL_REPO" "$MODEL_DIR" "$MM_PROJ_FILE"
|
||||
echo "draft: $DRAFT_REPO"
|
||||
dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE"
|
||||
fi
|
||||
@@ -72,6 +77,7 @@ check() { # check <path> <expected>
|
||||
fi
|
||||
}
|
||||
check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE"
|
||||
check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"
|
||||
check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE"
|
||||
[[ -z "$token" ]] && echo "note: no token used; the byte counts above are the integrity check"
|
||||
[[ "$ok" -eq 1 ]] || exit 1
|
||||
|
||||
+30
-15
@@ -35,7 +35,7 @@
|
||||
# 29.29 GiB, arch qwen35, Unsloth Dynamic 3.0 (per-tensor types from Unsloth's
|
||||
# own quant of the same size, on Unsloth's importance matrix). Abliteration is
|
||||
# orcarouter's single refusal direction projected out of 131 tensors; the vision
|
||||
# tower is untouched but unused here (text only, see NO_MMPROJ). The file also
|
||||
# tower loads from mmproj-BF16.gguf (see NO_MMPROJ). The file also
|
||||
# carries the model's MTP head (blk.64.nextn.*), so a file-level switch to
|
||||
# --spec-type draft-mtp is available without a second copy in VRAM.
|
||||
#
|
||||
@@ -121,6 +121,11 @@ LLAMA_BIN=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama
|
||||
# Payload, fetched with scripts/download-frank-medium-gguf.sh.
|
||||
MODEL_DIR=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF
|
||||
MODEL_FILE=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf
|
||||
# The BF16 vision projector, same repo as the target. llama-server autoloads
|
||||
# any mmproj*.gguf beside the model; the serve script refuses to start without
|
||||
# it when vision is enabled, so a missing file is a deploy error, not a
|
||||
# text-only server that pretends to be multimodal.
|
||||
MM_PROJ_FILE=mmproj-BF16.gguf
|
||||
DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF
|
||||
DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q8_0.gguf
|
||||
|
||||
@@ -136,23 +141,30 @@ HIP_VISIBLE_DEVICES=0,1
|
||||
DEVICES=ROCm0,ROCm1
|
||||
SPLIT_MODE=layer
|
||||
|
||||
# 35:30 -- the measured balance at the deployed batch, and the split that leaves
|
||||
# the most room: ROCm1 carries the drafter, its KV and the output layers, so it
|
||||
# takes fewer target layers. See SIZING above for the measurement table; one
|
||||
# -ts step is one LAYER, worth ~0.43 GiB (DeltaNet) or ~1.43 GiB (attention), so
|
||||
# do not move it without re-measuring.
|
||||
TENSOR_SPLIT=35,30
|
||||
# 36:29 -- one layer over the 35:30 balance, moved to buy headroom on ROCm1,
|
||||
# the card that carries the drafter, its KV, the output layers and the heavier
|
||||
# compute buffers, now that the context grew to 327680 (see below). Fit at
|
||||
# the deployed point (36:29, ctx 327680, batch 6144 x ubatch 512, two slots,
|
||||
# DFlash2 + ngram-mod, mmproj on ROCm0): ROCm0 29.19 / ROCm1 26.04 GiB of
|
||||
# 31.86, i.e. 2.67 / 5.82 GiB free and a 3.15 GiB gap BY CONSTRUCTION -- the
|
||||
# balance point at this context is 34:31 (0.11 GiB gap) and the operator chose
|
||||
# headroom on the drafter card over balance. See SIZING above for the
|
||||
# measurement table; one -ts step is one LAYER, worth ~0.43 GiB (DeltaNet) or
|
||||
# ~1.43 GiB (attention), so do not move it without re-measuring.
|
||||
TENSOR_SPLIT=36,29
|
||||
|
||||
NGL=999
|
||||
# Auto-fit has no business resizing a context we set explicitly.
|
||||
FIT=off
|
||||
FLASH_ATTN=on
|
||||
|
||||
# The model's full window (n_ctx_train 262144) divided over TWO slots, so each
|
||||
# conversation gets 131072 -- verified at load as `n_slots = 2, n_ctx_slot =
|
||||
# 131072`. --parallel DIVIDES --ctx-size, so more slots would shrink every
|
||||
# conversation rather than grow the cache.
|
||||
CTX_SIZE=262144
|
||||
# 327680 positions over TWO slots, so each conversation gets 163840 --
|
||||
# --parallel DIVIDES --ctx-size, so more slots would shrink every
|
||||
# conversation rather than grow the cache. The checkpoint's trained window is
|
||||
# 262144 and the GGUF carries NO rope scaling (freq_base 1e7, no YaRN keys),
|
||||
# so positions beyond 262144 extrapolate on the base rope: it runs, and 1.25x
|
||||
# is a modest ask of it, but the extra 65k is untrained territory.
|
||||
CTX_SIZE=327680
|
||||
PARALLEL=2
|
||||
|
||||
# MEASURED 2026-09-16 with scripts/bench-batch-ubatch.sh on this box: llama-bench,
|
||||
@@ -292,6 +304,9 @@ SPEC_NGRAM_N_MATCH=24
|
||||
SPEC_NGRAM_N_MIN=48
|
||||
SPEC_NGRAM_N_MAX=64
|
||||
|
||||
# Text only: the server must not autoload a vision tower. Vision stays on
|
||||
# frank-small (Dankclaw, RTX 3090), which is the fleet's image endpoint.
|
||||
NO_MMPROJ=1
|
||||
# Vision enabled: llama-server autoloads mmproj-BF16.gguf from MODEL_DIR
|
||||
# (888 MiB, lands on ROCm0 -- the fit above charges it there). frank-small
|
||||
# (Dankclaw, RTX 3090) remains the fleet's dedicated image endpoint; this
|
||||
# makes frank-medium multimodal too, which is why the seam flips from
|
||||
# "text only" to "vision loads with the model".
|
||||
NO_MMPROJ=0
|
||||
|
||||
@@ -89,7 +89,8 @@ fi
|
||||
: "${SPEC_NGRAM_N_MIN:=48}"
|
||||
: "${SPEC_NGRAM_N_MAX:=64}"
|
||||
|
||||
NO_MMPROJ="${NO_MMPROJ:-1}"
|
||||
NO_MMPROJ="${NO_MMPROJ:-0}"
|
||||
: "${MM_PROJ_FILE:=mmproj-BF16.gguf}"
|
||||
|
||||
export HIP_VISIBLE_DEVICES
|
||||
|
||||
@@ -106,6 +107,14 @@ if [[ ! -r "$MODEL" ]]; then
|
||||
echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$NO_MMPROJ" != "1" ]]; then
|
||||
MM_PROJ="$MODEL_DIR/$MM_PROJ_FILE"
|
||||
if [[ ! -r "$MM_PROJ" ]]; then
|
||||
echo "vision is enabled but the projector is missing: $MM_PROJ" >&2
|
||||
echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [[ -n "$CHAT_TEMPLATE_FILE" && ! -r "$CHAT_TEMPLATE_FILE" ]]; then
|
||||
echo "missing or unreadable chat template: $CHAT_TEMPLATE_FILE" >&2
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user