From d95f55b8223938e4ec34b4c39fe30d2353f50702 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 17 Sep 2026 01:37:55 -0700 Subject: [PATCH] frank-medium: 36:29 split, 327680 context, vision enabled The 320k pool (163840 per conversation over two slots) costs ~4.3 GiB of KV, and the BF16 projector (888 MiB) lands on ROCm0. Fit at the deployed point, computed, not estimated: ROCm0 29.19 / ROCm1 26.04 GiB of 31.86 -- 2.67 / 5.82 GiB free, both above the 1.5 GiB floor. 36:29 is one layer over the 35:30 balance on purpose: at this context the balance point is 34:31 (0.11 GiB gap), and the operator moved a layer to ROCm0 to buy headroom on the drafter card. The gap is 3.15 GiB by construction, so the seam's slack is that gap rounded up (3225 MiB) with the trade written down, and the ctx cap moves 262144 -> 327680 with the rope caveat (the GGUF carries no scaling; positions past the trained 262144 extrapolate on the base rope). Vision: NO_MMPROJ=0, the serve script now refuses to start without the projector when vision is on (a missing file is a deploy error, not a silent text-only server), the download script fetches and byte-verifies it, and the deploy handler's preflight checks for it. The projector is already on the host, byte- and sha256-verified against the Hub. --- scripts/deploy-remote.sh | 6 ++++ scripts/download-frank-medium-gguf.sh | 10 ++++-- scripts/frank-medium.env | 45 ++++++++++++++++++--------- scripts/serve-frank-medium.sh | 11 ++++++- tests/frank_medium_unit.bats | 38 ++++++++++++++++------ 5 files changed, 83 insertions(+), 27 deletions(-) diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 2c5418f..86ff4a5 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1114,6 +1114,7 @@ deploy_llm_small() { LLM_MEDIUM_BIN="${LLM_MEDIUM_BIN:-/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}" LLM_MEDIUM_MODEL="${LLM_MEDIUM_MODEL:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}" LLM_MEDIUM_DRAFT="${LLM_MEDIUM_DRAFT:-/home/dank/models/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q8_0.gguf}" +LLM_MEDIUM_MMPROJ="${LLM_MEDIUM_MMPROJ:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/mmproj-BF16.gguf}" # The three vLLM user units are retired HERE, not left to rot: the proxy # published 7770 on the host network, so the llama-server below cannot bind @@ -1153,6 +1154,11 @@ deploy_llm_medium() { echo "frank-medium: fetch it: bash scripts/download-frank-medium-gguf.sh" >&2 missing=1 fi + if [[ ! -f "$LLM_MEDIUM_MMPROJ" ]]; then + echo "ERROR: vision projector absent at $LLM_MEDIUM_MMPROJ" >&2 + echo "frank-medium: fetch it: bash scripts/download-frank-medium-gguf.sh" >&2 + missing=1 + fi [[ "$missing" -eq 0 ]] || return 1 retire_vllm_medium_quadlets diff --git a/scripts/download-frank-medium-gguf.sh b/scripts/download-frank-medium-gguf.sh index 90934a3..d599922 100755 --- a/scripts/download-frank-medium-gguf.sh +++ b/scripts/download-frank-medium-gguf.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# Download the frank-medium payload for Dankserver (2x Radeon AI PRO R9700): -# the target GGUF and the DFlash2 draft model, into /home/dank/models. +# the target GGUF, the BF16 vision projector and the DFlash2 draft model. # # bash scripts/download-frank-medium-gguf.sh # bash scripts/download-frank-medium-gguf.sh --verify-only @@ -27,6 +26,10 @@ MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF" MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF" MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf" MODEL_SIZE=31457990112 +# The BF16 vision projector from the same repo; llama-server autoloads any +# mmproj*.gguf beside the target, so it must land in MODEL_DIR. +MM_PROJ_FILE="mmproj-BF16.gguf" +MM_PROJ_SIZE=931145888 DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF" DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF" @@ -55,6 +58,8 @@ dl() { # dl if [[ "$mode" == "download" ]]; then echo "target: $MODEL_REPO" dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE" + echo "projector: $MODEL_REPO" + dl "$MODEL_REPO" "$MODEL_DIR" "$MM_PROJ_FILE" echo "draft: $DRAFT_REPO" dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE" fi @@ -72,6 +77,7 @@ check() { # check fi } check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE" +check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE" check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE" [[ -z "$token" ]] && echo "note: no token used; the byte counts above are the integrity check" [[ "$ok" -eq 1 ]] || exit 1 diff --git a/scripts/frank-medium.env b/scripts/frank-medium.env index 3e1f4a5..bcc28d5 100644 --- a/scripts/frank-medium.env +++ b/scripts/frank-medium.env @@ -35,7 +35,7 @@ # 29.29 GiB, arch qwen35, Unsloth Dynamic 3.0 (per-tensor types from Unsloth's # own quant of the same size, on Unsloth's importance matrix). Abliteration is # orcarouter's single refusal direction projected out of 131 tensors; the vision -# tower is untouched but unused here (text only, see NO_MMPROJ). The file also +# tower loads from mmproj-BF16.gguf (see NO_MMPROJ). The file also # carries the model's MTP head (blk.64.nextn.*), so a file-level switch to # --spec-type draft-mtp is available without a second copy in VRAM. # @@ -121,6 +121,11 @@ LLAMA_BIN=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama # Payload, fetched with scripts/download-frank-medium-gguf.sh. MODEL_DIR=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF MODEL_FILE=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf +# The BF16 vision projector, same repo as the target. llama-server autoloads +# any mmproj*.gguf beside the model; the serve script refuses to start without +# it when vision is enabled, so a missing file is a deploy error, not a +# text-only server that pretends to be multimodal. +MM_PROJ_FILE=mmproj-BF16.gguf DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q8_0.gguf @@ -136,23 +141,30 @@ HIP_VISIBLE_DEVICES=0,1 DEVICES=ROCm0,ROCm1 SPLIT_MODE=layer -# 35:30 -- the measured balance at the deployed batch, and the split that leaves -# the most room: ROCm1 carries the drafter, its KV and the output layers, so it -# takes fewer target layers. See SIZING above for the measurement table; one -# -ts step is one LAYER, worth ~0.43 GiB (DeltaNet) or ~1.43 GiB (attention), so -# do not move it without re-measuring. -TENSOR_SPLIT=35,30 +# 36:29 -- one layer over the 35:30 balance, moved to buy headroom on ROCm1, +# the card that carries the drafter, its KV, the output layers and the heavier +# compute buffers, now that the context grew to 327680 (see below). Fit at +# the deployed point (36:29, ctx 327680, batch 6144 x ubatch 512, two slots, +# DFlash2 + ngram-mod, mmproj on ROCm0): ROCm0 29.19 / ROCm1 26.04 GiB of +# 31.86, i.e. 2.67 / 5.82 GiB free and a 3.15 GiB gap BY CONSTRUCTION -- the +# balance point at this context is 34:31 (0.11 GiB gap) and the operator chose +# headroom on the drafter card over balance. See SIZING above for the +# measurement table; one -ts step is one LAYER, worth ~0.43 GiB (DeltaNet) or +# ~1.43 GiB (attention), so do not move it without re-measuring. +TENSOR_SPLIT=36,29 NGL=999 # Auto-fit has no business resizing a context we set explicitly. FIT=off FLASH_ATTN=on -# The model's full window (n_ctx_train 262144) divided over TWO slots, so each -# conversation gets 131072 -- verified at load as `n_slots = 2, n_ctx_slot = -# 131072`. --parallel DIVIDES --ctx-size, so more slots would shrink every -# conversation rather than grow the cache. -CTX_SIZE=262144 +# 327680 positions over TWO slots, so each conversation gets 163840 -- +# --parallel DIVIDES --ctx-size, so more slots would shrink every +# conversation rather than grow the cache. The checkpoint's trained window is +# 262144 and the GGUF carries NO rope scaling (freq_base 1e7, no YaRN keys), +# so positions beyond 262144 extrapolate on the base rope: it runs, and 1.25x +# is a modest ask of it, but the extra 65k is untrained territory. +CTX_SIZE=327680 PARALLEL=2 # MEASURED 2026-09-16 with scripts/bench-batch-ubatch.sh on this box: llama-bench, @@ -292,6 +304,9 @@ SPEC_NGRAM_N_MATCH=24 SPEC_NGRAM_N_MIN=48 SPEC_NGRAM_N_MAX=64 -# Text only: the server must not autoload a vision tower. Vision stays on -# frank-small (Dankclaw, RTX 3090), which is the fleet's image endpoint. -NO_MMPROJ=1 +# Vision enabled: llama-server autoloads mmproj-BF16.gguf from MODEL_DIR +# (888 MiB, lands on ROCm0 -- the fit above charges it there). frank-small +# (Dankclaw, RTX 3090) remains the fleet's dedicated image endpoint; this +# makes frank-medium multimodal too, which is why the seam flips from +# "text only" to "vision loads with the model". +NO_MMPROJ=0 diff --git a/scripts/serve-frank-medium.sh b/scripts/serve-frank-medium.sh index 3afcaff..0b1dbe3 100755 --- a/scripts/serve-frank-medium.sh +++ b/scripts/serve-frank-medium.sh @@ -89,7 +89,8 @@ fi : "${SPEC_NGRAM_N_MIN:=48}" : "${SPEC_NGRAM_N_MAX:=64}" -NO_MMPROJ="${NO_MMPROJ:-1}" +NO_MMPROJ="${NO_MMPROJ:-0}" +: "${MM_PROJ_FILE:=mmproj-BF16.gguf}" export HIP_VISIBLE_DEVICES @@ -106,6 +107,14 @@ if [[ ! -r "$MODEL" ]]; then echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2 exit 1 fi +if [[ "$NO_MMPROJ" != "1" ]]; then + MM_PROJ="$MODEL_DIR/$MM_PROJ_FILE" + if [[ ! -r "$MM_PROJ" ]]; then + echo "vision is enabled but the projector is missing: $MM_PROJ" >&2 + echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2 + exit 1 + fi +fi if [[ -n "$CHAT_TEMPLATE_FILE" && ! -r "$CHAT_TEMPLATE_FILE" ]]; then echo "missing or unreadable chat template: $CHAT_TEMPLATE_FILE" >&2 exit 1 diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats index b2c9c58..a9e042a 100755 --- a/tests/frank_medium_unit.bats +++ b/tests/frank_medium_unit.bats @@ -155,9 +155,12 @@ print('draft-model wiring ok') [ "$parallel" -ge 1 ] [ "$batch" -ge "$ubatch" ] [ "$cache_k" = "f16" ] && [ "$cache_v" = "f16" ] - # Per card, calibrated against rocm-smi at the deployed point (35:30, ctx - # 262144, batch 6144 x ubatch 512, two slots, DFlash2), where the two cards - # measured 6.44 / 6.90 GiB free. The fit reproduces that to 0.01 GiB. + # Per card, calibrated against rocm-smi at the 35:30 / ctx 262144 / batch + # 6144 x ubatch 512 / two slots / DFlash2 point, where the two cards + # measured 6.44 / 6.90 GiB free. The fit reproduces card 1 to 0.00 GiB + # and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the + # deployed point is re-measured on the host after every split/ctx/batch + # move (the 2026-09-17 move to 36:29 / 327680 / mmproj is the current one). # # card 0 = its share of the weights and of the KV (qwen35 spends KV on # its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim @@ -192,11 +195,20 @@ print('draft-model wiring ok') # allocation. local kv_pos=65536 draft_kv_pos=20480 floor slack local card_hi card_lo diff + local mmproj=931145888 # mmproj-BF16.gguf, autoloaded onto the primary device floor=$(( 1536 * 1024 * 1024 )) # 1.5 GiB per card - slack=$(( 512 * 1024 * 1024 )) # one DeltaNet layer, rounded up + # One DeltaNet layer at the old shape was the finest the balance could be + # tuned, so the slack used to be one DeltaNet layer. The 2026-09-17 move + # to 36:29 at ctx 327680 traded balance for headroom on the drafter card + # (the balance point at this context is 34:31, gap 0.11 GiB; 36:29's gap + # is 3.15 GiB BY CONSTRUCTION, and the operator chose it). The slack is + # that gap, rounded up: tighter rejects the deployed split, looser stops + # being a balance check at all. + slack=$(( 3225 * 1024 * 1024 )) card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \ + kv_pos * ctx * split_hi / (split_hi + split_lo) \ + 381898377 \ + + mmproj \ + 1363 * 1024 * ubatch )) card_lo=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \ + DRAFT_BYTES \ @@ -208,13 +220,21 @@ print('draft-model wiring ok') [ "$card_lo" -le $(( CARD_BYTES - floor )) ] diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff )) [ "$diff" -le "$slack" ] - # ...and the window may not exceed what the checkpoint was trained for. - [ "$ctx" -le 262144 ] + # ...and the window may not wander far past what the checkpoint was + # trained for (262144; the GGUF carries no rope scaling, so the extra 65k + # extrapolates on the base rope). + [ "$ctx" -le 327680 ] } -@test "frank-medium is text only: the vision tower stays on frank-small" { - grep -q '^NO_MMPROJ=1$' "$ENV_FILE" - grep -q -- '--no-mmproj' "$SERVE" +@test "vision is enabled: the BF16 projector loads with the model" { + grep -q '^NO_MMPROJ=0$' "$ENV_FILE" + grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE" + # The serve script refuses to start without the projector when vision is + # on, so a missing file is a deploy error, not a silent text-only server. + grep -q 'vision is enabled but the projector is missing' "$SERVE" + # And the download script fetches and byte-verifies it. + grep -q '^MM_PROJ_SIZE=931145888$' "$FETCH" + grep -q -- 'check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"' "$FETCH" } @test "the fleet endpoint and the predecessor's sampler defaults survive" {