diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 2c5418f..86ff4a5 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1114,6 +1114,7 @@ deploy_llm_small() { LLM_MEDIUM_BIN="${LLM_MEDIUM_BIN:-/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}" LLM_MEDIUM_MODEL="${LLM_MEDIUM_MODEL:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}" LLM_MEDIUM_DRAFT="${LLM_MEDIUM_DRAFT:-/home/dank/models/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q8_0.gguf}" +LLM_MEDIUM_MMPROJ="${LLM_MEDIUM_MMPROJ:-/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF/mmproj-BF16.gguf}" # The three vLLM user units are retired HERE, not left to rot: the proxy # published 7770 on the host network, so the llama-server below cannot bind @@ -1153,6 +1154,11 @@ deploy_llm_medium() { echo "frank-medium: fetch it: bash scripts/download-frank-medium-gguf.sh" >&2 missing=1 fi + if [[ ! -f "$LLM_MEDIUM_MMPROJ" ]]; then + echo "ERROR: vision projector absent at $LLM_MEDIUM_MMPROJ" >&2 + echo "frank-medium: fetch it: bash scripts/download-frank-medium-gguf.sh" >&2 + missing=1 + fi [[ "$missing" -eq 0 ]] || return 1 retire_vllm_medium_quadlets diff --git a/scripts/download-frank-medium-gguf.sh b/scripts/download-frank-medium-gguf.sh index 90934a3..d599922 100755 --- a/scripts/download-frank-medium-gguf.sh +++ b/scripts/download-frank-medium-gguf.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# Download the frank-medium payload for Dankserver (2x Radeon AI PRO R9700): -# the target GGUF and the DFlash2 draft model, into /home/dank/models. +# the target GGUF, the BF16 vision projector and the DFlash2 draft model. # # bash scripts/download-frank-medium-gguf.sh # bash scripts/download-frank-medium-gguf.sh --verify-only @@ -27,6 +26,10 @@ MODEL_REPO="ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF" MODEL_DIR="/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF" MODEL_FILE="Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf" MODEL_SIZE=31457990112 +# The BF16 vision projector from the same repo; llama-server autoloads any +# mmproj*.gguf beside the target, so it must land in MODEL_DIR. +MM_PROJ_FILE="mmproj-BF16.gguf" +MM_PROJ_SIZE=931145888 DRAFT_REPO="z-lab/Qwen3.8-27B-DFlash2-GGUF" DRAFT_DIR="/home/dank/models/Qwen3.8-27B-DFlash2-GGUF" @@ -55,6 +58,8 @@ dl() { # dl if [[ "$mode" == "download" ]]; then echo "target: $MODEL_REPO" dl "$MODEL_REPO" "$MODEL_DIR" "$MODEL_FILE" + echo "projector: $MODEL_REPO" + dl "$MODEL_REPO" "$MODEL_DIR" "$MM_PROJ_FILE" echo "draft: $DRAFT_REPO" dl "$DRAFT_REPO" "$DRAFT_DIR" "$DRAFT_FILE" fi @@ -72,6 +77,7 @@ check() { # check fi } check "$MODEL_DIR/$MODEL_FILE" "$MODEL_SIZE" +check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE" check "$DRAFT_DIR/$DRAFT_FILE" "$DRAFT_SIZE" [[ -z "$token" ]] && echo "note: no token used; the byte counts above are the integrity check" [[ "$ok" -eq 1 ]] || exit 1 diff --git a/scripts/frank-medium.env b/scripts/frank-medium.env index 3e1f4a5..bcc28d5 100644 --- a/scripts/frank-medium.env +++ b/scripts/frank-medium.env @@ -35,7 +35,7 @@ # 29.29 GiB, arch qwen35, Unsloth Dynamic 3.0 (per-tensor types from Unsloth's # own quant of the same size, on Unsloth's importance matrix). Abliteration is # orcarouter's single refusal direction projected out of 131 tensors; the vision -# tower is untouched but unused here (text only, see NO_MMPROJ). The file also +# tower loads from mmproj-BF16.gguf (see NO_MMPROJ). The file also # carries the model's MTP head (blk.64.nextn.*), so a file-level switch to # --spec-type draft-mtp is available without a second copy in VRAM. # @@ -121,6 +121,11 @@ LLAMA_BIN=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama # Payload, fetched with scripts/download-frank-medium-gguf.sh. MODEL_DIR=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF MODEL_FILE=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf +# The BF16 vision projector, same repo as the target. llama-server autoloads +# any mmproj*.gguf beside the model; the serve script refuses to start without +# it when vision is enabled, so a missing file is a deploy error, not a +# text-only server that pretends to be multimodal. +MM_PROJ_FILE=mmproj-BF16.gguf DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q8_0.gguf @@ -136,23 +141,30 @@ HIP_VISIBLE_DEVICES=0,1 DEVICES=ROCm0,ROCm1 SPLIT_MODE=layer -# 35:30 -- the measured balance at the deployed batch, and the split that leaves -# the most room: ROCm1 carries the drafter, its KV and the output layers, so it -# takes fewer target layers. See SIZING above for the measurement table; one -# -ts step is one LAYER, worth ~0.43 GiB (DeltaNet) or ~1.43 GiB (attention), so -# do not move it without re-measuring. -TENSOR_SPLIT=35,30 +# 36:29 -- one layer over the 35:30 balance, moved to buy headroom on ROCm1, +# the card that carries the drafter, its KV, the output layers and the heavier +# compute buffers, now that the context grew to 327680 (see below). Fit at +# the deployed point (36:29, ctx 327680, batch 6144 x ubatch 512, two slots, +# DFlash2 + ngram-mod, mmproj on ROCm0): ROCm0 29.19 / ROCm1 26.04 GiB of +# 31.86, i.e. 2.67 / 5.82 GiB free and a 3.15 GiB gap BY CONSTRUCTION -- the +# balance point at this context is 34:31 (0.11 GiB gap) and the operator chose +# headroom on the drafter card over balance. See SIZING above for the +# measurement table; one -ts step is one LAYER, worth ~0.43 GiB (DeltaNet) or +# ~1.43 GiB (attention), so do not move it without re-measuring. +TENSOR_SPLIT=36,29 NGL=999 # Auto-fit has no business resizing a context we set explicitly. FIT=off FLASH_ATTN=on -# The model's full window (n_ctx_train 262144) divided over TWO slots, so each -# conversation gets 131072 -- verified at load as `n_slots = 2, n_ctx_slot = -# 131072`. --parallel DIVIDES --ctx-size, so more slots would shrink every -# conversation rather than grow the cache. -CTX_SIZE=262144 +# 327680 positions over TWO slots, so each conversation gets 163840 -- +# --parallel DIVIDES --ctx-size, so more slots would shrink every +# conversation rather than grow the cache. The checkpoint's trained window is +# 262144 and the GGUF carries NO rope scaling (freq_base 1e7, no YaRN keys), +# so positions beyond 262144 extrapolate on the base rope: it runs, and 1.25x +# is a modest ask of it, but the extra 65k is untrained territory. +CTX_SIZE=327680 PARALLEL=2 # MEASURED 2026-09-16 with scripts/bench-batch-ubatch.sh on this box: llama-bench, @@ -292,6 +304,9 @@ SPEC_NGRAM_N_MATCH=24 SPEC_NGRAM_N_MIN=48 SPEC_NGRAM_N_MAX=64 -# Text only: the server must not autoload a vision tower. Vision stays on -# frank-small (Dankclaw, RTX 3090), which is the fleet's image endpoint. -NO_MMPROJ=1 +# Vision enabled: llama-server autoloads mmproj-BF16.gguf from MODEL_DIR +# (888 MiB, lands on ROCm0 -- the fit above charges it there). frank-small +# (Dankclaw, RTX 3090) remains the fleet's dedicated image endpoint; this +# makes frank-medium multimodal too, which is why the seam flips from +# "text only" to "vision loads with the model". +NO_MMPROJ=0 diff --git a/scripts/serve-frank-medium.sh b/scripts/serve-frank-medium.sh index 3afcaff..0b1dbe3 100755 --- a/scripts/serve-frank-medium.sh +++ b/scripts/serve-frank-medium.sh @@ -89,7 +89,8 @@ fi : "${SPEC_NGRAM_N_MIN:=48}" : "${SPEC_NGRAM_N_MAX:=64}" -NO_MMPROJ="${NO_MMPROJ:-1}" +NO_MMPROJ="${NO_MMPROJ:-0}" +: "${MM_PROJ_FILE:=mmproj-BF16.gguf}" export HIP_VISIBLE_DEVICES @@ -106,6 +107,14 @@ if [[ ! -r "$MODEL" ]]; then echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2 exit 1 fi +if [[ "$NO_MMPROJ" != "1" ]]; then + MM_PROJ="$MODEL_DIR/$MM_PROJ_FILE" + if [[ ! -r "$MM_PROJ" ]]; then + echo "vision is enabled but the projector is missing: $MM_PROJ" >&2 + echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2 + exit 1 + fi +fi if [[ -n "$CHAT_TEMPLATE_FILE" && ! -r "$CHAT_TEMPLATE_FILE" ]]; then echo "missing or unreadable chat template: $CHAT_TEMPLATE_FILE" >&2 exit 1 diff --git a/tests/frank_medium_unit.bats b/tests/frank_medium_unit.bats index b2c9c58..a9e042a 100755 --- a/tests/frank_medium_unit.bats +++ b/tests/frank_medium_unit.bats @@ -155,9 +155,12 @@ print('draft-model wiring ok') [ "$parallel" -ge 1 ] [ "$batch" -ge "$ubatch" ] [ "$cache_k" = "f16" ] && [ "$cache_v" = "f16" ] - # Per card, calibrated against rocm-smi at the deployed point (35:30, ctx - # 262144, batch 6144 x ubatch 512, two slots, DFlash2), where the two cards - # measured 6.44 / 6.90 GiB free. The fit reproduces that to 0.01 GiB. + # Per card, calibrated against rocm-smi at the 35:30 / ctx 262144 / batch + # 6144 x ubatch 512 / two slots / DFlash2 point, where the two cards + # measured 6.44 / 6.90 GiB free. The fit reproduces card 1 to 0.00 GiB + # and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the + # deployed point is re-measured on the host after every split/ctx/batch + # move (the 2026-09-17 move to 36:29 / 327680 / mmproj is the current one). # # card 0 = its share of the weights and of the KV (qwen35 spends KV on # its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim @@ -192,11 +195,20 @@ print('draft-model wiring ok') # allocation. local kv_pos=65536 draft_kv_pos=20480 floor slack local card_hi card_lo diff + local mmproj=931145888 # mmproj-BF16.gguf, autoloaded onto the primary device floor=$(( 1536 * 1024 * 1024 )) # 1.5 GiB per card - slack=$(( 512 * 1024 * 1024 )) # one DeltaNet layer, rounded up + # One DeltaNet layer at the old shape was the finest the balance could be + # tuned, so the slack used to be one DeltaNet layer. The 2026-09-17 move + # to 36:29 at ctx 327680 traded balance for headroom on the drafter card + # (the balance point at this context is 34:31, gap 0.11 GiB; 36:29's gap + # is 3.15 GiB BY CONSTRUCTION, and the operator chose it). The slack is + # that gap, rounded up: tighter rejects the deployed split, looser stops + # being a balance check at all. + slack=$(( 3225 * 1024 * 1024 )) card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \ + kv_pos * ctx * split_hi / (split_hi + split_lo) \ + 381898377 \ + + mmproj \ + 1363 * 1024 * ubatch )) card_lo=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \ + DRAFT_BYTES \ @@ -208,13 +220,21 @@ print('draft-model wiring ok') [ "$card_lo" -le $(( CARD_BYTES - floor )) ] diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff )) [ "$diff" -le "$slack" ] - # ...and the window may not exceed what the checkpoint was trained for. - [ "$ctx" -le 262144 ] + # ...and the window may not wander far past what the checkpoint was + # trained for (262144; the GGUF carries no rope scaling, so the extra 65k + # extrapolates on the base rope). + [ "$ctx" -le 327680 ] } -@test "frank-medium is text only: the vision tower stays on frank-small" { - grep -q '^NO_MMPROJ=1$' "$ENV_FILE" - grep -q -- '--no-mmproj' "$SERVE" +@test "vision is enabled: the BF16 projector loads with the model" { + grep -q '^NO_MMPROJ=0$' "$ENV_FILE" + grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE" + # The serve script refuses to start without the projector when vision is + # on, so a missing file is a deploy error, not a silent text-only server. + grep -q 'vision is enabled but the projector is missing' "$SERVE" + # And the download script fetches and byte-verifies it. + grep -q '^MM_PROJ_SIZE=931145888$' "$FETCH" + grep -q -- 'check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"' "$FETCH" } @test "the fleet endpoint and the predecessor's sampler defaults survive" {