frank-medium: 36:29 split, 327680 context, vision enabled
The 320k pool (163840 per conversation over two slots) costs ~4.3 GiB of KV, and the BF16 projector (888 MiB) lands on ROCm0. Fit at the deployed point, computed, not estimated: ROCm0 29.19 / ROCm1 26.04 GiB of 31.86 -- 2.67 / 5.82 GiB free, both above the 1.5 GiB floor. 36:29 is one layer over the 35:30 balance on purpose: at this context the balance point is 34:31 (0.11 GiB gap), and the operator moved a layer to ROCm0 to buy headroom on the drafter card. The gap is 3.15 GiB by construction, so the seam's slack is that gap rounded up (3225 MiB) with the trade written down, and the ctx cap moves 262144 -> 327680 with the rope caveat (the GGUF carries no scaling; positions past the trained 262144 extrapolate on the base rope). Vision: NO_MMPROJ=0, the serve script now refuses to start without the projector when vision is on (a missing file is a deploy error, not a silent text-only server), the download script fetches and byte-verifies it, and the deploy handler's preflight checks for it. The projector is already on the host, byte- and sha256-verified against the Hub.
This commit is contained in:
1 parent
24cdbb0e45
commit
d95f55b822
5 files changed
+83
-27
No files matched your search
@@ -155,9 +155,12 @@ print('draft-model wiring ok')
|
||||
[ "$parallel" -ge 1 ]
|
||||
[ "$batch" -ge "$ubatch" ]
|
||||
[ "$cache_k" = "f16" ] && [ "$cache_v" = "f16" ]
|
||||
# Per card, calibrated against rocm-smi at the deployed point (35:30, ctx
|
||||
# 262144, batch 6144 x ubatch 512, two slots, DFlash2), where the two cards
|
||||
# measured 6.44 / 6.90 GiB free. The fit reproduces that to 0.01 GiB.
|
||||
# Per card, calibrated against rocm-smi at the 35:30 / ctx 262144 / batch
|
||||
# 6144 x ubatch 512 / two slots / DFlash2 point, where the two cards
|
||||
# measured 6.44 / 6.90 GiB free. The fit reproduces card 1 to 0.00 GiB
|
||||
# and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the
|
||||
# deployed point is re-measured on the host after every split/ctx/batch
|
||||
# move (the 2026-09-17 move to 36:29 / 327680 / mmproj is the current one).
|
||||
#
|
||||
# card 0 = its share of the weights and of the KV (qwen35 spends KV on
|
||||
# its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim
|
||||
@@ -192,11 +195,20 @@ print('draft-model wiring ok')
|
||||
# allocation.
|
||||
local kv_pos=65536 draft_kv_pos=20480 floor slack
|
||||
local card_hi card_lo diff
|
||||
local mmproj=931145888 # mmproj-BF16.gguf, autoloaded onto the primary device
|
||||
floor=$(( 1536 * 1024 * 1024 )) # 1.5 GiB per card
|
||||
slack=$(( 512 * 1024 * 1024 )) # one DeltaNet layer, rounded up
|
||||
# One DeltaNet layer at the old shape was the finest the balance could be
|
||||
# tuned, so the slack used to be one DeltaNet layer. The 2026-09-17 move
|
||||
# to 36:29 at ctx 327680 traded balance for headroom on the drafter card
|
||||
# (the balance point at this context is 34:31, gap 0.11 GiB; 36:29's gap
|
||||
# is 3.15 GiB BY CONSTRUCTION, and the operator chose it). The slack is
|
||||
# that gap, rounded up: tighter rejects the deployed split, looser stops
|
||||
# being a balance check at all.
|
||||
slack=$(( 3225 * 1024 * 1024 ))
|
||||
card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \
|
||||
+ kv_pos * ctx * split_hi / (split_hi + split_lo) \
|
||||
+ 381898377 \
|
||||
+ mmproj \
|
||||
+ 1363 * 1024 * ubatch ))
|
||||
card_lo=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \
|
||||
+ DRAFT_BYTES \
|
||||
@@ -208,13 +220,21 @@ print('draft-model wiring ok')
|
||||
[ "$card_lo" -le $(( CARD_BYTES - floor )) ]
|
||||
diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff ))
|
||||
[ "$diff" -le "$slack" ]
|
||||
# ...and the window may not exceed what the checkpoint was trained for.
|
||||
[ "$ctx" -le 262144 ]
|
||||
# ...and the window may not wander far past what the checkpoint was
|
||||
# trained for (262144; the GGUF carries no rope scaling, so the extra 65k
|
||||
# extrapolates on the base rope).
|
||||
[ "$ctx" -le 327680 ]
|
||||
}
|
||||
|
||||
@test "frank-medium is text only: the vision tower stays on frank-small" {
|
||||
grep -q '^NO_MMPROJ=1$' "$ENV_FILE"
|
||||
grep -q -- '--no-mmproj' "$SERVE"
|
||||
@test "vision is enabled: the BF16 projector loads with the model" {
|
||||
grep -q '^NO_MMPROJ=0$' "$ENV_FILE"
|
||||
grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE"
|
||||
# The serve script refuses to start without the projector when vision is
|
||||
# on, so a missing file is a deploy error, not a silent text-only server.
|
||||
grep -q 'vision is enabled but the projector is missing' "$SERVE"
|
||||
# And the download script fetches and byte-verifies it.
|
||||
grep -q '^MM_PROJ_SIZE=931145888$' "$FETCH"
|
||||
grep -q -- 'check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"' "$FETCH"
|
||||
}
|
||||
|
||||
@test "the fleet endpoint and the predecessor's sampler defaults survive" {
|
||||
|
||||
Reference in new issue
Block a user