frank-medium: pay for DSpark's draft KV with a 38:27 split
DSpark declares no sliding window, so its draft context -- built with n_ctx = llama_n_ctx(ctx_tgt) -- caches the whole target window: 5 layers x 2 x 8 kv heads x 128 dim x 2 B = 20 KiB/position, i.e. 5.0 GiB at 262144, where DFlash2's sliding_window 2048 cost ~40 MiB. At a 1:1 split that put ROCm1 at 33.9 GB of 34.2 GB and the first draft graph capture died inside common_speculative_impl_draft_dflash::process with "ROCm error: out of memory" -- after the model had loaded and /health had answered, which is why the deploy looked like a crash-loop rather than a sizing failure. Handing three target layers to ROCm0 restores headroom on both cards at the full window. Measured at 262144: 28.52 GB / 30.71 GB, i.e. 5.30 GiB and 3.26 GiB free (was 2.7 GB free on the draft-carrying card with 1:1 + DFlash2). The unit seam now models the busier card as a skewed share plus the drafter's full-context KV, calibrated to the rocm-smi figure. AGENTS.md gains the durable lesson: read sliding_window out of the drafter's GGUF header before sizing a spec-decode deployment.
This commit is contained in:
@@ -47,7 +47,7 @@ DRAFT_BYTES=1359284737 # DSpark Q8_0 drafter
|
||||
@test "both R9700s serve ONE model, layer-split inside one process" {
|
||||
grep -q '^SPLIT_MODE=layer$' "$ENV_FILE"
|
||||
grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE"
|
||||
grep -q '^TENSOR_SPLIT=1,1$' "$ENV_FILE"
|
||||
grep -q '^TENSOR_SPLIT=38,27$' "$ENV_FILE"
|
||||
grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE"
|
||||
grep -q -- '--device "$DEVICES"' "$SERVE"
|
||||
grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE"
|
||||
@@ -130,25 +130,35 @@ print('draft-model wiring ok')
|
||||
}
|
||||
|
||||
@test "one slot's KV still fits a card beside the weights that card holds" {
|
||||
local ctx parallel cache_k cache_v
|
||||
local ctx parallel cache_k cache_v split_hi split_lo
|
||||
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
|
||||
cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
|
||||
split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")"
|
||||
[ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ]
|
||||
[ -n "$split_hi" ] && [ -n "$split_lo" ]
|
||||
[ "$parallel" -ge 1 ]
|
||||
[ "$cache_k" = "f16" ] && [ "$cache_v" = "f16" ]
|
||||
# qwen35 spends KV on its 16 full-attention layers only: 2 (K,V) x 4 heads
|
||||
# x 256 dim x 2 B = 64 KiB per token at f16. A layer split puts those
|
||||
# layers (and their KV) on both cards, so the worst card carries half the
|
||||
# weights, half the KV, the drafter and ~2 GiB of compute buffers.
|
||||
# x 256 dim x 2 B = 64 KiB per token at f16, carried by whichever card a
|
||||
# layer split puts those layers on. The busier card is the SECOND one: its
|
||||
# share of the weights and KV, plus the WHOLE drafter -- whose KV spans the
|
||||
# full context, because DSpark declares no sliding window (5 x 2 x 8 x 128
|
||||
# x 2 B = 20480 B per position) -- plus the loader's compute buffers.
|
||||
#
|
||||
# This model is deliberately calibrated against rocm-smi: at 262144 it
|
||||
# predicts 28.53e9 B against the 29.14e9 B measured on the busier card
|
||||
# (2026-09-16), i.e. ~2 % optimistic. Keep it that way -- a model that is
|
||||
# pessimistic beyond a few GiB would reject a context that fits.
|
||||
local per_card
|
||||
per_card=$(( MODEL_BYTES / 2 + DRAFT_BYTES + (ctx * 65536) / 2 + 2 * 1024 * 1024 * 1024 ))
|
||||
# This model is deliberately calibrated against rocm-smi: at 262144 / 38:27
|
||||
# it predicts 30.69e9 B against the 30.71e9 B measured on the busier card
|
||||
# (2026-09-16). Keep it that way -- a model that is pessimistic beyond a few
|
||||
# GiB would reject a context that fits.
|
||||
local per_card bufs
|
||||
bufs=$(( 3584 * 1024 * 1024 ))
|
||||
per_card=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \
|
||||
+ DRAFT_BYTES \
|
||||
+ (ctx * 65536) * split_lo / (split_hi + split_lo) \
|
||||
+ ctx * 20480 \
|
||||
+ bufs ))
|
||||
[ "$per_card" -le "$CARD_BYTES" ]
|
||||
# ...and the window may not exceed what the checkpoint was trained for.
|
||||
[ "$ctx" -le 262144 ]
|
||||
|
||||
Reference in New Issue
Block a user