frank-small: pin VLLM_ATTENTION_BACKEND=TURBOQUANT
Gemma4's config hook force-selects TRITON_ATTN for the mixed 256/512 head dims whenever no backend is selected, and TRITON_ATTN rejects the turboquant KV dtype (measured: 'kv_cache_dtype not supported', crash loop). The TURBOQUANT backend accepts any head size and the turboquant_* dtypes, and the Gemma4 MTP speculator carries the target's selected backend through to the draft layers.
This commit is contained in:
1 parent
5e581666f2
commit
966b70db1a
2 files changed
+13
-1
No files matched your search
@@ -98,7 +98,7 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
|
||||
! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE"
|
||||
}
|
||||
|
||||
@test "frank-small serves the full window with turboquant k8v4 KV on a dedicated card" {
|
||||
@test "frank-small serves the full window with turboquant KV on a dedicated card" {
|
||||
local maxlen kvdtype util seqs
|
||||
maxlen="$(sed -nE 's/^MAX_MODEL_LEN=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
kvdtype="$(sed -nE 's/^KV_CACHE_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
|
||||
@@ -113,6 +113,10 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
|
||||
# (head_dim 256) and full-attention (head_dim 512) layers, and fp8 is
|
||||
# hardware-dead on sm_86 anyway.
|
||||
[ "$kvdtype" = "turboquant_k8v4" ]
|
||||
# The TURBOQUANT backend must be pinned explicitly: Gemma4's config
|
||||
# hook force-selects TRITON_ATTN (mixed 256/512 head dims) whenever no
|
||||
# backend is selected, and TRITON_ATTN rejects the turboquant dtype.
|
||||
grep -q '^VLLM_ATTENTION_BACKEND=TURBOQUANT$' "$ENV_FILE"
|
||||
# The 3090 is dedicated to this service, so the engine may take most of
|
||||
# it; the KV pool, not the seq ceiling, caps full-context concurrency.
|
||||
[ "$util" = "0.95" ]
|
||||
|
||||
Reference in new issue
Block a user