frank-small: revert to bf16 KV — the only dtype that boots Gemma4
int8_per_token_head: page-size unification fails (scale tensor breaks the 256/512 head-dim ratio). turboquant_k8v4: backend validation fails (no mm-prefix attention, which Gemma4's vision/audio path requires). fp8: hardware-dead on sm_86. bf16 it is; history documented in frank-small.env, seam pins auto + no backend pin.
This commit is contained in:
1 parent
7e6ef1ac99
commit
bab8f7f839
3 files changed
+24
-27
No files matched your search
+12
-14
@@ -98,7 +98,7 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
|
||||
! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE"
|
||||
}
|
||||
|
||||
@test "frank-small serves the full window with turboquant KV on a dedicated card" {
|
||||
@test "frank-small serves the full window with bf16 KV on a dedicated card" {
|
||||
local maxlen kvdtype util seqs
|
||||
maxlen="$(sed -nE 's/^MAX_MODEL_LEN=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
kvdtype="$(sed -nE 's/^KV_CACHE_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
|
||||
@@ -106,19 +106,17 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
|
||||
seqs="$(sed -nE 's/^MAX_NUM_SEQS=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
# The model's FULL trained window (262144), not a divided one.
|
||||
[ "$maxlen" = "262144" ]
|
||||
# TurboQuant k8v4 KV (K 8-bit, V 4-bit, in-tensor block scales,
|
||||
# dedicated TURBOQUANT backend with an Ampere path). The int8/fp8
|
||||
# per-token-head dtypes cannot boot this model: their separate fp32
|
||||
# scale tensor breaks the page-size ratio between Gemma4's SWA
|
||||
# (head_dim 256) and full-attention (head_dim 512) layers, and fp8 is
|
||||
# hardware-dead on sm_86 anyway.
|
||||
[ "$kvdtype" = "turboquant_k8v4" ]
|
||||
# The TURBOQUANT backend must be pinned via --attention-backend:
|
||||
# Gemma4's config hook force-selects TRITON_ATTN (mixed 256/512 head
|
||||
# dims) whenever no backend is selected, and TRITON_ATTN rejects the
|
||||
# turboquant dtype. This vLLM build has no attention-backend env var.
|
||||
grep -q '^ATTENTION_BACKEND=TURBOQUANT$' "$ENV_FILE"
|
||||
grep -q -- '--attention-backend "$ATTENTION_BACKEND"' "$SERVE"
|
||||
# bf16 KV (auto): the only dtype that boots Gemma4 in this vLLM build.
|
||||
# int8_per_token_head dies in unify_kv_cache_spec_page_size (separate
|
||||
# fp32 scale tensor breaks the 256/512 head-dim page-size ratio),
|
||||
# turboquant_k8v4 dies in backend validation (no mm-prefix attention),
|
||||
# and fp8 is hardware-dead on sm_86 — see frank-small.env.
|
||||
[ "$kvdtype" = "auto" ]
|
||||
# No pinned attention backend: Gemma4's config hook selects TRITON_ATTN
|
||||
# for the mixed head dims, and pinning any other backend breaks the KV
|
||||
# dtype or mm-prefix validation.
|
||||
! grep -q '^ATTENTION_BACKEND' "$ENV_FILE"
|
||||
! grep -q -- '--attention-backend' "$SERVE"
|
||||
# The 3090 is dedicated to this service, so the engine may take most of
|
||||
# it; the KV pool, not the seq ceiling, caps full-context concurrency.
|
||||
[ "$util" = "0.95" ]
|
||||
|
||||
Reference in new issue
Block a user