diff --git a/scripts/frank-small.env b/scripts/frank-small.env index e107eb4..fa62f6e 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -84,17 +84,18 @@ LISTEN_PORT=8000 # 16k prefill chunk, 4-token MTP draft (the assistant carries 4 layers). MAX_MODEL_LEN=262144 GPU_MEM_UTIL=0.95 -KV_CACHE_DTYPE=turboquant_k8v4 -# ATTENTION_BACKEND: Gemma4's config hook force-selects TRITON_ATTN for the -# mixed 256/512 head dims whenever no backend is selected, and TRITON_ATTN -# rejects the turboquant KV dtype (measured: ValueError "kv_cache_dtype not -# supported", crash loop). Pin the TURBOQUANT backend explicitly via the -# --attention-backend flag (this vLLM build removed the VLLM_ATTENTION_ -# BACKEND env var — it warns "Unknown vLLM environment variable" and -# ignores it). The TURBOQUANT backend accepts any head size and the -# turboquant_* dtypes, and the Gemma4 MTP speculator carries the target's -# selected backend through to the draft layers, so the drafter follows. -ATTENTION_BACKEND=TURBOQUANT +KV_CACHE_DTYPE=auto +# KV quant history (measured 2026-09-19, all on this image): bf16 (auto) +# is the ONLY dtype that boots Gemma4 here. int8_per_token_head dies in +# unify_kv_cache_spec_page_size — its separate fp32 scale tensor makes the +# per-token page bytes 4160 (SWA, head_dim 256) vs 8256 (full, head_dim +# 512), not an integer ratio. turboquant_k8v4 dies in backend validation — +# the TURBOQUANT backend rejects "partial multimodal token full attention" +# (the mm-prefix path Gemma4's vision/audio tokens require), and the +# VLLM_ATTENTION_BACKEND env var does not exist in this build (the pin +# goes through --attention-backend). fp8 is hardware-dead on sm_86 +# (cvt.e4m3x2 is sm_89+). Revisit on a vLLM release that documents TurboQuant +# mm-prefix support. MAX_NUM_SEQS=16 MAX_NUM_BATCHED_TOKENS=16384 SPEC_NUM_TOKENS=4 diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index bbefb9a..c5fcb03 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -44,7 +44,6 @@ fi : "${MAX_NUM_SEQS:=16}" : "${MAX_NUM_BATCHED_TOKENS:=16384}" : "${SPEC_NUM_TOKENS:=4}" -: "${ATTENTION_BACKEND:=TURBOQUANT}" : "${ENFORCE_EAGER:=0}" # Fail loudly rather than half-loading a model into VRAM: a missing file here @@ -67,7 +66,6 @@ VLLM_FLAGS=( --kv-cache-dtype "$KV_CACHE_DTYPE" --max-num-seqs "$MAX_NUM_SEQS" --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" ---attention-backend "$ATTENTION_BACKEND" --speculative-config "$SPEC_CONFIG" --reasoning-parser gemma4 --enable-auto-tool-choice diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 89a32df..c2bf275 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -98,7 +98,7 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE" } -@test "frank-small serves the full window with turboquant KV on a dedicated card" { +@test "frank-small serves the full window with bf16 KV on a dedicated card" { local maxlen kvdtype util seqs maxlen="$(sed -nE 's/^MAX_MODEL_LEN=([0-9]+)$/\1/p' "$ENV_FILE")" kvdtype="$(sed -nE 's/^KV_CACHE_DTYPE=(.+)$/\1/p' "$ENV_FILE")" @@ -106,19 +106,17 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" seqs="$(sed -nE 's/^MAX_NUM_SEQS=([0-9]+)$/\1/p' "$ENV_FILE")" # The model's FULL trained window (262144), not a divided one. [ "$maxlen" = "262144" ] - # TurboQuant k8v4 KV (K 8-bit, V 4-bit, in-tensor block scales, - # dedicated TURBOQUANT backend with an Ampere path). The int8/fp8 - # per-token-head dtypes cannot boot this model: their separate fp32 - # scale tensor breaks the page-size ratio between Gemma4's SWA - # (head_dim 256) and full-attention (head_dim 512) layers, and fp8 is - # hardware-dead on sm_86 anyway. - [ "$kvdtype" = "turboquant_k8v4" ] - # The TURBOQUANT backend must be pinned via --attention-backend: - # Gemma4's config hook force-selects TRITON_ATTN (mixed 256/512 head - # dims) whenever no backend is selected, and TRITON_ATTN rejects the - # turboquant dtype. This vLLM build has no attention-backend env var. - grep -q '^ATTENTION_BACKEND=TURBOQUANT$' "$ENV_FILE" - grep -q -- '--attention-backend "$ATTENTION_BACKEND"' "$SERVE" + # bf16 KV (auto): the only dtype that boots Gemma4 in this vLLM build. + # int8_per_token_head dies in unify_kv_cache_spec_page_size (separate + # fp32 scale tensor breaks the 256/512 head-dim page-size ratio), + # turboquant_k8v4 dies in backend validation (no mm-prefix attention), + # and fp8 is hardware-dead on sm_86 — see frank-small.env. + [ "$kvdtype" = "auto" ] + # No pinned attention backend: Gemma4's config hook selects TRITON_ATTN + # for the mixed head dims, and pinning any other backend breaks the KV + # dtype or mm-prefix validation. + ! grep -q '^ATTENTION_BACKEND' "$ENV_FILE" + ! grep -q -- '--attention-backend' "$SERVE" # The 3090 is dedicated to this service, so the engine may take most of # it; the KV pool, not the seq ceiling, caps full-context concurrency. [ "$util" = "0.95" ]