diff --git a/scripts/frank-small.env b/scripts/frank-small.env index c84a785..b3d68b1 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -85,6 +85,14 @@ LISTEN_PORT=8000 MAX_MODEL_LEN=262144 GPU_MEM_UTIL=0.95 KV_CACHE_DTYPE=turboquant_k8v4 +# Gemma4's config hook force-selects TRITON_ATTN for the mixed 256/512 head +# dims, but only when no backend is selected — and TRITON_ATTN rejects the +# turboquant KV dtype (measured: ValueError "kv_cache_dtype not supported", +# crash loop). Pin the TURBOQUANT backend explicitly: it accepts any head +# size, the turboquant dtypes, and has the Ampere path, and the Gemma4 MTP +# speculator carries the target's selected backend through to the draft +# layers, so the drafter follows. +VLLM_ATTENTION_BACKEND=TURBOQUANT MAX_NUM_SEQS=16 MAX_NUM_BATCHED_TOKENS=16384 SPEC_NUM_TOKENS=4 diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 2fdef35..d64f7b0 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -98,7 +98,7 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE" } -@test "frank-small serves the full window with turboquant k8v4 KV on a dedicated card" { +@test "frank-small serves the full window with turboquant KV on a dedicated card" { local maxlen kvdtype util seqs maxlen="$(sed -nE 's/^MAX_MODEL_LEN=([0-9]+)$/\1/p' "$ENV_FILE")" kvdtype="$(sed -nE 's/^KV_CACHE_DTYPE=(.+)$/\1/p' "$ENV_FILE")" @@ -113,6 +113,10 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" # (head_dim 256) and full-attention (head_dim 512) layers, and fp8 is # hardware-dead on sm_86 anyway. [ "$kvdtype" = "turboquant_k8v4" ] + # The TURBOQUANT backend must be pinned explicitly: Gemma4's config + # hook force-selects TRITON_ATTN (mixed 256/512 head dims) whenever no + # backend is selected, and TRITON_ATTN rejects the turboquant dtype. + grep -q '^VLLM_ATTENTION_BACKEND=TURBOQUANT$' "$ENV_FILE" # The 3090 is dedicated to this service, so the engine may take most of # it; the KV pool, not the seq ceiling, caps full-context concurrency. [ "$util" = "0.95" ]