diff --git a/scripts/frank-small.env b/scripts/frank-small.env index b3d68b1..e107eb4 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -85,14 +85,16 @@ LISTEN_PORT=8000 MAX_MODEL_LEN=262144 GPU_MEM_UTIL=0.95 KV_CACHE_DTYPE=turboquant_k8v4 -# Gemma4's config hook force-selects TRITON_ATTN for the mixed 256/512 head -# dims, but only when no backend is selected — and TRITON_ATTN rejects the -# turboquant KV dtype (measured: ValueError "kv_cache_dtype not supported", -# crash loop). Pin the TURBOQUANT backend explicitly: it accepts any head -# size, the turboquant dtypes, and has the Ampere path, and the Gemma4 MTP -# speculator carries the target's selected backend through to the draft -# layers, so the drafter follows. -VLLM_ATTENTION_BACKEND=TURBOQUANT +# ATTENTION_BACKEND: Gemma4's config hook force-selects TRITON_ATTN for the +# mixed 256/512 head dims whenever no backend is selected, and TRITON_ATTN +# rejects the turboquant KV dtype (measured: ValueError "kv_cache_dtype not +# supported", crash loop). Pin the TURBOQUANT backend explicitly via the +# --attention-backend flag (this vLLM build removed the VLLM_ATTENTION_ +# BACKEND env var — it warns "Unknown vLLM environment variable" and +# ignores it). The TURBOQUANT backend accepts any head size and the +# turboquant_* dtypes, and the Gemma4 MTP speculator carries the target's +# selected backend through to the draft layers, so the drafter follows. +ATTENTION_BACKEND=TURBOQUANT MAX_NUM_SEQS=16 MAX_NUM_BATCHED_TOKENS=16384 SPEC_NUM_TOKENS=4 diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index c5fcb03..bbefb9a 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -44,6 +44,7 @@ fi : "${MAX_NUM_SEQS:=16}" : "${MAX_NUM_BATCHED_TOKENS:=16384}" : "${SPEC_NUM_TOKENS:=4}" +: "${ATTENTION_BACKEND:=TURBOQUANT}" : "${ENFORCE_EAGER:=0}" # Fail loudly rather than half-loading a model into VRAM: a missing file here @@ -66,6 +67,7 @@ VLLM_FLAGS=( --kv-cache-dtype "$KV_CACHE_DTYPE" --max-num-seqs "$MAX_NUM_SEQS" --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" +--attention-backend "$ATTENTION_BACKEND" --speculative-config "$SPEC_CONFIG" --reasoning-parser gemma4 --enable-auto-tool-choice diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index d64f7b0..89a32df 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -113,10 +113,12 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" # (head_dim 256) and full-attention (head_dim 512) layers, and fp8 is # hardware-dead on sm_86 anyway. [ "$kvdtype" = "turboquant_k8v4" ] - # The TURBOQUANT backend must be pinned explicitly: Gemma4's config - # hook force-selects TRITON_ATTN (mixed 256/512 head dims) whenever no - # backend is selected, and TRITON_ATTN rejects the turboquant dtype. - grep -q '^VLLM_ATTENTION_BACKEND=TURBOQUANT$' "$ENV_FILE" + # The TURBOQUANT backend must be pinned via --attention-backend: + # Gemma4's config hook force-selects TRITON_ATTN (mixed 256/512 head + # dims) whenever no backend is selected, and TRITON_ATTN rejects the + # turboquant dtype. This vLLM build has no attention-backend env var. + grep -q '^ATTENTION_BACKEND=TURBOQUANT$' "$ENV_FILE" + grep -q -- '--attention-backend "$ATTENTION_BACKEND"' "$SERVE" # The 3090 is dedicated to this service, so the engine may take most of # it; the KV pool, not the seq ceiling, caps full-context concurrency. [ "$util" = "0.95" ]