frank-small: --attention-backend flag (env var removed in this build)
The VLLM_ATTENTION_BACKEND env var does not exist in this vLLM dev build (warns 'Unknown vLLM environment variable' and is ignored), so the pin landed nowhere and Gemma4's hook force-selected TRITON_ATTN again. Pass --attention-backend TURBOQUANT as a CLI flag instead.
This commit is contained in:
1 parent
966b70db1a
commit
7e6ef1ac99
3 files changed
+18
-12
No files matched your search
@@ -113,10 +113,12 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
|
||||
# (head_dim 256) and full-attention (head_dim 512) layers, and fp8 is
|
||||
# hardware-dead on sm_86 anyway.
|
||||
[ "$kvdtype" = "turboquant_k8v4" ]
|
||||
# The TURBOQUANT backend must be pinned explicitly: Gemma4's config
|
||||
# hook force-selects TRITON_ATTN (mixed 256/512 head dims) whenever no
|
||||
# backend is selected, and TRITON_ATTN rejects the turboquant dtype.
|
||||
grep -q '^VLLM_ATTENTION_BACKEND=TURBOQUANT$' "$ENV_FILE"
|
||||
# The TURBOQUANT backend must be pinned via --attention-backend:
|
||||
# Gemma4's config hook force-selects TRITON_ATTN (mixed 256/512 head
|
||||
# dims) whenever no backend is selected, and TRITON_ATTN rejects the
|
||||
# turboquant dtype. This vLLM build has no attention-backend env var.
|
||||
grep -q '^ATTENTION_BACKEND=TURBOQUANT$' "$ENV_FILE"
|
||||
grep -q -- '--attention-backend "$ATTENTION_BACKEND"' "$SERVE"
|
||||
# The 3090 is dedicated to this service, so the engine may take most of
|
||||
# it; the KV pool, not the seq ceiling, caps full-context concurrency.
|
||||
[ "$util" = "0.95" ]
|
||||
|
||||
Reference in new issue
Block a user