diff --git a/scripts/frank-small.env b/scripts/frank-small.env index d0f2a93..c84a785 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -29,21 +29,24 @@ # for as many concurrent sessions as the card holds. # MAX_MODEL_LEN=262144 the model's max_position_embeddings # GPU_MEM_UTIL=0.95 the card is dedicated to this service -# KV_CACHE_DTYPE=int8_per_token_head int8 KV (per-token, per-head scales) -# in the same TRITON_ATTN backend — pure ALU -# dequant, no fp8 hardware needed, so it runs -# on sm_86. fp8_e4m3 is the cheaper option but -# the 3090 is sm_86 (Ampere) and has no native -# fp8: the TRITON_ATTN backend Gemma4 forces -# emits cvt.e4m3x2 instructions that exist only -# from sm_89 (Ada), so fp8 KV cannot boot on -# this card (measured 2026-09-19; the "sm_89 -# patch" experiment is in .AGENTTRASH). int8 -# halves the KV charge (289,647 -> ~579k-token -# pool, two -> four full-context sessions) at -# the cost of a dequant in the attention path; -# quality verified against the bf16 baseline -# (needle + short battery) on 2026-09-19. +# KV_CACHE_DTYPE=turboquant_k8v4 TurboQuant KV: K at 8-bit, V at 4-bit, +# per-16-element block scales packed INSIDE the +# tensor (Hadamard-rotated, bf16 compute, +# dedicated TURBOQUANT attention backend with +# an explicit Ampere path). 2.67x smaller KV +# than bf16 (289,647 -> ~772k-token pool, two +# -> five-or-six full-context sessions). +# The cheaper int8/fp8 per-token-head dtypes +# CANNOT boot this model: they keep a separate +# fp32 scale tensor per (token, head), and +# Gemma4's hybrid head dims (SWA 256 / full +# 512) make the per-token page bytes 4160 vs +# 8256 — not an integer ratio, so vLLM's +# unify_kv_cache_spec_page_size raises +# NotImplementedError (measured 2026-09-19, +# nine crash-loop restarts). fp8 KV is +# hardware-dead on this card anyway (sm_86, +# no cvt.e4m3x2). # MAX_NUM_SEQS=16 the batch ceiling; the KV pool, not this, # caps full-context concurrency # MAX_NUM_BATCHED_TOKENS=16384 the prefill chunk; larger chunks prefill @@ -77,11 +80,11 @@ ALIAS=frank-small BIND_ADDR=0.0.0.0 LISTEN_PORT=8000 -# Full window, dedicated card, int8 KV (see above), roomy batch ceiling, +# Full window, dedicated card, turboquant k8v4 KV (see above), roomy batch ceiling, # 16k prefill chunk, 4-token MTP draft (the assistant carries 4 layers). MAX_MODEL_LEN=262144 GPU_MEM_UTIL=0.95 -KV_CACHE_DTYPE=int8_per_token_head +KV_CACHE_DTYPE=turboquant_k8v4 MAX_NUM_SEQS=16 MAX_NUM_BATCHED_TOKENS=16384 SPEC_NUM_TOKENS=4 diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 369dfd3..2fdef35 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -98,7 +98,7 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE" } -@test "frank-small serves the full window with int8 KV on a dedicated card" { +@test "frank-small serves the full window with turboquant k8v4 KV on a dedicated card" { local maxlen kvdtype util seqs maxlen="$(sed -nE 's/^MAX_MODEL_LEN=([0-9]+)$/\1/p' "$ENV_FILE")" kvdtype="$(sed -nE 's/^KV_CACHE_DTYPE=(.+)$/\1/p' "$ENV_FILE")" @@ -106,12 +106,13 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" seqs="$(sed -nE 's/^MAX_NUM_SEQS=([0-9]+)$/\1/p' "$ENV_FILE")" # The model's FULL trained window (262144), not a divided one. [ "$maxlen" = "262144" ] - # int8 KV (per-token, per-head scales): fp8_e4m3 would halve the pool - # charge further, but the 3090 is sm_86 (Ampere) — no native fp8, and - # the TRITON_ATTN backend that Gemma4 forces emits cvt.e4m3x2 - # instructions that exist only from sm_89. int8 is pure ALU dequant in - # the same backend, so it runs. - [ "$kvdtype" = "int8_per_token_head" ] + # TurboQuant k8v4 KV (K 8-bit, V 4-bit, in-tensor block scales, + # dedicated TURBOQUANT backend with an Ampere path). The int8/fp8 + # per-token-head dtypes cannot boot this model: their separate fp32 + # scale tensor breaks the page-size ratio between Gemma4's SWA + # (head_dim 256) and full-attention (head_dim 512) layers, and fp8 is + # hardware-dead on sm_86 anyway. + [ "$kvdtype" = "turboquant_k8v4" ] # The 3090 is dedicated to this service, so the engine may take most of # it; the KV pool, not the seq ceiling, caps full-context concurrency. [ "$util" = "0.95" ]