frank-small: int8 KV (per-token, per-head) — doubles the pool
fp8 KV is hardware-dead on the 3090 (sm_86, no cvt.e4m3x2), so int8 is the cheapest quant the TRITON_ATTN backend can run: pure ALU dequant, same backend, no fp8 tensor cores needed. 289,647 -> ~579k-token pool, two -> four full-context sessions. Quality verified against the bf16 baseline (needle-in-haystack at 213k + short battery) before the swap.
This commit is contained in:
1 parent
ae137317ab
commit
8d88b3a0a9
2 files changed
+24
-16
No files matched your search
@@ -98,7 +98,7 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
|
||||
! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE"
|
||||
}
|
||||
|
||||
@test "frank-small serves the full window with bf16 KV on a dedicated card" {
|
||||
@test "frank-small serves the full window with int8 KV on a dedicated card" {
|
||||
local maxlen kvdtype util seqs
|
||||
maxlen="$(sed -nE 's/^MAX_MODEL_LEN=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
kvdtype="$(sed -nE 's/^KV_CACHE_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
|
||||
@@ -106,10 +106,12 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
|
||||
seqs="$(sed -nE 's/^MAX_NUM_SEQS=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
# The model's FULL trained window (262144), not a divided one.
|
||||
[ "$maxlen" = "262144" ]
|
||||
# bf16 KV: fp8_e4m3 would halve the pool charge, but the 3090 is
|
||||
# sm_86 (Ampere) — no native fp8, and the TRITON_ATTN backend that
|
||||
# Gemma4 forces emits cvt.e4m3x2 instructions that exist only from sm_89.
|
||||
[ "$kvdtype" = "auto" ]
|
||||
# int8 KV (per-token, per-head scales): fp8_e4m3 would halve the pool
|
||||
# charge further, but the 3090 is sm_86 (Ampere) — no native fp8, and
|
||||
# the TRITON_ATTN backend that Gemma4 forces emits cvt.e4m3x2
|
||||
# instructions that exist only from sm_89. int8 is pure ALU dequant in
|
||||
# the same backend, so it runs.
|
||||
[ "$kvdtype" = "int8_per_token_head" ]
|
||||
# The 3090 is dedicated to this service, so the engine may take most of
|
||||
# it; the KV pool, not the seq ceiling, caps full-context concurrency.
|
||||
[ "$util" = "0.95" ]
|
||||
|
||||
Reference in new issue
Block a user