frank-primary: four full-window slots, V q8_0 KV, hot expert blocks on-GPU
- CONTEXT_SIZE=1048576 / PARALLEL_SLOTS=4: four conversations, each the full native 262144 window (per-slot window unchanged; slots divide the pool, they never multiply it). - CACHE_TYPE_V=q8_0 / CACHE_TYPE_K=f16: main KV drops 24,576 -> 18,816 B/token (-23.4%); the shared MTP drafter's KV ignores the V type (measured: 2,048 MiB draft pool = the f16 law at 1,048,576 cells). - OT_OVERRIDES: blocks 0-3 on ROCm0 and 45-47 on ROCm1 stay GPU-resident (2.42/2.49 GiB per block, measured); the rest stay CPU-computed. Measured free under load: ~4.9 GB card0 / ~3.4 GB card1; a 5th block on card0 lands at 2.27 GB, under the 3 GB checkpoint floor, so blocks are atomic and 4 is the safe fill. - serve script: --cache-type-k/v env-wired tunables; ONE joined --override-tensor flag (b11030 deprecates repeated flags, and the loader is first-match-wins, so the specific patterns must precede the broad exps=CPU sweep). - seams: the window test asserts per-slot = 262144 at 4 slots; new KV-type test; OT exact-value assertion pins the ordering law.
This commit is contained in:
1 parent
de4d22785d
commit
de85654b76
3 files changed
+76
-34
No files matched your search
@@ -59,12 +59,25 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
# computing them in RAM does not care (20+ tok/s on the same model
|
||||
# shape). exps=CPU plus the n-gram embedding table on CPU; the PLE
|
||||
# table must NEVER carry a GPU target (the 23 -> 13.4 t/s regression).
|
||||
grep -q -- '--override-tensor "$ot"' "$SERVE"
|
||||
# ...and mmap must stay on: --no-mmap forces the 51B-param table into
|
||||
# anonymous memory and the cgroup kill is silent.
|
||||
# ONE joined flag (b11030 deprecates repeated --override-tensor), and
|
||||
# first-match-wins means the block-specific patterns must precede the
|
||||
# broad exps=CPU sweep — the exact value pins the whole law.
|
||||
grep -q -- '--override-tensor "$OT_OVERRIDES"' "$SERVE"
|
||||
! grep -q -- '--no-mmap' "$SERVE"
|
||||
! grep -q '^MMPROJ_ON_GPU' "$ENV_FILE"
|
||||
grep -q '^OT_OVERRIDES=exps=CPU,per_layer_token_embd.weight=CPU$' "$ENV_FILE"
|
||||
ot="$(sed -nE "s/^OT_OVERRIDES='(.*)'$/\1/p" "$ENV_FILE")"
|
||||
[ "$ot" = 'blk\.[0-3]\.ffn_(up|gate|down)_exps=ROCm0,blk\.4[5-7]\.ffn_(up|gate|down)_exps=ROCm1,exps=CPU,per_layer_token_embd.weight=CPU' ]
|
||||
}
|
||||
|
||||
@test "the KV pool is V-quantized, K held at f16" {
|
||||
# V q8_0 halves nothing (it is 1/4 of the per-token law: 18,816 of the
|
||||
# 24,576 B/token f16/f16 cost) but it is the deploy's stated shape; the
|
||||
# flags must be env-wired tunables so the quant can be reverted
|
||||
# without touching the serve script.
|
||||
grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE"
|
||||
grep -q '^CACHE_TYPE_V=q8_0$' "$ENV_FILE"
|
||||
grep -q -- '--cache-type-k "$CACHE_TYPE_K"' "$SERVE"
|
||||
grep -q -- '--cache-type-v "$CACHE_TYPE_V"' "$SERVE"
|
||||
}
|
||||
|
||||
@test "the projector is explicit, never autoloaded" {
|
||||
@@ -83,13 +96,16 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
grep -q -- '--port "$PORT"' "$SERVE"
|
||||
}
|
||||
|
||||
@test "one stream, the model's full native window" {
|
||||
@test "four streams, each the model's full native window" {
|
||||
# --parallel divides --ctx-size into slots; the contract is four
|
||||
# conversations each holding the FULL native 262144 window, so the
|
||||
# per-slot window (ctx / parallel) equals the native size.
|
||||
local ctx parallel
|
||||
ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
parallel="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
[ -n "$ctx" ] && [ -n "$parallel" ]
|
||||
[ "$ctx" -eq 262144 ]
|
||||
[ "$parallel" -eq 1 ]
|
||||
[ "$parallel" -eq 4 ]
|
||||
[ "$((ctx / parallel))" -eq 262144 ]
|
||||
grep -q -- '--ctx-size "$CONTEXT_SIZE"' "$SERVE"
|
||||
grep -q -- '--parallel "$PARALLEL_SLOTS"' "$SERVE"
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user