diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index a3531e3..b2f3b68 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -26,30 +26,43 @@ RUNTIME_DIR=/home/dank/GitPackages/unsloth-llama-b11030 BIND_HOST=0.0.0.0 PORT=7770 -# One conversation, the model's full native 262144 window. -CONTEXT_SIZE=262144 -PARALLEL_SLOTS=1 +# Four conversations, each the model's full native 262144 window. +# --parallel divides --ctx-size into slots — the pool is 4x262144 cells +# (~23 GiB at the V-quantized law below, split across both cards). Slots +# DIVIDE the pool; they never multiply it. Per-slot window stays the +# full native 262144, so no client or models.yml change is implied. +CONTEXT_SIZE=1048576 +PARALLEL_SLOTS=4 # Spec chain: draft-mtp (the shared Q8_0 head) at depth 2, with the # draftless ngram-mod matcher chained behind it. Depth 2 is the measured # optimum class for this pair at long contexts (the retired Dankclaw # Flash-Next deployment ran exactly this head at depth 2 with 0.83-0.92 -# acceptance); each extra verified position re-batches the CPU expert reads, -# so depth buys less than bandwidth here. +# acceptance); each extra verified position re-batches the CPU expert +# reads, so depth buys less than bandwidth here. SPEC_DRAFT_N_MAX=2 # Device placement. -ngl 999 puts every layer's non-overridden tensors on # the GPUs; the big overrides then move the bandwidth-huge tensors to host # RAM (computed there — the x4-pair measurement: streaming over PCIe loses, -# computing in RAM does not). OT_OVERRIDES is comma-separated -# = pairs passed as repeated --override-tensor -# flags. Both overrides are pinned by the header dump of parts 2-3 -# (2026-09-26, /home/dank/models/dump-gguf.py — GGUF v3, 512 experts -# top-10, qwen4exp.ple.ngram_size=3, heads_per_ngram=8): -# * exps=CPU — the routed experts (blk.*.ffn_*_exps.weight, ~100 GiB at -# Q8_0), read once per token; the CPU backend computes them from page -# cache. -# * per_layer_token_embd.weight=CPU — the 51.2B-element PLE n-gram table +# computing in RAM does not care). OT_OVERRIDES is ONE comma-separated +# = list passed as a single --override-tensor +# flag (b11030 logs DEPRECATED for repeated flags and wants the joined +# form), and the loader applies the FIRST matching pattern per tensor — +# so the block-specific patterns MUST precede the broad exps=CPU sweep. +# +# Hot expert blocks stay GPU-resident (2026-09-26): -ot granularity is the +# whole per-block expert tensor (512 experts share one tensor), so "critical +# experts" = blocks co-located on the card that holds their layers. +# Measured cost: 2.42 GiB/block on ROCm0, 2.49 GiB/block on ROCm1. +# Blocks 0-3 on ROCm0 + 45-47 on ROCm1 leave ~4.9 GB / ~3.4 GB free under +# load (vision graph included). A 5th block on ROCm0 lands at 2.27 GB — +# under the 3 GB checkpoint floor — so it stays off; blocks are atomic, +# nothing lands card0 inside the 3-4 GB band. +# * exps=CPU — the remaining routed experts (blk.*.ffn_*_exps.weight, +# ~100 GiB at Q8_0 even with 7 blocks GPU-side), read once per token; +# the CPU backend computes them from page cache. +# * per_layer_token_embd.weight=CPU — the 51B-element PLE n-gram table # (dims [160, 320001536], Q8_0, 54.4 GiB — the whole of part 3): # sparse row lookups, tiny PCIe traffic from CPU RAM. It doubles as # the shared MTP head's borrow tensor — GPU-resident, the draft load @@ -59,7 +72,7 @@ SPEC_DRAFT_N_MAX=2 # (the table must stay page-cache-backed; anonymous memory # cgroup-kills silently). NGL=999 -OT_OVERRIDES=exps=CPU,per_layer_token_embd.weight=CPU +OT_OVERRIDES='blk\.[0-3]\.ffn_(up|gate|down)_exps=ROCm0,blk\.4[5-7]\.ffn_(up|gate|down)_exps=ROCm1,exps=CPU,per_layer_token_embd.weight=CPU' TENSOR_SPLIT=1,1 # The drafter MUST sit on the SECOND R9700 (ROCm1): with -ts 1,1 the main @@ -80,9 +93,19 @@ SPEC_DRAFT_DEVICE=ROCm1 BATCH_SIZE=8192 UBATCH_SIZE=512 -# Prompt-cache budget: 16 GiB. The KV itself is small (the hybrid -# DeltaNet/QSA pattern keeps a 262144 window a fraction of the full-ctx -# x all-layers number), so the prompt cache is the RAM-side spend that -# actually moves prefill on repeat traffic. Host RAM budget: ~175 GiB of -# expert pages + this cache + the server's own arenas inside 246 GiB. +# KV cache types: V quantized to q8_0, K held at the f16 default. The V +# half is what quantizes on this build: main KV drops 24,576 -> 18,816 +# B/token (12 of 48 layers carry KV; the 36 SSM layers are constant-state +# and never scale with ctx). NOTE the shared MTP drafter's KV ignores the +# V type (measured: draft pool 2,048 MiB = the f16 law at 1,048,576 +# cells) — one layer's worth, so the miss costs ~0.5 GiB on ROCm1. +CACHE_TYPE_K=f16 +CACHE_TYPE_V=q8_0 + +# Prompt-cache budget: 16 GiB. The V-quantized KV pool at four full +# windows is ~23 GiB spread across both cards (the hybrid DeltaNet/QSA +# pattern keeps even 4x262144 a fraction of a full-ctx x all-layers +# pool); the prompt cache is the RAM-side spend that actually moves +# prefill on repeat traffic. Host RAM budget: ~90 GiB of expert pages + +# this cache + the server's own arenas inside 246 GiB. CACHE_RAM=16384 diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index d91748d..031522f 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -55,7 +55,8 @@ fi : "${SPEC_DRAFT_DEVICE:=ROCm0}" : "${BATCH_SIZE:=8192}" : "${UBATCH_SIZE:=512}" -: "${CACHE_RAM:=16384}" +: "${CACHE_TYPE_V:=f16}" +: "${CACHE_TYPE_K:=f16}" : "${VERBOSITY:=4}" SERVER_BIN="$RUNTIME_DIR/llama-server" @@ -108,7 +109,8 @@ ARGS=( --reasoning auto --reasoning-format deepseek --reasoning-budget -1 - --flash-attn on + --cache-type-k "$CACHE_TYPE_K" + --cache-type-v "$CACHE_TYPE_V" --batch-size "$BATCH_SIZE" --ubatch-size "$UBATCH_SIZE" --cache-ram "$CACHE_RAM" @@ -128,10 +130,11 @@ ARGS=( --min-p 0.0 ) -# Comma-separated = pairs -> repeated --override-tensor. -IFS=',' read -ra _ots <<< "$OT_OVERRIDES" -for ot in "${_ots[@]}"; do - ARGS+=(--override-tensor "$ot") -done +# Comma-separated = pairs, ONE flag: b11030 logs +# DEPRECATED for repeated --override-tensor and wants the joined list. +# The loader applies the FIRST matching pattern per tensor, so the +# specific block patterns must precede the broad exps=CPU sweep in +# frank-primary.env. +ARGS+=(--override-tensor "$OT_OVERRIDES") exec "$SERVER_BIN" "${ARGS[@]}" diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index 3864756..8b56d17 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -59,12 +59,25 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" # computing them in RAM does not care (20+ tok/s on the same model # shape). exps=CPU plus the n-gram embedding table on CPU; the PLE # table must NEVER carry a GPU target (the 23 -> 13.4 t/s regression). - grep -q -- '--override-tensor "$ot"' "$SERVE" - # ...and mmap must stay on: --no-mmap forces the 51B-param table into - # anonymous memory and the cgroup kill is silent. + # ONE joined flag (b11030 deprecates repeated --override-tensor), and + # first-match-wins means the block-specific patterns must precede the + # broad exps=CPU sweep — the exact value pins the whole law. + grep -q -- '--override-tensor "$OT_OVERRIDES"' "$SERVE" ! grep -q -- '--no-mmap' "$SERVE" ! grep -q '^MMPROJ_ON_GPU' "$ENV_FILE" - grep -q '^OT_OVERRIDES=exps=CPU,per_layer_token_embd.weight=CPU$' "$ENV_FILE" + ot="$(sed -nE "s/^OT_OVERRIDES='(.*)'$/\1/p" "$ENV_FILE")" + [ "$ot" = 'blk\.[0-3]\.ffn_(up|gate|down)_exps=ROCm0,blk\.4[5-7]\.ffn_(up|gate|down)_exps=ROCm1,exps=CPU,per_layer_token_embd.weight=CPU' ] +} + +@test "the KV pool is V-quantized, K held at f16" { + # V q8_0 halves nothing (it is 1/4 of the per-token law: 18,816 of the + # 24,576 B/token f16/f16 cost) but it is the deploy's stated shape; the + # flags must be env-wired tunables so the quant can be reverted + # without touching the serve script. + grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE" + grep -q '^CACHE_TYPE_V=q8_0$' "$ENV_FILE" + grep -q -- '--cache-type-k "$CACHE_TYPE_K"' "$SERVE" + grep -q -- '--cache-type-v "$CACHE_TYPE_V"' "$SERVE" } @test "the projector is explicit, never autoloaded" { @@ -83,13 +96,16 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" grep -q -- '--port "$PORT"' "$SERVE" } -@test "one stream, the model's full native window" { +@test "four streams, each the model's full native window" { + # --parallel divides --ctx-size into slots; the contract is four + # conversations each holding the FULL native 262144 window, so the + # per-slot window (ctx / parallel) equals the native size. local ctx parallel ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" parallel="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" [ -n "$ctx" ] && [ -n "$parallel" ] - [ "$ctx" -eq 262144 ] - [ "$parallel" -eq 1 ] + [ "$parallel" -eq 4 ] + [ "$((ctx / parallel))" -eq 262144 ] grep -q -- '--ctx-size "$CONTEXT_SIZE"' "$SERVE" grep -q -- '--parallel "$PARALLEL_SLOTS"' "$SERVE" }