frank-primary: four full-window slots, V q8_0 KV, hot expert blocks on-GPU

- CONTEXT_SIZE=1048576 / PARALLEL_SLOTS=4: four conversations, each the
  full native 262144 window (per-slot window unchanged; slots divide
  the pool, they never multiply it).
- CACHE_TYPE_V=q8_0 / CACHE_TYPE_K=f16: main KV drops 24,576 -> 18,816
  B/token (-23.4%); the shared MTP drafter's KV ignores the V type
  (measured: 2,048 MiB draft pool = the f16 law at 1,048,576 cells).
- OT_OVERRIDES: blocks 0-3 on ROCm0 and 45-47 on ROCm1 stay
  GPU-resident (2.42/2.49 GiB per block, measured); the rest stay
  CPU-computed. Measured free under load: ~4.9 GB card0 / ~3.4 GB
  card1; a 5th block on card0 lands at 2.27 GB, under the 3 GB
  checkpoint floor, so blocks are atomic and 4 is the safe fill.
- serve script: --cache-type-k/v env-wired tunables; ONE joined
  --override-tensor flag (b11030 deprecates repeated flags, and the
  loader is first-match-wins, so the specific patterns must precede
  the broad exps=CPU sweep).
- seams: the window test asserts per-slot = 262144 at 4 slots; new
  KV-type test; OT exact-value assertion pins the ordering law.

scripts/frank-primary.env

@@ -26,30 +26,43 @@ RUNTIME_DIR=/home/dank/GitPackages/unsloth-llama-b11030
 BIND_HOST=0.0.0.0
 PORT=7770
 
-# One conversation, the model's full native 262144 window.
+# Four conversations, each the model's full native 262144 window.
-CONTEXT_SIZE=262144
+# --parallel divides --ctx-size into slots — the pool is 4x262144 cells
-PARALLEL_SLOTS=1
+# (~23 GiB at the V-quantized law below, split across both cards). Slots
+# DIVIDE the pool; they never multiply it. Per-slot window stays the
+# full native 262144, so no client or models.yml change is implied.
+CONTEXT_SIZE=1048576
+PARALLEL_SLOTS=4
 
 # Spec chain: draft-mtp (the shared Q8_0 head) at depth 2, with the
 # draftless ngram-mod matcher chained behind it. Depth 2 is the measured
 # optimum class for this pair at long contexts (the retired Dankclaw
 # Flash-Next deployment ran exactly this head at depth 2 with 0.83-0.92
-# acceptance); each extra verified position re-batches the CPU expert reads,
+# acceptance); each extra verified position re-batches the CPU expert
-# so depth buys less than bandwidth here.
+# reads, so depth buys less than bandwidth here.
 SPEC_DRAFT_N_MAX=2
 
 # Device placement. -ngl 999 puts every layer's non-overridden tensors on
 # the GPUs; the big overrides then move the bandwidth-huge tensors to host
 # RAM (computed there — the x4-pair measurement: streaming over PCIe loses,
-# computing in RAM does not). OT_OVERRIDES is comma-separated
+# computing in RAM does not care). OT_OVERRIDES is ONE comma-separated
-# <tensor-pattern>=<backend> pairs passed as repeated --override-tensor
+# <tensor-pattern>=<backend> list passed as a single --override-tensor
-# flags. Both overrides are pinned by the header dump of parts 2-3
+# flag (b11030 logs DEPRECATED for repeated flags and wants the joined
-# (2026-09-26, /home/dank/models/dump-gguf.py — GGUF v3, 512 experts
+# form), and the loader applies the FIRST matching pattern per tensor —
-# top-10, qwen4exp.ple.ngram_size=3, heads_per_ngram=8):
+# so the block-specific patterns MUST precede the broad exps=CPU sweep.
-#   * exps=CPU — the routed experts (blk.*.ffn_*_exps.weight, ~100 GiB at
+#
-#     Q8_0), read once per token; the CPU backend computes them from page
+# Hot expert blocks stay GPU-resident (2026-09-26): -ot granularity is the
-#     cache.
+# whole per-block expert tensor (512 experts share one tensor), so "critical
-#   * per_layer_token_embd.weight=CPU — the 51.2B-element PLE n-gram table
+# experts" = blocks co-located on the card that holds their layers.
+# Measured cost: 2.42 GiB/block on ROCm0, 2.49 GiB/block on ROCm1.
+# Blocks 0-3 on ROCm0 + 45-47 on ROCm1 leave ~4.9 GB / ~3.4 GB free under
+# load (vision graph included). A 5th block on ROCm0 lands at 2.27 GB —
+# under the 3 GB checkpoint floor — so it stays off; blocks are atomic,
+# nothing lands card0 inside the 3-4 GB band.
+#   * exps=CPU — the remaining routed experts (blk.*.ffn_*_exps.weight,
+#     ~100 GiB at Q8_0 even with 7 blocks GPU-side), read once per token;
+#     the CPU backend computes them from page cache.
+#   * per_layer_token_embd.weight=CPU — the 51B-element PLE n-gram table
 #     (dims [160, 320001536], Q8_0, 54.4 GiB — the whole of part 3):
 #     sparse row lookups, tiny PCIe traffic from CPU RAM. It doubles as
 #     the shared MTP head's borrow tensor — GPU-resident, the draft load
@@ -59,7 +72,7 @@ SPEC_DRAFT_N_MAX=2
 #     (the table must stay page-cache-backed; anonymous memory
 #     cgroup-kills silently).
 NGL=999
-OT_OVERRIDES=exps=CPU,per_layer_token_embd.weight=CPU
+OT_OVERRIDES='blk\.[0-3]\.ffn_(up|gate|down)_exps=ROCm0,blk\.4[5-7]\.ffn_(up|gate|down)_exps=ROCm1,exps=CPU,per_layer_token_embd.weight=CPU'
 TENSOR_SPLIT=1,1
 
 # The drafter MUST sit on the SECOND R9700 (ROCm1): with -ts 1,1 the main
@@ -80,9 +93,19 @@ SPEC_DRAFT_DEVICE=ROCm1
 BATCH_SIZE=8192
 UBATCH_SIZE=512
 
-# Prompt-cache budget: 16 GiB. The KV itself is small (the hybrid
+# KV cache types: V quantized to q8_0, K held at the f16 default. The V
-# DeltaNet/QSA pattern keeps a 262144 window a fraction of the full-ctx
+# half is what quantizes on this build: main KV drops 24,576 -> 18,816
-# x all-layers number), so the prompt cache is the RAM-side spend that
+# B/token (12 of 48 layers carry KV; the 36 SSM layers are constant-state
-# actually moves prefill on repeat traffic. Host RAM budget: ~175 GiB of
+# and never scale with ctx). NOTE the shared MTP drafter's KV ignores the
-# expert pages + this cache + the server's own arenas inside 246 GiB.
+# V type (measured: draft pool 2,048 MiB = the f16 law at 1,048,576
+# cells) — one layer's worth, so the miss costs ~0.5 GiB on ROCm1.
+CACHE_TYPE_K=f16
+CACHE_TYPE_V=q8_0
+
+# Prompt-cache budget: 16 GiB. The V-quantized KV pool at four full
+# windows is ~23 GiB spread across both cards (the hybrid DeltaNet/QSA
+# pattern keeps even 4x262144 a fraction of a full-ctx x all-layers
+# pool); the prompt cache is the RAM-side spend that actually moves
+# prefill on repeat traffic. Host RAM budget: ~90 GiB of expert pages +
+# this cache + the server's own arenas inside 246 GiB.
 CACHE_RAM=16384

scripts/serve-frank-primary.sh

@@ -55,7 +55,8 @@ fi
 : "${SPEC_DRAFT_DEVICE:=ROCm0}"
 : "${BATCH_SIZE:=8192}"
 : "${UBATCH_SIZE:=512}"
-: "${CACHE_RAM:=16384}"
+: "${CACHE_TYPE_V:=f16}"
+: "${CACHE_TYPE_K:=f16}"
 : "${VERBOSITY:=4}"
 
 SERVER_BIN="$RUNTIME_DIR/llama-server"
@@ -108,7 +109,8 @@ ARGS=(
     --reasoning auto
     --reasoning-format deepseek
     --reasoning-budget -1
-    --flash-attn on
+    --cache-type-k "$CACHE_TYPE_K"
+    --cache-type-v "$CACHE_TYPE_V"
     --batch-size "$BATCH_SIZE"
     --ubatch-size "$UBATCH_SIZE"
     --cache-ram "$CACHE_RAM"
@@ -128,10 +130,11 @@ ARGS=(
     --min-p 0.0
 )
 
-# Comma-separated <pattern>=<backend> pairs -> repeated --override-tensor.
+# Comma-separated <pattern>=<backend> pairs, ONE flag: b11030 logs
-IFS=',' read -ra _ots <<< "$OT_OVERRIDES"
+# DEPRECATED for repeated --override-tensor and wants the joined list.
-for ot in "${_ots[@]}"; do
+# The loader applies the FIRST matching pattern per tensor, so the
-    ARGS+=(--override-tensor "$ot")
+# specific block patterns must precede the broad exps=CPU sweep in
-done
+# frank-primary.env.
+ARGS+=(--override-tensor "$OT_OVERRIDES")
 
 exec "$SERVER_BIN" "${ARGS[@]}"

tests/frank_primary_unit.bats

@@ -59,12 +59,25 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
     # computing them in RAM does not care (20+ tok/s on the same model
     # shape). exps=CPU plus the n-gram embedding table on CPU; the PLE
     # table must NEVER carry a GPU target (the 23 -> 13.4 t/s regression).
-    grep -q -- '--override-tensor "$ot"' "$SERVE"
+    # ONE joined flag (b11030 deprecates repeated --override-tensor), and
-    # ...and mmap must stay on: --no-mmap forces the 51B-param table into
+    # first-match-wins means the block-specific patterns must precede the
-    # anonymous memory and the cgroup kill is silent.
+    # broad exps=CPU sweep — the exact value pins the whole law.
+    grep -q -- '--override-tensor "$OT_OVERRIDES"' "$SERVE"
     ! grep -q -- '--no-mmap' "$SERVE"
     ! grep -q '^MMPROJ_ON_GPU' "$ENV_FILE"
-    grep -q '^OT_OVERRIDES=exps=CPU,per_layer_token_embd.weight=CPU$' "$ENV_FILE"
+    ot="$(sed -nE "s/^OT_OVERRIDES='(.*)'$/\1/p" "$ENV_FILE")"
+    [ "$ot" = 'blk\.[0-3]\.ffn_(up|gate|down)_exps=ROCm0,blk\.4[5-7]\.ffn_(up|gate|down)_exps=ROCm1,exps=CPU,per_layer_token_embd.weight=CPU' ]
+}
+
+@test "the KV pool is V-quantized, K held at f16" {
+    # V q8_0 halves nothing (it is 1/4 of the per-token law: 18,816 of the
+    # 24,576 B/token f16/f16 cost) but it is the deploy's stated shape; the
+    # flags must be env-wired tunables so the quant can be reverted
+    # without touching the serve script.
+    grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE"
+    grep -q '^CACHE_TYPE_V=q8_0$' "$ENV_FILE"
+    grep -q -- '--cache-type-k "$CACHE_TYPE_K"' "$SERVE"
+    grep -q -- '--cache-type-v "$CACHE_TYPE_V"' "$SERVE"
 }
 
 @test "the projector is explicit, never autoloaded" {
@@ -83,13 +96,16 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
     grep -q -- '--port "$PORT"' "$SERVE"
 }
 
-@test "one stream, the model's full native window" {
+@test "four streams, each the model's full native window" {
+    # --parallel divides --ctx-size into slots; the contract is four
+    # conversations each holding the FULL native 262144 window, so the
+    # per-slot window (ctx / parallel) equals the native size.
     local ctx parallel
     ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
     parallel="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")"
     [ -n "$ctx" ] && [ -n "$parallel" ]
-    [ "$ctx" -eq 262144 ]
+    [ "$parallel" -eq 4 ]
-    [ "$parallel" -eq 1 ]
+    [ "$((ctx / parallel))" -eq 262144 ]
     grep -q -- '--ctx-size "$CONTEXT_SIZE"' "$SERVE"
     grep -q -- '--parallel "$PARALLEL_SLOTS"' "$SERVE"
 }