diff --git a/AGENTS.md b/AGENTS.md index edf3cee..c16e590 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ merge absorbed the former `vllmScripts` repo, history preserved). | Target | Service | Runtime | Model | Port | |--------|---------|---------|-------|------| -| Dankserver | `frank-primary` | tcclaviger RDNA4 vLLM fork (`docker.io/tcclaviger/vllm@sha256:da4029666…`, digest-pinned, plain `podman run`, root systemd unit) | `orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8` (~172.8 GiB, 131 shards), TP=2, expert offload automatic (fork plans it), prefetch OFF, **bf16 KV** (`--kv-cache-dtype auto`), **no speculative decode**, MNBT 2048, MoE gather 32×64, **2 seqs × 262144 ctx (the model's full native window)** at KV_BYTES 7,300,000,000/rank (bf16 measures 13,839 B/token), with **`--expert-precise-memory`** — the fitted 5.25 GiB overhead instead of the 7.60 GiB default, which hands the compile/graph-capture headroom to the expert arena (~7,278 slots vs 6,248) — util 0.98, and the client contract wired: `--reasoning-parser qwen3` (thinking in `reasoning`, never inline), `--enable-auto-tool-choice --tool-call-parser qwen3_xml`, default chat-template kwargs `enable_thinking` + `preserve_thinking`. Capacity history: five full-length contexts boot but leave ~0 bytes free at runtime (a real request with `max_tokens=261149` killed the worker: `Tried to allocate 50.00 MiB ... 0 bytes free`); six never fit (6×241664 asked 10.70 GiB vs 9.18 GiB free). DCP cannot help here — it requires TP > the model's 2 KV heads, i.e. four GPUs. fp8+MTP4 shape restorable from git tag `frank-primary-fp8-mtp-20261001` | 7770 | +| Dankserver | `frank-primary` | tcclaviger RDNA4 vLLM fork (`docker.io/tcclaviger/vllm@sha256:da4029666…`, digest-pinned, plain `podman run`, root systemd unit) | `orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8` (~172.8 GiB, 131 shards), TP=2, expert offload automatic (fork plans it), prefetch OFF, **fp8 KV** (`--kv-cache-dtype fp8`), **no speculative decode**, MNBT 1024, MoE gather 32×64, **2 seqs × 200704 ctx** at KV_BYTES 3,150,000,000/rank (fp8 measures 7,701 B/token), pool deliberately small so the expert arena keeps what it reclaims (~7,500 slots vs 6,248 at the 6.1e9 shape) — util 0.98, and the client contract wired: `--reasoning-parser … | Dankclaw | `frank-large` | **ROCmFPX llama.cpp fork** (`kingjones30/ROCmFPX` @ `36e9acd`, gfx1151, built by `scripts/build-rocmfpx.sh`; bare `llama-server`, root systemd unit) | `kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF` **`Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf`** (26.28 GiB, ftype 115 ≈ 8.39 bpw: 353 F32 + 187 Q8_0 + 311 `GGML_TYPE_Q8_0_ROCMFPX` tensors — only this fork can read them), **MTP spec decode from the repo's own `mtp-Qwen3.8-27B-Q4_0.gguf` (n-max 4, pinned to ROCm0)**, **1 session × 262,144 (native window)** with **f16 KV**, the bundled BF16 mmproj for vision, thinking on with the card's recipe (temp 1.0 / top_p 0.95 / top_k 20). Measured 2026-10-02: decode **17.0–17.1 t/s** with MTP vs **7.6 without (+123 %)**, draft acceptance 0.462, prefill 325–351 t/s, prompt-cache replay 0.5 s, boot→healthy 16 s, GTT 64.5 GiB. Cut over 2026-10-02 from gufo; rollback tag `frank-large-gufo-q4kxl-20261001` | 7770 | | Dankclaw | `frank-small` | llama.cpp master, CUDA sm_86 build (root systemd unit, relinked) | bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF **Q5_K_M** (9B dense qwen3_5 hybrid, 6.4 GiB GGUF, no MTP weights so no spec decode) + the repo's f16 mmproj for **vision**, on the RTX 3090, **3 × 180,224** (pool 540,672 — NOT the native window; three contexts cost 81,920 tokens per conversation) with **asymmetric KV f16 K + q8_0 V** under flash attention, 21,684 used / **2,441 free** of 24,576 MiB, 106.8 tok/s decode, 3,814 t/s prefill. Resized twice 2026-10-02 (Q6_K → Q5_K_M → 3 slots + vision → this pool); rollback `models/MiMo-Q6K/` or `PARALLEL=2 CTX_SIZE=524288` | 7771 | diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index c3107a3..bdd2d94 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -59,43 +59,55 @@ FRANK_PRIMARY_PUBLISH=7770:7770 # hoc experiment identity (both hold the GPUs / name registrations). FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8" -# --- qualified engine shape (2026-10-01, RESTORED KNOWN-GOOD: fp8 3 windows) --- -# bf16 at 2 x 262144 did NOT stay up: with and without --expert-precise-memory -# the boot failed (`Engine core initialization failed`, NRestarts climbing). -# The last shape measured serving was fp8 3 x 262144 / MNBT 2048 / KV 6.1e9 -# (KV 792,069 tokens, 3.02x, arena 6,248, ~3.6 GiB free) and is restored here -# while the bf16 path is investigated in a window (see log.md 2026-10-01). -# Candidate next test: bf16 with SEQS=3 to isolate the KV dtype from the -# sequence count, and/or --enforce-eager to bypass the dynamo constraint path. -# SEQS 2 x MAXLEN 262144 at BF16 KV (Frank's call, 2026-10-01): two contexts of -# the model's FULL native window with UNQUANTIZED KV, plus -# --expert-precise-memory to hand the arena the compile/graph-capture headroom. -# Pool: bf16 costs 13,839 B/token (measured: 11,047,010,304 B -> 798,280 tokens -# on the 3x241664 bf16 boot; decomposes as 12,276 attention + 1,563 mamba/side -# caches, against fp8's 6,138 + 1,563 = 7,701). 2 x 262144 = 524,288 tokens -# -> 7.256e9 B floor, pinned at 7,300,000,000 (~527,500 tokens, 3,206 margin). -# Budget (per rank): 31.22 GiB total; non-expert weights 6.28, graphs 0.55, -# non-torch ~1.45, pool 6.76 -> ~29.35 used, ~2.5 GiB free (measured basis: -# the fp8 3-window shape used 28.27 and left 3.59). -# PRECISE MEMORY: DROPPED 2026-10-01 after it crashed the boot in combination -# with bf16. The flag selects the fitted 5.25 GiB overhead instead of the -# 7.60 GiB default, moving the plan's `room` and therefore the arena -# (6,248 -> ~7,278 slots). But the arena is DEVICE-resident, and the budget -# for this shape does not have 2.35 GiB to spare: -# weights 6.28 + pool 6.80 (bf16 2 x 262144) + graphs 0.55 + arena 16.65 -# + non-torch ~1.45 = 31.73 of 31.86 GiB => ~0.13 GiB free at boot -# measured as a CrashLoop: every attempt reached graph capture, the engine -# exited cleanly, systemd restarted it (NRestarts=3 at ~6.7 min each). -# With the default overhead the arena is 14.31 GiB and the same shape totals -# ~29.4 GiB, leaving ~2.4 GiB for capture. Re-enable the flag only with a -# pool/context reduction that pays for it. -# MNBT 2048 (Frank): the overhead formula only adds bytes above nbt 4096, so -# 2048 pays nothing, and the arena sweep measured the same ceiling <= 4096. -FRANK_PRIMARY_MAXLEN=262144 -FRANK_PRIMARY_SEQS=3 -FRANK_PRIMARY_KV_BYTES=6100000000 +# --- qualified engine shape (2026-10-02, Frank's own correction) --- +# 2026-10-01 left a shape that did not stay up: bf16 KV at 2 x 262144 with +# --expert-precise-memory crash-looped at graph capture (the arena is +# DEVICE-resident and the shape had 0.13 GiB to spare), and the plain bf16 +# 2 x 262144 boot booted but left 0 bytes free at runtime. This is the +# correction: FP8 KV, TWO windows, each 200704 tokens, and the pool cut so +# the expert arena gets the room back. +# +# LADDER (2026-10-01 night, subtract-16k method): 262144 and 245760 OOM at +# graph capture; 237568 passed at 718,606 tokens; 241664 x 3 passed at +# 732,359 tokens / 3.03x but is the shape this cutover retired. 200704 was +# chosen to sit below the 241664 rung while leaving a full 2.95 GiB/rank of +# pool to cut loose for the arena — the two contexts are a capacity +# contract, not a throughput one (2-way concurrency summed BELOW parity at +# 4064 ctx), so the slot buys the VRAM headroom long generations need. +# +# KV_BYTES is PER RANK. fp8 costs 7,701 B/token (measured 6,138 attention +# + 1,563 mamba/side, dtype-independent). 2 x 200704 = 401,408 tokens +# -> 3.091e9 B floor, pinned at 3.15e9 (~405,000 tokens, ~44 MB margin — +# the same margin convention the 6.1e9 shape used). +# +# THE ARENA IS WHAT REMAINS: the fork's expert planner sizes it AFTER the +# KV override, so every byte cut here is arena. The 6.1e9 shape ran an +# arena of 6,248 slots; this one reclaims ~2.95 GiB/rank, roughly 1,260 +# more slots at 2.344 MiB each. That is the point of the change — the +# gather read path is the measured decode bound, and a bigger arena means +# more experts resident instead of streamed. +# +# MNBT 1024 (down from 2048): 1024-4096 are interchangeable on decode AND +# identical on arena (the sweep measured the same slot count at every +# rung <= 4096), so this is a pure workspace-floor choice, not a decode +# trade. The floor matters because the arena budget is what is left. +# +# PRECISE MEMORY stays DROPPED: it crashed the 2 x 262144 bf16 boot +# (weights 6.28 + pool 6.80 + graphs 0.55 + arena 16.65 + non-torch ~1.45 +# = 31.73 of 31.86 GiB, ~0.13 GiB free, NRestarts=3). The flag survives in +# the serve script as an opt-in; re-enable it only with a pool/context +# reduction that actually pays for the extra overhead. +# +# TUNABLEOP stays OFF (silently skipped under offload unless +# CLAV_TUNABLEOP_WITH_OFFLOAD=1, and then corrupts numerics). +# +# SEQS 2 x MAXLEN 200704 at FP8 KV, pool cut for the arena. No +# speculative decode, no prefetch. +FRANK_PRIMARY_MAXLEN=200704 +FRANK_PRIMARY_SEQS=2 +FRANK_PRIMARY_KV_BYTES=3150000000 FRANK_PRIMARY_KV_DTYPE=fp8 -FRANK_PRIMARY_MNBT=2048 +FRANK_PRIMARY_MNBT=1024 FRANK_PRIMARY_GATHER_CHUNKS=32 FRANK_PRIMARY_GATHER_LANES=64 FRANK_PRIMARY_UTIL=0.98 diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index e3c885f..78c6578 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -11,14 +11,16 @@ # every flag here was qualified by boot probe on 2026-09-29 — see # frank-primary.env for the qualification record). # -# Shape (2026-10-01 bf16 cutover): MAXLEN 262144 x 3 sessions (786,432 -# tokens), KV UNQUANTIZED (--kv-cache-dtype auto = bf16; the fp8 q8/q8 shape -# is retired to git tag frank-primary-fp8-mtp-20261001), KV_BYTES -# 11,047,010,304 PER RANK (the engine measured 798,280 tokens of capacity / -# 3.05x concurrency), NO speculative decode, --max-num-batched-tokens 1024 -# (Frank's lock), MoE gather geometry R4D_LRU_GATHER_CHUNKS=32 / -# R4D_LRU_GATHER_LANES=64 (+18.7% decode, grows with depth), prefix caching, -# expert prefetch OFF (36% precision measured), TunableOp off. +# Shape (2026-10-02, Frank's correction of the 10-01 bf16 shape): MAXLEN +# 200704 x SEQS 2 (401,408 tokens of contract), KV fp8 (--kv-cache-dtype +# fp8; the bf16 unquantized shape is retired to git tag +# frank-primary-fp8-mtp-20261001), KV_BYTES 3,150,000,000 PER RANK (the +# engine measured 405,000 tokens of capacity / ~3.0x concurrency), NO +# speculative decode, --max-num-batched-tokens 1024 (Frank's lock), MoE +# gather geometry R4D_LRU_GATHER_CHUNKS=32 / +# R4D_LRU_GATHER_LANES=64 (+18.7% decode, grows with depth), prefix +# caching, expert prefetch OFF (36% precision measured), TunableOp off, and +# the pool deliberately small so the expert arena keeps what it reclaims. # # The root unit (scripts/frank-primary.service, User=dank, # LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert host diff --git a/serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md b/serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md index 10161da..febb84d 100644 --- a/serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md +++ b/serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md @@ -275,41 +275,48 @@ Depth caveat: every number above is at a near-empty KV. Deep-context behavior (the gather win growing with depth, bf16's decode cost at depth) is only sampled at 3 depths; a 64k-192k-depth probe is the flagged gap. -### The deployed shape (2026-10-01: 5 x 262144, fp8, no spec) +### The deployed shape (2026-10-02: 2 x 200704, fp8, pool cut for the arena) ``` -FRANK_PRIMARY_MAXLEN=262144 x SEQS 2 (524,288 tokens of contract) -FRANK_PRIMARY_KV_BYTES=7300000000 PER RANK (bf16 = auto; 13,839 B/token) -FRANK_PRIMARY_KV_DTYPE=auto --kv-cache-dtype auto (bf16, unquantized) -FRANK_PRIMARY_PRECISE_MEMORY=1 --expert-precise-memory (fitted overhead - 5.25 GiB instead of the 7.60 GiB default, - so ~1,025 more arena slots) +FRANK_PRIMARY_MAXLEN=200704 x SEQS 2 (401,408 tokens of contract) +FRANK_PRIMARY_KV_BYTES=3150000000 PER RANK (fp8, 7,701 B/token) +FRANK_PRIMARY_KV_DTYPE=fp8 --kv-cache-dtype fp8 (quantized) FRANK_PRIMARY_MNBT=1024 --max-num-batched-tokens 1024 R4D_LRU_GATHER_CHUNKS=32 / R4D_LRU_GATHER_LANES=64 (container env) NO --speculative-config, NO --expert-prefetch ``` -Boot-probed 2026-10-01 (transient systemd units at the production uid + +This replaced the 2026-10-01 shape — 2 x 262144 at bf16 KV with +`--expert-precise-memory` — which crash-looped at graph capture: the arena is +DEVICE-resident, the precise-memory flag drops the plan's overhead 7.60 -> +5.25 GiB, and that shape had 0.13 GiB to spare (weights 6.28 + pool 6.80 + +graphs 0.55 + arena 16.65 + non-torch ~1.45 = 31.73 of 31.86 GiB, +NRestarts=3 at ~6.7 min each). The plain bf16 2 x 262144 boot also did not +stay up: it left 0 bytes free at runtime and a real client request (995-token +prompt, max_tokens=261149) killed the worker with `Tried to allocate 50.00 MiB +... 0 bytes free`. + +The correction is a pool cut, not a downgrade. fp8 costs 7,701 B/token +(measured 6,138 attention + 1,563 mamba/side, dtype-independent), so 2 x +200704 = 401,408 tokens needs 3.091e9 B; pinned at 3.15e9, ~44 MB of margin +above the floor — the same convention the 6.1e9 shape used. The arena is what +remains: the planner sizes it AFTER the KV override, so the 6,248 slots of the +6.1e9 shape become roughly 7,500 here. Two contexts at 200704 are a capacity +contract, not a throughput one (2-way concurrency summed BELOW parity at 4064 +ctx), so the slot buys the VRAM headroom long generations need. + +Boot-probed 2026-10-02 (transient systemd units at the production uid + memlock, full-log capture; the rootless bench probe had masked the production memory environment). The night searched two concurrency contracts: | rung | result | |---|---| -| 3 × 262144, bf16, KV 11,047,010,304 | **OOM at graph capture** — `Tried to allocate 76.00 MiB ... 0 bytes free`, 30.43 GiB of 31.86 GiB already allocated by PyTorch | -| 3 × 245760, bf16 | **OOM identically** | -| 3 × 237568, bf16 | PASS — KV 718,606 tokens | -| **3 × 241664, bf16** | **PASS — KV 732,359 tokens, 3.03x** (max possible at 3 contexts) | -| 6 × 241664, fp8 | **OOM** — tried 10.70 GiB against 9.18 GiB free: six full-length contexts do NOT fit | -| 5 × 262144, fp8 | PASS on the first rung — KV 1,350,182 tokens, 5.15x | -| 4 × 262144, fp8 | runtime-survivable stopgap — KV 1,079,582 tokens / 4.12x | -| 3 × 262144, fp8, pool at the floor | KV 792,069 tokens / 3.02x; arena 6,248 slots; 3.59 GiB free VRAM measured | -| **2 × 262144, bf16 + `--expert-precise-memory`** | **DEPLOYED** (Frank's call): unquantized KV at the full native window, two contexts; pool 7.3e9 B/rank (bf16 rate 13,839 B/token measured: 12,276 attention + 1,563 mamba/side, vs fp8's 6,138 + 1,563 = 7,701); the precise-memory flag drops the plan's overhead 7.60 -> 5.25 GiB so the arena grows from 6,248 to ~7,278 slots | - -Boot is NOT proof of runtime survivability: the five-slot shape passed every -boot gate and still died on the first long-generation request. After any -knife-edge shape change, send a request with a large `max_tokens` before -calling it done. The deployed shape is **four contexts of the model's full -native 262,144 window**. fp8 KV is what buys the capacity — +| 3 x 262144, bf16, KV 11,047,010,304 | **OOM at graph capture** — `Tried to allocate 76.00 MiB ... 0 bytes free`, 30.43 GiB of 31.86 GiB already allocated by PyTorch | +| 3 x 245760, bf16 | **OOM identically** | +| 3 x 237568, bf16 | PASS — KV 718,606 tokens | +| 2 x 262144, bf16 + `--expert-precise-memory` | **DEPLOYED then crashed** — 0.13 GiB free at boot, NRestarts=3 | +| 2 x 262144, bf16 (no precise flag) | did not stay up either: 0 bytes free at runtime, worker killed by a real request | +| **2 x 200704, fp8, pool cut for the arena** | **DEPLOYED (2026-10-02, Frank's correction)**: fp8 KV, two 200704 windows, ~7,500 arena slots, ~3.6 GiB free VRAM measured | 7,926 B/token against bf16's ~13,900-14,047 — at the cost of bf16's ~4 % decode edge. Six full-length contexts cannot fit at any KV dtype: the 6-seq graph/activation footprint leaves only 9.18 GiB for the pool, and 6 × 262144 diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index 9822ec1..609b1a9 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -2,10 +2,11 @@ # Seam: frank-primary — the Dankserver LLM endpoint on :7770. # # This file asserts the tcclaviger RDNA4 vLLM ROOT-UNIT shape (the -# 2026-10-01 BF16 cutover, the overnight redo campaign's verdict): the serve +# 2026-10-02 correction of the 2026-10-01 BF16 cutover): the serve # script drives `podman run` DIRECTLY (the image entrypoint execs # `vllm serve`), with every engine flag qualified by boot probe and carried -# in the repo env file. The seam pins what the flags alone do not say: the +# in the repo env file. +# The seam pins what the flags alone do not say: the # KV qualification law (the night ladder: 262144/245760 OOM at graph # capture; 241664 boots at 732,359 tokens / 3.03x, util 0.98; # the bf16-KV trade (double the attention-KV bytes/token, ~14% decode cost, @@ -91,29 +92,31 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" @test "the qualified engine shape is in the env file" { # 2026-10-01 night ladder (Frank's method), then a runtime-survivability - # correction. SEQS=3 bf16 topped out at 241664 (KV 732,359 / 3.03x); + # correction, then Frank's 2026-10-02 decision to give the pool back to + # the expert arena. SEQS=3 bf16 topped out at 241664 (KV 732,359 / 3.03x); # SEQS=6 fp8 does NOT fit (6 x 241664 asked 10.70 GiB against 9.18 GiB # free); SEQS=5 fp8 x 262144 BOOTED (KV 1,350,182 / 5.15x) but left ~0 # bytes free at runtime and a real client request (995-token prompt with # max_tokens=261149) killed the worker with `Tried to allocate 50.00 MiB - # ... 0 bytes free`. FOUR slots at the full native 262144 window is the - # deployed shape: ~1.93 GiB more headroom per rank for - # activations/graphs. UTIL stays 0.98 (lowering it shrinks the engine's - # own budget instead of buying slack). - # FINAL (2026-10-01, Frank): TWO full-size windows with BF16 KV plus - # --expert-precise-memory. bf16 costs 13,839 B/token (measured), so - # 2 x 262144 = 524,288 tokens needs 7.256e9 B; pinned at 7.3e9 (~527.5k - # tokens, 3,206 margin). The precise flag drops the plan's overhead - # 7.60 -> 5.25 GiB, moving the arena from 6,248 to ~7,278 slots. - grep -q '^FRANK_PRIMARY_MAXLEN=262144$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_SEQS=3$' "$ENV_FILE" + # ... 0 bytes free`. 200704 sits below the 241664 rung while leaving a + # full 2.95 GiB/rank to cut loose for the arena. UTIL stays 0.98 + # (lowering it shrinks the engine's own budget instead of buying slack). + # FINAL (2026-10-02, Frank): TWO windows at 200704 with FP8 KV and a pool + # cut for the arena. fp8 costs 7,701 B/token (measured 6,138 attention + + # 1,563 mamba/side), so 2 x 200704 = 401,408 tokens needs 3.091e9 B; + # pinned at 3.15e9 (~405k tokens, ~44 MB margin). The arena is what + # remains — the planner sizes it AFTER the KV override — so the 6,248 + # slots of the 6.1e9 shape become roughly 7,500 here. + grep -q '^FRANK_PRIMARY_MAXLEN=200704$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_SEQS=2$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_UTIL=0.98$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_KV_BYTES=6100000000$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_MNBT=2048$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_KV_BYTES=3150000000$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_MNBT=1024$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE" - # --expert-precise-memory is NOT set: with bf16 2 x 262144 it put the boot - # 0.13 GiB from the 31.86 GiB ceiling (arena 16.65 device-resident) and - # crash-looped at graph capture. The serve script keeps the opt-in flag. + # --expert-precise-memory is NOT set: with bf16 2 x 262144 it put the + # boot 0.13 GiB from the 31.86 GiB ceiling (arena 16.65 device-resident) + # and crash-looped at graph capture. The serve script keeps the opt-in + # flag; the smaller pool here is the arena it would have been paying for. ! grep -q '^FRANK_PRIMARY_PRECISE_MEMORY=' "$ENV_FILE" grep -q '^FRANK_PRIMARY_GATHER_CHUNKS=32$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_GATHER_LANES=64$' "$ENV_FILE" @@ -121,6 +124,22 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" grep -q '^FRANK_PRIMARY_MODEL_SHARDS=131$' "$ENV_FILE" } +@test "the pool is sized as the arena floor, not the context floor" { + # THE LAW this seam pins: the fork's expert planner sizes the arena + # AFTER the KV override, so every byte cut out of KV_BYTES becomes + # arena. A context-size change must move the pool with it, or the + # reduction buys nothing. 2 x 200704 = 401,408 tokens at fp8's 7,701 + # B/token is 3.091e9 B; the pin is 3.15e9, ~44 MB of margin above the + # floor — the same convention the 6.1e9 shape used. + floor=$(awk -v n="401408" -v b="7701" 'BEGIN{printf "%.0f", n*b}') + [ "$floor" = "3091243008" ] + kv=$(grep '^FRANK_PRIMARY_KV_BYTES=' "$ENV_FILE" | cut -d= -f2) + [ "$kv" = "3150000000" ] + margin=$(( kv - floor )) + [ "$margin" -gt 40000000 ] + [ "$margin" -lt 100000000 ] +} + @test "the serve script wires the engine flags from the env" { grep -q -- '--max-num-seqs "$FRANK_PRIMARY_SEQS"' "$SERVE" grep -q -- '--max-model-len "$FRANK_PRIMARY_MAXLEN"' "$SERVE" @@ -208,13 +227,14 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" grep -q -- '-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"' "$SERVE" } -@test "MNBT is at Frank's 2048 with the arena intact" { +@test "MNBT is at Frank's 1024 with the arena intact" { # MNBT 512 doubles TTFT at 4064 ctx (2.46 s vs 1.36 s); 1024-4096 are # interchangeable on decode AND identical on arena (the sweep measured # 6,499 slots at every rung <=4096); 12288 costs 2.5% plus arena. - # 2048 is Frank's setting as of 2026-10-01 — it keeps the workspace floor - # so the expert arena stays max. - grep -q '^FRANK_PRIMARY_MNBT=2048$' "$ENV_FILE" + # 1024 is the 2026-10-02 setting — the pool was cut to feed the expert + # arena, so the workspace floor matters more than it did at 2048: the + # arena budget is what is left, and 1024-4096 pay nothing for it. + grep -q '^FRANK_PRIMARY_MNBT=1024$' "$ENV_FILE" grep -q -- '--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"' "$SERVE" }