Merge pull request 'feat(frank-archon): four decode slots over a >=700k-token KV pool' (#59) from wt/20261020-213526 into main

scripts/frank-archon.env

@@ -46,31 +46,35 @@ FRANK_ARCHON_TP=2
 # stays EXACT for single-sequence decode and any step of <= 25 rows — see
 # MAX_SEQS x (1 + SPEC_TOKENS) below.
 FRANK_ARCHON_TP_WIRE=wht6
-# Decode slots. A slot costs KV, never decode; the pool is sized to this
-# contract and nothing more. At 1.8.1 the budget resolver caps the cache at
-# 556,348 fp8 tokens — 2.8 sessions at 200,000 — so a THIRD full-height
-# conversation answers 400 while /health and every shorter chat work fine
-# (3 x 50k streams run at 77.6 t/s aggregate). Frank's trade: declare the two
-# slots the engine can hold and let the freed ~1.1 GiB go to the expert plane,
-# instead of advertising a third slot that only exists at shorter prompts.
-FRANK_ARCHON_MAX_SEQS=2
+# Decode slots. A slot costs KV and pinned RAM, never decode: the cache target is
+# exactly MAX_SEQS x MAX_MODEL_LEN and the checkpoint store asks (1 + what the
+# retention policy keeps) per slot. Ordered 2026-10-21: FOUR slots over a pool of at
+# least 700,000 fp8 tokens, rounding UP if the resolver cannot land it exactly. The
+# target is therefore 4 x 200,000 = 800,000 tokens and HOST_POOL_MIB decides whether
+# the cache gets it (expert_vram_floor, core/place/planner.cpp:100-104: the card must
+# keep expert-plane-minus-host-room). The 2-slot shape was the answer to a
+# 365,204-token pool that backed 1.8 full-height sessions; at this pool the third and
+# fourth conversations are backed by the cache instead of over-committing it.
+FRANK_ARCHON_MAX_SEQS=4
 # Served window. The engine treats `kv group tokens >= max-model-len` as a HARD
 # limit, and models.yml contextWindow MUST quote this number (over-length is a
 # refusal, not a truncation). The pool is sized for seqs x maxlen, not one
 # sequence.
 FRANK_ARCHON_MAX_MODEL_LEN=200000
 FRANK_ARCHON_PLACEMENT=expert_tiered
-# Pinned host pool for expert rows that do not fit VRAM. Under the budget
-# resolver this is ALSO the lever that decides which experts stay on the card:
-# the card's leftover goes to the expert plane first and the KV cache takes the
-# rest, so growing the pool moves experts off the card and shrinking it spills
-# them onto VRAM the cache wanted. Under-sizing it is a boot failure (36,864
-# refused a boot). The ledger's coverage line is the referee, not this file.
-# Measured, not guessed: 45,568 (PR #30) moved host_pool to 45.09 GiB and left
-# vram_kv byte-identical at 4.20 GiB / 556,348 tokens on both ranks — extra
-# host RAM above what the expert plane's footprint needs buys no cache. Back at
-# 44,928, the size that boots clean.
-FRANK_ARCHON_HOST_POOL_MIB=44928
+# Pinned host pool for expert rows that do not fit VRAM, PINNED ONCE PER RANK: the
+# host must pin tp x this number, and the refusal that names it is
+# core/engine_bringup.cpp:250-265. Under the budget resolver this is ALSO the KV
+# lever -- the card keeps expert-plane-minus-this-room, so every MiB added moves one
+# MiB of experts off the card and into the cache. Derived from the boot ledger (a
+# 23.73 GiB floor at 44,928 room puts plane + 2 units at 67.60 GiB a rank), 49,152
+# drops the floor to 19.6 GiB and leaves the cache ~6.9 GiB, which is enough to back
+# the whole 4 x 200,000 target; the leftover returns to the experts. Cost is paid
+# twice over: pinned RAM (tp x 48 GiB, against the TTM target below) and expert
+# residency -- ~3.2 GiB a card of experts now lives in host RAM and is read over the
+# x8 link when the router picks it. The ledger's coverage line is the referee, not
+# this file: if vram_kv comes back under 700,000 tokens, raise this 1024 and redeploy.
+FRANK_ARCHON_HOST_POOL_MIB=49152
 # VRAM left unclaimed per card (publisher default 96 MiB); the engine's settle
 # floor is built from it.
 FRANK_ARCHON_GPU_HEADROOM_MIB=96
@@ -131,9 +135,10 @@ FRANK_ARCHON_NGRAM_PLACEMENT=ram
 # RESETS pages_limit to half of RAM on every boot; below the plan the load dies
 # mid-pin with an anonymous allocator error. ttm-preflight.sh raises the knob
 # itself before refusing by name, so recovery is a boot, not a visit.
-# Floor = 2 x HOST_POOL + 48,824 (static weights + arena) + PREFIX_CACHE + MNBT
-# per the bats ledger test; at the 44,928 pool that computes to 148,920 MiB.
-FRANK_ARCHON_TTM_MIN_MIB=150000
+# Floor = 2 x HOST_POOL + 48,824 (static weights + arena) + PREFIX_CACHE + MNBT + the
+# auto checkpoint store (MAX_SEQS x 10 slots of 111.98 MiB), per the bats ledger test;
+# at the 49,152 pool with 4 slots that computes to 161,848 MiB.
+FRANK_ARCHON_TTM_MIN_MIB=162000
 FRANK_ARCHON_TTM_TARGET_MIB=204800
 # Escape hatch for experiments. Unset -> no extra argv; the serve script expands
 # this unquoted on purpose, so keep multi-flag values as one string.

scripts/serve-frank-archon.sh

@@ -15,9 +15,9 @@
 # log states every budget before anything is allocated (weights vs KV
 # split, the expert host pool's coverage of the worst case, the PLE pin,
 # the P2P answer per pair) — read it after every resize.
-# Deployed shape (2026-10-19, radiance 1.8.1): TWO decode slots x 200,000 tokens,
-# fp8 KV, 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache in pinned host
-# RAM, PLE pinned in RAM. The per-card split between the expert slab and the KV
+# Deployed shape (2026-10-21, radiance 1.8.1): FOUR decode slots x 200,000 tokens over a
+# >= 700,000-token fp8 KV pool, 48 GiB/rank pinned expert pool, MTP depth 2, prefix cache
+# in pinned host RAM, PLE pinned in RAM. The per-card split between the expert slab and the KV
 # pool is NO LONGER an operator number — 1.3.0 removed the VRAM pair and the budget
 # resolver derives it from headroom, the cache bound and the host pool, so the slab
 # size and the pool's token count are LEDGER lines. After every boot read them, plus
@@ -27,9 +27,9 @@
 # pooled experts are read synchronously) — and the cache's session backing.
 #
 # The root unit (scripts/frank-archon.service, User=dank,
-# LimitMEMLOCK=infinity) grants the pin budget: 2 x 44 GiB expert pools +
-# the 47.7 GiB PLE table + 8 GiB prefix host tier — ~147 GiB of pinned
-# pages, which is ALSO why this script refuses to start when ttm's
+# LimitMEMLOCK=infinity) grants the pin budget: 2 x 48 GiB expert pools + the 47.7 GiB PLE
+# table + 8 GiB prefix host tier + the checkpoint store (slots x 112 MiB) — ~159 GiB of
+# pinned pages, which is ALSO why this script refuses to start when ttm's
 # pages_limit/pages_max cap is under FRANK_ARCHON_TTM_MIN_MIB (see below).
 set -euo pipefail
 

tests/frank_archon_unit.bats

@@ -145,7 +145,7 @@ nosudo_bin() {
     # (torch.compile query_start_loc constraint, 2026-10-02); radiance booted
     # seqs 2 in the fp8-native era and the 2026-10-06 sweep had seqs 2 and 3
     # tied inside the +-3% noise floor — slots buy concurrency, not speed.
-    grep -q '^FRANK_ARCHON_MAX_SEQS=2$' "$ENV_FILE"
+    grep -q '^FRANK_ARCHON_MAX_SEQS=4$' "$ENV_FILE"
     grep -q '^FRANK_ARCHON_TP=2$' "$ENV_FILE"
     grep -q '^FRANK_ARCHON_TP_WIRE=wht6$' "$ENV_FILE"
     grep -q '^FRANK_ARCHON_SPEC_TOKENS=2$' "$ENV_FILE"
@@ -158,7 +158,7 @@ nosudo_bin() {
     # so the pool's token count is a boot-ledger line — verified live by the
     # capacity leg of scripts/probe-frank-archon-kv.py at every cutover.
     grep -q '^FRANK_ARCHON_PLACEMENT=expert_tiered$' "$ENV_FILE"
-    grep -q '^FRANK_ARCHON_HOST_POOL_MIB=44928$' "$ENV_FILE"
+    grep -q '^FRANK_ARCHON_HOST_POOL_MIB=49152$' "$ENV_FILE"
     grep -q '^FRANK_ARCHON_KV_DTYPE=fp8$' "$ENV_FILE"
     grep -q '^FRANK_ARCHON_NGRAM_PLACEMENT=ram$' "$ENV_FILE"
     # Prefix cache: pinned host RAM only (the disk tier was removed), and the
@@ -177,8 +177,11 @@ nosudo_bin() {
     [ "$ttmt" -gt "$ttmm" ]
     pool=$(awk -F= '/^FRANK_ARCHON_HOST_POOL_MIB=/{print $2}' "$ENV_FILE")
     ph=$(awk -F= '/^FRANK_ARCHON_PREFIX_CACHE_MIB=/{print $2}' "$ENV_FILE")
-    # 2 pools + PLE 47.7 GiB (48,824 MiB) + prefix host, rounded up.
-    ledger=$(( 2 * pool + 48824 + ph + 2048 ))
+    # 2 pools + PLE 47.7 GiB (48,824 MiB) + prefix host + MNBT + the auto
+    # checkpoint store (MAX_SEQS x 10 slots of 112 MiB -- the store is pinned host
+    # memory too, and it grows with the slot count).
+    seqs=$(awk -F= '/^FRANK_ARCHON_MAX_SEQS=/{print $2}' "$ENV_FILE")
+    ledger=$(( 2 * pool + 48824 + ph + 2048 + seqs * 10 * 112 ))
     [ "$ttmm" -ge "$ledger" ]
     # The .rad size floor is positive (a truncated convert must fail fast).
     grep -q '^FRANK_ARCHON_RAD_MIN_MIB=' "$ENV_FILE"