From 1bb639952434580fbf0f395d348f8c0867a5a375 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 20 Oct 2026 23:10:05 -0700 Subject: [PATCH] feat(frank-archon): four decode slots over a >=700k-token KV pool Frank's order: 4 slots, a 700k pool, round UP if the resolver cannot land it exactly. The cache target is MAX_SEQS x MAX_MODEL_LEN, so the demand is 800,000 fp8 tokens; the resolver funds ~91-93 % of demand (three boots of evidence on the endpoint page), which clears 700k, and the VRAM has to come from the expert plane -- which has to go somewhere, so HOST_POOL_MIB moves with it. expert_vram_floor (core/place/planner.cpp:100-104) is exactly plane + 2 units - host_room, and the live boot ledger pins that plane at 67.60 GiB a rank (23.73 GiB floor at 44,928 of room), so 49,152 drops the floor to 19.61 GiB and leaves the cache ~6.9 GiB: enough for the whole target, with the leftover returned to the experts. Costs, both real: pinned host RAM 87.75 -> 96.0 GiB (the pool is pinned once per rank) and ~3.2 GiB a card of expert residency, read over the x8 Gen5 link when the router picks a host-resident expert. The boot ledger's vram_kv line is the referee; if it comes back under 700,000 tokens HOST_POOL_MIB goes up 1024 and this redeploys. TTM_MIN_MIB 150,000 -> 162,000 for the new pin ledger (2 x pool + PLE 48,824 + prefix 8,192 + MNBT 2,048 + the auto checkpoint store, now 40 slots of 112 MiB = 161,848 MiB), and the bats ledger guard now counts the checkpoint store because it grows with the slot count. The two env-value assertions in that seam are bumped with the values they guard. --- scripts/frank-archon.env | 49 +++++++++++++++++++---------------- scripts/serve-frank-archon.sh | 12 ++++----- tests/frank_archon_unit.bats | 11 +++++--- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/scripts/frank-archon.env b/scripts/frank-archon.env index 3adb083..2266c2c 100644 --- a/scripts/frank-archon.env +++ b/scripts/frank-archon.env @@ -46,31 +46,35 @@ FRANK_ARCHON_TP=2 # stays EXACT for single-sequence decode and any step of <= 25 rows — see # MAX_SEQS x (1 + SPEC_TOKENS) below. FRANK_ARCHON_TP_WIRE=wht6 -# Decode slots. A slot costs KV, never decode; the pool is sized to this -# contract and nothing more. At 1.8.1 the budget resolver caps the cache at -# 556,348 fp8 tokens — 2.8 sessions at 200,000 — so a THIRD full-height -# conversation answers 400 while /health and every shorter chat work fine -# (3 x 50k streams run at 77.6 t/s aggregate). Frank's trade: declare the two -# slots the engine can hold and let the freed ~1.1 GiB go to the expert plane, -# instead of advertising a third slot that only exists at shorter prompts. -FRANK_ARCHON_MAX_SEQS=2 +# Decode slots. A slot costs KV and pinned RAM, never decode: the cache target is +# exactly MAX_SEQS x MAX_MODEL_LEN and the checkpoint store asks (1 + what the +# retention policy keeps) per slot. Ordered 2026-10-21: FOUR slots over a pool of at +# least 700,000 fp8 tokens, rounding UP if the resolver cannot land it exactly. The +# target is therefore 4 x 200,000 = 800,000 tokens and HOST_POOL_MIB decides whether +# the cache gets it (expert_vram_floor, core/place/planner.cpp:100-104: the card must +# keep expert-plane-minus-host-room). The 2-slot shape was the answer to a +# 365,204-token pool that backed 1.8 full-height sessions; at this pool the third and +# fourth conversations are backed by the cache instead of over-committing it. +FRANK_ARCHON_MAX_SEQS=4 # Served window. The engine treats `kv group tokens >= max-model-len` as a HARD # limit, and models.yml contextWindow MUST quote this number (over-length is a # refusal, not a truncation). The pool is sized for seqs x maxlen, not one # sequence. FRANK_ARCHON_MAX_MODEL_LEN=200000 FRANK_ARCHON_PLACEMENT=expert_tiered -# Pinned host pool for expert rows that do not fit VRAM. Under the budget -# resolver this is ALSO the lever that decides which experts stay on the card: -# the card's leftover goes to the expert plane first and the KV cache takes the -# rest, so growing the pool moves experts off the card and shrinking it spills -# them onto VRAM the cache wanted. Under-sizing it is a boot failure (36,864 -# refused a boot). The ledger's coverage line is the referee, not this file. -# Measured, not guessed: 45,568 (PR #30) moved host_pool to 45.09 GiB and left -# vram_kv byte-identical at 4.20 GiB / 556,348 tokens on both ranks — extra -# host RAM above what the expert plane's footprint needs buys no cache. Back at -# 44,928, the size that boots clean. -FRANK_ARCHON_HOST_POOL_MIB=44928 +# Pinned host pool for expert rows that do not fit VRAM, PINNED ONCE PER RANK: the +# host must pin tp x this number, and the refusal that names it is +# core/engine_bringup.cpp:250-265. Under the budget resolver this is ALSO the KV +# lever -- the card keeps expert-plane-minus-this-room, so every MiB added moves one +# MiB of experts off the card and into the cache. Derived from the boot ledger (a +# 23.73 GiB floor at 44,928 room puts plane + 2 units at 67.60 GiB a rank), 49,152 +# drops the floor to 19.6 GiB and leaves the cache ~6.9 GiB, which is enough to back +# the whole 4 x 200,000 target; the leftover returns to the experts. Cost is paid +# twice over: pinned RAM (tp x 48 GiB, against the TTM target below) and expert +# residency -- ~3.2 GiB a card of experts now lives in host RAM and is read over the +# x8 link when the router picks it. The ledger's coverage line is the referee, not +# this file: if vram_kv comes back under 700,000 tokens, raise this 1024 and redeploy. +FRANK_ARCHON_HOST_POOL_MIB=49152 # VRAM left unclaimed per card (publisher default 96 MiB); the engine's settle # floor is built from it. FRANK_ARCHON_GPU_HEADROOM_MIB=96 @@ -131,9 +135,10 @@ FRANK_ARCHON_NGRAM_PLACEMENT=ram # RESETS pages_limit to half of RAM on every boot; below the plan the load dies # mid-pin with an anonymous allocator error. ttm-preflight.sh raises the knob # itself before refusing by name, so recovery is a boot, not a visit. -# Floor = 2 x HOST_POOL + 48,824 (static weights + arena) + PREFIX_CACHE + MNBT -# per the bats ledger test; at the 44,928 pool that computes to 148,920 MiB. -FRANK_ARCHON_TTM_MIN_MIB=150000 +# Floor = 2 x HOST_POOL + 48,824 (static weights + arena) + PREFIX_CACHE + MNBT + the +# auto checkpoint store (MAX_SEQS x 10 slots of 111.98 MiB), per the bats ledger test; +# at the 49,152 pool with 4 slots that computes to 161,848 MiB. +FRANK_ARCHON_TTM_MIN_MIB=162000 FRANK_ARCHON_TTM_TARGET_MIB=204800 # Escape hatch for experiments. Unset -> no extra argv; the serve script expands # this unquoted on purpose, so keep multi-flag values as one string. diff --git a/scripts/serve-frank-archon.sh b/scripts/serve-frank-archon.sh index d56fbdc..708214a 100755 --- a/scripts/serve-frank-archon.sh +++ b/scripts/serve-frank-archon.sh @@ -15,9 +15,9 @@ # log states every budget before anything is allocated (weights vs KV # split, the expert host pool's coverage of the worst case, the PLE pin, # the P2P answer per pair) — read it after every resize. -# Deployed shape (2026-10-19, radiance 1.8.1): TWO decode slots x 200,000 tokens, -# fp8 KV, 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache in pinned host -# RAM, PLE pinned in RAM. The per-card split between the expert slab and the KV +# Deployed shape (2026-10-21, radiance 1.8.1): FOUR decode slots x 200,000 tokens over a +# >= 700,000-token fp8 KV pool, 48 GiB/rank pinned expert pool, MTP depth 2, prefix cache +# in pinned host RAM, PLE pinned in RAM. The per-card split between the expert slab and the KV # pool is NO LONGER an operator number — 1.3.0 removed the VRAM pair and the budget # resolver derives it from headroom, the cache bound and the host pool, so the slab # size and the pool's token count are LEDGER lines. After every boot read them, plus @@ -27,9 +27,9 @@ # pooled experts are read synchronously) — and the cache's session backing. # # The root unit (scripts/frank-archon.service, User=dank, -# LimitMEMLOCK=infinity) grants the pin budget: 2 x 44 GiB expert pools + -# the 47.7 GiB PLE table + 8 GiB prefix host tier — ~147 GiB of pinned -# pages, which is ALSO why this script refuses to start when ttm's +# LimitMEMLOCK=infinity) grants the pin budget: 2 x 48 GiB expert pools + the 47.7 GiB PLE +# table + 8 GiB prefix host tier + the checkpoint store (slots x 112 MiB) — ~159 GiB of +# pinned pages, which is ALSO why this script refuses to start when ttm's # pages_limit/pages_max cap is under FRANK_ARCHON_TTM_MIN_MIB (see below). set -euo pipefail diff --git a/tests/frank_archon_unit.bats b/tests/frank_archon_unit.bats index de4ed5a..bf298d8 100755 --- a/tests/frank_archon_unit.bats +++ b/tests/frank_archon_unit.bats @@ -145,7 +145,7 @@ nosudo_bin() { # (torch.compile query_start_loc constraint, 2026-10-02); radiance booted # seqs 2 in the fp8-native era and the 2026-10-06 sweep had seqs 2 and 3 # tied inside the +-3% noise floor — slots buy concurrency, not speed. - grep -q '^FRANK_ARCHON_MAX_SEQS=2$' "$ENV_FILE" + grep -q '^FRANK_ARCHON_MAX_SEQS=4$' "$ENV_FILE" grep -q '^FRANK_ARCHON_TP=2$' "$ENV_FILE" grep -q '^FRANK_ARCHON_TP_WIRE=wht6$' "$ENV_FILE" grep -q '^FRANK_ARCHON_SPEC_TOKENS=2$' "$ENV_FILE" @@ -158,7 +158,7 @@ nosudo_bin() { # so the pool's token count is a boot-ledger line — verified live by the # capacity leg of scripts/probe-frank-archon-kv.py at every cutover. grep -q '^FRANK_ARCHON_PLACEMENT=expert_tiered$' "$ENV_FILE" - grep -q '^FRANK_ARCHON_HOST_POOL_MIB=44928$' "$ENV_FILE" + grep -q '^FRANK_ARCHON_HOST_POOL_MIB=49152$' "$ENV_FILE" grep -q '^FRANK_ARCHON_KV_DTYPE=fp8$' "$ENV_FILE" grep -q '^FRANK_ARCHON_NGRAM_PLACEMENT=ram$' "$ENV_FILE" # Prefix cache: pinned host RAM only (the disk tier was removed), and the @@ -177,8 +177,11 @@ nosudo_bin() { [ "$ttmt" -gt "$ttmm" ] pool=$(awk -F= '/^FRANK_ARCHON_HOST_POOL_MIB=/{print $2}' "$ENV_FILE") ph=$(awk -F= '/^FRANK_ARCHON_PREFIX_CACHE_MIB=/{print $2}' "$ENV_FILE") - # 2 pools + PLE 47.7 GiB (48,824 MiB) + prefix host, rounded up. - ledger=$(( 2 * pool + 48824 + ph + 2048 )) + # 2 pools + PLE 47.7 GiB (48,824 MiB) + prefix host + MNBT + the auto + # checkpoint store (MAX_SEQS x 10 slots of 112 MiB -- the store is pinned host + # memory too, and it grows with the slot count). + seqs=$(awk -F= '/^FRANK_ARCHON_MAX_SEQS=/{print $2}' "$ENV_FILE") + ledger=$(( 2 * pool + 48824 + ph + 2048 + seqs * 10 * 112 )) [ "$ttmm" -ge "$ledger" ] # The .rad size floor is positive (a truncated convert must fail fast). grep -q '^FRANK_ARCHON_RAD_MIN_MIB=' "$ENV_FILE"