From 09a0251d2dacd5d1cd1b5375b08e80fae7dc7029 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 13 Oct 2026 20:18:41 -0700 Subject: [PATCH] llm(frank-archon): claim the card's settling slack, slab 24000 -> 25600 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2 GiB/card that \`rocm-smi\` shows free is not idle: the engine's budget table calls it driver context + the 896 MiB activation arena + 96 MiB headroom, and says the rest is `throughput on the floor`. Its own settle line is the number that matters — `card 0 settled at 2.01 GiB free against 209.19 MiB (the 96.00 MiB --gpu-headroom-mib and 113.19 MiB kept for the static region)` — so 1,849 MiB above the ENGINE's floor is claimable and the real ceiling is 30,969 MiB, not the 29,120 my 1 GiB reserve invented on 2026-10-11. Shipped pair 25600/5120 = 30,720: +1,600 MiB of slab, ~458 MiB above the floor. The slab is already saturated at 24,000 (mover took all of it, 451 KiB slack), so this is the only direction left — KV is at its 3-slot floor (5,063 needed of 5,120). Named prediction, because today's two points say the slab is a THRESHOLD and not a slope (22,208 -> 3,397 t/s, 24,000 -> 3,290 t/s at 43.5k single-stream): this arm buys ~1,600 resident expert slots and ~0 prefill. The point that moves prefill is 26,071 (2026-10-11: 4,379 t/s), fundable only at two slots. Prefill moving > +-3 % here falsifies the threshold reading. Seam: the ceiling gate now derives from the engine's budget table (30,969) and is the single owner of the pair arithmetic; the duplicate I put in the shape test is removed. --- scripts/frank-archon.env | 46 +++++++++++++++++++++++------------ scripts/serve-frank-archon.sh | 8 +++--- tests/frank_archon_unit.bats | 43 +++++++++----------------------- 3 files changed, 46 insertions(+), 51 deletions(-) diff --git a/scripts/frank-archon.env b/scripts/frank-archon.env index 82f81ad..e6a6322 100644 --- a/scripts/frank-archon.env +++ b/scripts/frank-archon.env @@ -4,12 +4,13 @@ # dankserver :7770. # # SHAPE (2026-10-13): THREE decode slots x 200,000 tokens, fp8 (quantized) KV, -# with the per-card budget re-cut in the slab's favour: 24,000 MiB of resident -# experts + 5,120 MiB of pool = the measured 29,120 MiB ceiling. The pool is -# sized to the contract and nothing more (3 x 200,000 needs 5,063 MiB at fp8; -# 5,120 predicts 608,027 tokens, 1.3 % slack), because every MiB taken from the -# slab is prefill and TTFT (2026-10-11 A/B: slab -14.8 % cost prefill -29 %, -# TTFT +25 % at the full window) while KV beyond the contract buys nothing. +# with the per-card budget spent on resident experts: 25,600 MiB of slab + +# 5,120 MiB of pool = 30,720 MiB, inside the 30,969 MiB ceiling the engine's own +# budget table allows (card 32,624 MiB, settle floor 209 MiB). The pool is sized +# to the contract and nothing more (3 x 200,000 needs 5,063 MiB at fp8; 5,120 +# predicts 608,027 tokens, 1.3 % slack), because every MiB taken from the slab +# is prefill and TTFT (2026-10-11 A/B: slab -14.8 % cost prefill -29 %, TTFT +# +25 % at the full window) while KV beyond the contract buys nothing. # Consequence, deliberate: a FOURTH full-height conversation does not fit this # pair, and the engine refuses it — it never truncates. See the KV block for the # arithmetic and the measured dtype law. The fp8-native payload this file was @@ -164,11 +165,17 @@ FRANK_ARCHON_HOST_POOL_MIB=44928 # VRAM left unclaimed per card (publisher default 96 MiB). FRANK_ARCHON_GPU_HEADROOM_MIB=96 # VRAM split of claimed memory per card, MiB (0 = derive from the card). -# THE CEILING, measured on this host 2026-10-11: card 31,856 MiB, a boot -# consumes 1,710 MiB OUTSIDE both budgets (driver context + activation arena -# 990 + hardware queues 100 + headroom 96 + step staging), and the fp8 pair -# settled at 1,379 MiB free — so `vram_weights + vram_kv <= 29,120 MiB` keeps -# >= 1 GiB free. The pair below is exactly that sum. +# THE CEILING, read off this engine's own VRAM-budget table (boot 2026-10-13, +# pair 24000/5120): card 32,624 MiB; budgets claimed 29,120; `3.42 GiB is +# neither budgeted nor accounted — the driver's context, the activation arena +# and --gpu-headroom-mib come out of it; anything past those is throughput on +# the floor`; and `card 0 settled at 2.01 GiB free against 209.19 MiB (the +# 96.00 MiB --gpu-headroom-mib and 113.19 MiB kept for the static region)`. +# So the reserve above the ENGINE's floor is 2,058 - 209 = 1,849 MiB and the +# ceiling is 29,120 + 1,849 = 30,969 MiB. The 2026-10-11 bound of 29,120 +# ("31,856 - 1,710 outside both budgets - keep 1 GiB free") was MY reserve, not +# the engine's; the pair below spends the difference and lands 458 MiB above +# the engine's floor. `weights + kv <= 30,969` is the gate. # THE PRICE, and it is paid twice: on this engine idle KV VRAM is ALSO the # expert cache's spillover (`mover: 2275 slots can be lent out of the KV # cache (2.69 GiB)`, `arena: lends 740 MiB while steps are <= 256 tokens`), @@ -179,14 +186,21 @@ FRANK_ARCHON_GPU_HEADROOM_MIB=96 # exact-KV A/B measured the slab's own price (22208 vs the fp8-era 26071): # prefill -29 %, TTFT at the full window +25 %, decode inside the noise band on # 2 of 3 passes, draft acceptance UP 0.661 -> 0.740. -# 2026-10-13 (late): THREE slots, so the pool is cut to the contract and the -# freed 1,792 MiB goes to the slab (22208 -> 24000, +8.1 %). The direction the -# A/B data points: prefill is slab-limited, KV beyond the contract is not. The -# margin is thin by design (608,027 predicted tokens against the 600,000 +# 2026-10-13 (late): THREE slots, so the pool is cut to the contract (5,120) +# and every spare MiB goes to the slab: 22208 -> 24000 (the third slot's +# surplus KV), then 24000 -> 25600 (the card's settling slack). The direction +# the A/B data points: prefill is slab-limited, KV beyond the contract is not. +# The margin is thin by design (608,027 predicted tokens against the 600,000 # contract), so the boot ledger's `backs N sequences all at 200000 tokens at # once` line is the referee for THIS pair and the concurrent probe is its # regression test. -FRANK_ARCHON_VRAM_WEIGHTS_MIB=24000 +# FALSIFIABLE CLAIM for the 25,600 arm: today measured slab 22,208 -> 3,397 +# t/s and 24,000 -> 3,290 t/s at 43.5k single-stream, so below ~26 GiB the slab +# is a THRESHOLD, not a slope. Expected: +1,600 resident expert slots, ~0 +# prefill. The point that DOES move prefill is 26,071 (the 2026-10-11 fp8 arm, +# 4,379 t/s), fundable only by dropping to two slots. Prefill moving more than +# +-3 % on this arm falsifies the threshold reading. +FRANK_ARCHON_VRAM_WEIGHTS_MIB=25600 FRANK_ARCHON_VRAM_KV_MIB=5120 # KV cache QUANTIZED (fp8). This engine's flag takes ONLY bf16|fp8 — there is # ONE kv_dtype for the whole cache, so "exact K, quantized V" (llama.cpp's diff --git a/scripts/serve-frank-archon.sh b/scripts/serve-frank-archon.sh index a5725bc..8c8c9b4 100755 --- a/scripts/serve-frank-archon.sh +++ b/scripts/serve-frank-archon.sh @@ -15,10 +15,10 @@ # the P2P answer per pair) — read it after every resize. # Deployed shape (2026-10-13): THREE decode slots x 200,000 tokens, fp8 # (quantized) KV in a 5,120 MiB pool (contract 3 x 200,000 + 1.3 %) and a -# 24,000 MiB resident expert slab — the pair is the measured 29,120 MiB/card -# ceiling, spent on the slab because prefill is slab-limited. 44 GiB/rank -# pinned expert pool, MTP depth 2, prefix cache host 8 GiB + disk 64 GiB, PLE -# pinned in RAM. +# 25,600 MiB resident expert slab — 30,720 MiB of the 30,969 MiB the engine's +# own budget table makes claimable, spent on the slab because prefill is +# slab-limited. 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache host +# 8 GiB + disk 64 GiB, PLE pinned in RAM. # # The root unit (scripts/frank-archon.service, User=dank, # LimitMEMLOCK=infinity) grants the pin budget: 2 x 44 GiB expert pools + diff --git a/tests/frank_archon_unit.bats b/tests/frank_archon_unit.bats index 5d24c2b..8ce2be3 100755 --- a/tests/frank_archon_unit.bats +++ b/tests/frank_archon_unit.bats @@ -154,30 +154,6 @@ nosudo_bin() { grep -q '^FRANK_ARCHON_VRAM_WEIGHTS_MIB=[1-9]' "$ENV_FILE" grep -q '^FRANK_ARCHON_VRAM_KV_MIB=[1-9]' "$ENV_FILE" grep -q '^FRANK_ARCHON_KV_DTYPE=fp8$' "$ENV_FILE" - # Two arithmetic contracts the engine ENFORCES at request/boot time, so - # they are gates and not lore (a bad pair boots, then 400s every long - # request — the 2026-10-11 pool of 6,912 MiB at seqs 4 accepted 200k-token - # chats and refused them as "too large to ever fit"): - # 1. vram_weights + vram_kv <= the measured 29,120 MiB/card ceiling - # (31,856 card - 1,710 consumed outside both budgets - 1,024 kept free) - # 2. the pool must back seqs x max-model-len at the dtype's rate, plus - # the dtype-INDEPENDENT terms that live in the pool: ~110 MiB of - # recurrent state a sequence and 448 MiB of prefix checkpoints. - # Rates are the two boot ledgers' group tables: 7,488 B/token fp8, - # 14,144 B/token bf16 (the QSA block key stays 3.25 KiB at both). - w=$(awk -F= '/^FRANK_ARCHON_VRAM_WEIGHTS_MIB=/{print $2}' "$ENV_FILE") - kv=$(awk -F= '/^FRANK_ARCHON_VRAM_KV_MIB=/{print $2}' "$ENV_FILE") - seqs=$(awk -F= '/^FRANK_ARCHON_MAX_SEQS=/{print $2}' "$ENV_FILE") - dt=$(awk -F= '/^FRANK_ARCHON_KV_DTYPE=/{print $2}' "$ENV_FILE") - [ $(( w + kv )) -le 29120 ] - case "$dt" in - fp8) rate=7488 ;; - bf16) rate=14144 ;; - *) return 1 ;; - esac - tokens=$(( seqs * ml )) - need=$(( tokens * rate / 1048576 + 110 * seqs + 448 )) - [ "$kv" -ge "$need" ] grep -q '^FRANK_ARCHON_NGRAM_PLACEMENT=ram$' "$ENV_FILE" # Prefix cache tiers (publisher pattern; disk tier sized for 2 slots). grep -q '^FRANK_ARCHON_PREFIX_HOST_MIB=8192$' "$ENV_FILE" @@ -228,15 +204,20 @@ nosudo_bin() { } @test "the per-card VRAM pair stays inside the measured card ceiling" { - # Measured on Dankserver 2026-10-11: card 31,856 MiB, a boot consumes - # 1,710 MiB OUTSIDE both budgets (driver context, the 990 MiB activation - # arena, hardware queues, headroom, step staging) and the fp8 pair settled - # at 1,379 MiB free. Claiming more than 29,120 MiB in total is an - # allocation failure at graph capture, not a warning — so the KV pool and - # the weights slab are one budget, and this seam is the guard. + # The engine's OWN budget table, not a hand-picked reserve. Boot of + # 2026-10-13 at the pair 24000/5120: card 32,624 MiB (31.86 GiB), budgets + # claimed 29,120, `3.42 GiB is neither budgeted nor accounted — the + # driver's context, the activation arena and --gpu-headroom-mib come out of + # it; anything past those is throughput on the floor`, and `card 0 settled + # at 2.01 GiB free against 209.19 MiB (the 96.00 MiB --gpu-headroom-mib and + # 113.19 MiB kept for the static region)`. Claimable reserve above the + # engine's floor: 2,058 - 209 = 1,849 MiB, so the ceiling is 30,969 MiB. + # The 2026-10-11 bound of 29,120 ("keep >= 1 GiB free", card read as 31,856 + # MiB) was MY reserve, which the engine never asked for; the shipped pair + # spends the difference and lands ~458 MiB above the floor. w=$(awk -F= '/^FRANK_ARCHON_VRAM_WEIGHTS_MIB=/{print $2}' "$ENV_FILE") kv=$(awk -F= '/^FRANK_ARCHON_VRAM_KV_MIB=/{print $2}' "$ENV_FILE") - [ $(( w + kv )) -le 29120 ] + [ $(( w + kv )) -le 30969 ] } @test "the serve script wires the engine flags from the env" {