From 0bf26b07419080c211e2e570842768fc14a57077 Mon Sep 17 00:00:00 2001 From: Frank Date: Sun, 11 Oct 2026 01:01:58 -0700 Subject: [PATCH] llm(frank-archon): unquantized bf16 KV at 2 x 200,000 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The KV pool at the fp8 shape backed 386,904 tokens — 1.93 sessions of the served 200,000 window, so a second full-height conversation preempted to the host/disk prefix tiers. Moving the cache to bf16 (the engine's only unquantized rung; `--kv-cache-dtype` takes bf16|fp8 and has no per-tensor K/V split) doubles what a token costs: 7.3125 -> 14.625 KiB per rank (kv_attn 26.00 -> 52.00 KiB per 4-token page + kv_qsa_bkey 3.25 -> 6.50). Pool 3543 -> 6912 MiB (437k tokens, >= 2 x 200,000), and the weights slab pays for all of it: 26071 -> 22208 MiB, the pair pinned to the measured card ceiling 29,120 MiB (card 31,856 - 1,710 consumed outside both budgets - 1,024 free). The resident set that leaves VRAM spills into the pinned host pool, so 40960 -> 44928 MiB, and the prefix host tier doubles with the entry size (58.50 -> 117 KiB): 4096 -> 8192 MiB. Pinned ledger ~147 GiB, ttm floor 140000 -> 150000 (cap 200 GiB). --max-num-seqs 3 -> 2: two slots x the window is the capacity contract, and the "seqs 2 cannot boot" law is the retired tcclaviger fork's torch.compile query_start_loc constraint, not radiance's (radiance booted seqs 2 in the fp8-native era; the 2026-10-06 sweep had 2 and 3 tied inside +-3%). Bonus: the fused all-reduce is now declared over 2 x 3 = 6 rows, under the 25-row bound at which the wht6 wire is bit-exact. Cost, paid twice on this engine: idle KV VRAM is also the expert cache's spillover (lends 2.69 GiB + 740 MiB of arena), so the pool steals weight residency from the slab AND the lendable cache. Priors: 2.14 GiB of stolen weights cost 2.5% decode (MAX_MODEL_LEN 262144), the EVC ladder's other direction cost 8.9%. Prediction -3..-8% vs the fp8 control (136 t/s at 44.5k, acceptance 0.7246); the deploy boot ledger and live probes are the referee. Seams: MAX_SEQS/HOST_POOL/KV_DTYPE/PREFIX_HOST literals follow the shape, and two new invariants guard it — the KV pool must back seqs x max-model-len at the declared dtype (an engine hard limit: an over-long request is accepted then refused as too large to ever fit; the formula reproduces the fp8 boot's 386,904 to within 0.1%), and the VRAM pair must stay inside the ceiling. --- scripts/frank-archon.env | 122 ++++++++++++++++++++++++---------- scripts/serve-frank-archon.sh | 12 ++-- tests/frank_archon_unit.bats | 95 ++++++++++++++++++-------- 3 files changed, 162 insertions(+), 67 deletions(-) diff --git a/scripts/frank-archon.env b/scripts/frank-archon.env index 9d5eec5..390a4c2 100644 --- a/scripts/frank-archon.env +++ b/scripts/frank-archon.env @@ -1,8 +1,13 @@ -# frank-archon.env — radiance (StillDeadcode, Apache-2.0) serving the -# OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint at NATIVE precision -# (modelopt block-FP8 throughout, no requantisation) on BOTH R9700s (TP=2), +# frank-archon.env — radiance (StillDeadcode, Apache-2.0) serving +# ukisai/Swift1.5-Qwen3.8-Flash-Next converted to GPTQ w4 (MODEL_FILE below, +# built by scripts/convert-swift15-w4dn8.sh) on BOTH R9700s (TP=2), # dankserver :7770. # +# SHAPE (2026-10-11): 2 decode slots x 200,000 tokens with an UNQUANTIZED +# (bf16) KV cache — the pool is sized for the pair, and the weights slab pays +# for it (see the KV block). The fp8-native payload this file was written for +# is the rollback target and keeps its own trio under the fp8-native runbook. +# # 2026-10-05 cutover from FreeToken (PR #587 ROCm, one card, experts in # pinned host RAM; git tag frank-archon-freetoken-pr587-20261005 is the # pre-cutover state — its unit/env/serve trio and Dockerfile are all in the @@ -50,7 +55,8 @@ # publisher's throughput peak for this model. Stated explicitly with # --num-speculative-tokens 3. # - The fused TP all-reduce is declared over max-num-seqs x -# (1 + --num-speculative-tokens) rows; 3 x 3 = 9 rows fits either wire. +# (1 + --num-speculative-tokens) rows; 2 x 3 = 6 rows, under the 25-row +# exactness bound of the wht6 wire. # Engine pin: Codeberg tag v1.0.13 (commit 53282fa), Docker Hub tag 1.0.13, # index digest below. REJECTED UPGRADE: 1.2.0 (sha256:665775ed1d90de6dbb # 25558e07fc60d20328f1346f96f3c6e00845eb763f01fa, tag 1.2.0, commit @@ -98,7 +104,7 @@ FRANK_ARCHON_PORT=7770 # Radiance derives the reasoning/tool-call reply format from it. FRANK_ARCHON_TEMPLATE=/home/dank/serverAppConfigs/scripts/qwen38-froggeric-v22.5.jinja -# --- engine shape (first-boot conservative; the boot ledger arbitrates) -- +# --- engine shape (the boot ledger arbitrates every number below) --------- # Tensor parallel: BOTH R9700s. The board swap (2026-09-20, ProArt B850 # x8/x8 on root ports) made collectives legal on this host; radiance's TP # wire is libr4d's own, not RCCL. @@ -107,55 +113,99 @@ FRANK_ARCHON_TP=2 # +5.6-6.3% prefill, ~-1.3% 8-stream decode at TP2). min-kb stays the # engine default. FRANK_ARCHON_TP_WIRE=wht6 -# Decode slots. 2 is Frank's first-boot value ("use parallelism = 2 to -# start"); raise only after a real-traffic look at slot occupancy and the -# KV pool's headroom line. The recurrent-state cost scales with it (54 -# MiB/copy/card, 2 copies with speculation). -FRANK_ARCHON_MAX_SEQS=3 +# Decode slots: TWO, so the served window is a CAPACITY CONTRACT — two +# concurrent conversations of --max-model-len fit the KV pool (see the KV +# block below). The "seqs 2 cannot boot" law is NOT radiance's: it was the +# tcclaviger vLLM fork's torch.compile shape specialization on +# query_start_loc (ConstraintViolationError, measured 2026-10-02, retired +# with that runtime). Radiance booted seqs 2 at the fp8-native first boot, +# and the 2026-10-06 decode sweep measured seqs 2 and 3 statistically tied +# (30.9-33.1 t/s, inside the +-3% noise floor). Side benefit: the fused TP +# all-reduce is declared over seqs x (1 + spec) = 6 rows, under the 25-row +# bound at which the wht6 wire stays EXACT (single-sequence decode and any +# step of <= 25 rows is bit-exact; only busier steps are lossy). +FRANK_ARCHON_MAX_SEQS=2 # Served window. Native is 262,144; 200,000 is the publisher's production # value (compose) and what models.yml's contextWindow must quote — a longer -# prompt is refused by the engine, never truncated. +# prompt is refused by the engine, never truncated. The engine treats +# `kv group tokens >= max-model-len` as a HARD limit ("ONE request longer +# than N tokens is accepted by the API and then refused as too large to ever +# fit — this is a hard limit, not a backing ratio"), so the pool is sized +# for seqs x maxlen, not for one sequence. FRANK_ARCHON_MAX_MODEL_LEN=200000 FRANK_ARCHON_PLACEMENT=expert_tiered -# Pinned host pool for the expert rows that do not fit VRAM. Native FP8 -# experts are 112.8 GiB total (56.4 GiB per rank slice) vs the 4-bit -# publisher shape's ~29 GiB — this pool is the cost of native precision. -# The boot ledger prints the pool's coverage ("the pool backs N% of the -# worst case ... M full-length sessions"); resize from that line, not from -# arithmetic. -FRANK_ARCHON_HOST_POOL_MIB=40960 +# Pinned host pool for the expert rows that do not fit VRAM. vram_weights and +# this pool serve the per-rank resident set TOGETHER: at the fp8-KV pair +# (26071/3543) the ledger showed the VRAM slab filled to 259 KiB of slack, +# i.e. the pair was exactly sized. Cutting vram-weights by 3.8 GiB to pay for +# the bf16 KV pool SPILLS ~3.8 GiB/rank here, so the pool moves with it — +# under-sizing it is a boot failure (36864 refused the boot in the fp8 era). +# The boot ledger's coverage line remains the referee, not this arithmetic. +FRANK_ARCHON_HOST_POOL_MIB=44928 # VRAM left unclaimed per card (publisher default 96 MiB). FRANK_ARCHON_GPU_HEADROOM_MIB=96 # VRAM split of claimed memory per card, MiB (0 = derive from the card). -# Engine 1.2.0 REMOVED --expert-vs-cache-ratio; these two knobs replace -# it. The values transcribe the live 1.0.13 boot ledger at ratio 0.82: -# vram_weights 25.46 GiB, vram_kv 3.46 GiB per card -> 26071 / 3543. -# Bracket arms scale the KV term linearly in (1-r) from the 0.82 anchor -# with weights fixed: ratio 0.90 -> KV 1968, ratio 0.95 -> KV 984. The -# boot ledger arbitrates; re-derive after any payload change. -FRANK_ARCHON_VRAM_WEIGHTS_MIB=26071 -FRANK_ARCHON_VRAM_KV_MIB=3543 -# KV cache in FP8 (7.32 KiB/token at this model's 12 full-attention layers, -# 2 KV heads, 256 head_dim, K+V). -FRANK_ARCHON_KV_DTYPE=fp8 +# THE CEILING, measured on this host 2026-10-11: card 31,856 MiB, a boot +# consumes 1,710 MiB OUTSIDE both budgets (driver context + activation arena +# 990 + hardware queues 100 + headroom 96 + step staging), and the fp8 pair +# settled at 1,379 MiB free — so `vram_weights + vram_kv <= 29,120 MiB` keeps +# >= 1 GiB free. The pair below is exactly that sum. +# THE PRICE, and it is paid twice: on this engine idle KV VRAM is ALSO the +# expert cache's spillover (`mover: 2275 slots can be lent out of the KV +# cache (2.69 GiB)`, `arena: lends 740 MiB while steps are <= 256 tokens`), +# so a bigger KV pool steals weight residency from the slab AND from the +# lendable cache. Priors on this card/model: stealing 2.14 GiB of weights for +# a bigger pool (MAX_MODEL_LEN 262144) cost 2.5% decode; the EVC ladder's +# other direction (0.82 -> 0.84, +2.5 GiB weights) cost 8.9%. Expect -3..-8% +# vs the fp8 control (136 t/s median at 44.5k depth, acceptance 0.7246, +# ~190 t/s on a live 512-token probe); re-tune from a boot ledger + probe, +# never from that estimate. +FRANK_ARCHON_VRAM_WEIGHTS_MIB=22208 +FRANK_ARCHON_VRAM_KV_MIB=6912 +# KV cache UNQUANTIZED (bf16). This engine's flag takes ONLY bf16|fp8 — there +# is ONE kv_dtype for the whole cache, so "exact K, quantized V" (llama.cpp's +# --cache-type-k/--cache-type-v pair, which is what frank-zealot runs) is +# UNSERVABLE here: the rung below bf16 is fp8 on both halves. +# Measured per-rank cost, read out of the boot ledger's group table: +# kv_attn 13 layers, 4-token blocks, page 26.00 KiB (fp8) -> 52.00 KiB +# kv_qsa_bkey 13 layers, 4-token blocks, page 3.25 KiB (fp8) -> 6.50 KiB +# => 7.3125 KiB/token fp8, 14.625 KiB/token bf16 +# The dtype-INDEPENDENT terms also live in the pool: kv_gdn_state 54 MiB x 2 +# copies per sequence (+ conv/tail/ple ~= 110 MiB a sequence) and the prefix +# checkpoint region 8 x 55.99 = 448 MiB. So: +# pool_mib >= tokens x 14.625 KiB + 110 x seqs + 448 +# 6912 -> (6912 - 220 - 448) x 1024 / 14.625 = 437,320 tokens >= 2 x 200,000 +# (the fp8 pool of 3543 gave 386,904 — it never held 2 x 200k, it held 1.93 +# sessions, and a third long conversation preempted to the host/disk tiers). +# Secondary cost: bf16 doubles the prefix-cache entry (58.50 -> 117 KiB), so +# the host tier below doubles too. +FRANK_ARCHON_KV_DTYPE=bf16 # Spec decode: the checkpoint's own MTP head at the publisher's depth-3 # operating point (probe() returns 3 when mtp_num_hidden_layers > 0; the # flag states it so the ledger records the intent). FRANK_ARCHON_SPEC_TOKENS=2 FRANK_ARCHON_MAX_BATCHED_TOKENS=2048 # Prefix cache tiers (publisher pattern; disk tier sized for 2 slots). -FRANK_ARCHON_PREFIX_HOST_MIB=4096 +# The host tier is sized in ENTRIES, and one entry doubles with the cache +# dtype (58.50 KiB at fp8 -> 117 KiB at bf16, per the ledger's `kv tier +# transfer` line): 4096 MiB at bf16 parks ~1.4 sessions of 200k where it +# parked ~2.9 at fp8. Doubled to keep the capacity, at RAM the host has +# (104 GiB available with the service loaded). +FRANK_ARCHON_PREFIX_HOST_MIB=8192 FRANK_ARCHON_PREFIX_DISK_MIB=65536 # Frank's pin law: the PLE n-gram table (47.7 GiB) serves from pinned host # RAM the cards read. `ram` REFUSES the start if the memlock limit is too # small — the unit's LimitMEMLOCK=infinity is the other half of this law. FRANK_ARCHON_NGRAM_PLACEMENT=ram -# TTM preflight: the pinned ledger (2 x 40 GiB pools + 47.7 GiB PLE + 4 GiB -# prefix host) is ~132 GiB. ttm's pages_limit/pages_max cap the GPU-mappable -# pinned pages; the current cap (32,252,917 pages = ~123 GiB) is BELOW the -# plan, so the serve script refuses to start until both are raised. This is -# the floor it enforces (in MiB), with ~8 GiB of margin. -FRANK_ARCHON_TTM_MIN_MIB=140000 +# TTM preflight: the pinned ledger (2 x 44 GiB pools + 47.7 GiB PLE + 8 GiB +# prefix host = ~147 GiB) rides TTM's shared pages, capped by ttm's +# pages_limit (the current kernel exposes NO pages_max; the cap function +# takes the min over the knobs that exist). Below the plan the load dies +# mid-pin with an allocator error instead of this named refusal — raise the +# knob first (runtime sysfs write; kernel cmdline fallback + one reboot only +# if the file is read-only), see the runbook. Cap raised 2026-10-05 to +# 52,428,800 pages = 200 GiB, which still clears this floor. +FRANK_ARCHON_TTM_MIN_MIB=150000 # Escape hatch for boot-ladder experiments. Unset -> no extra argv. Keep # multi-flag values as one string; the serve script expands it unquoted on # purpose. diff --git a/scripts/serve-frank-archon.sh b/scripts/serve-frank-archon.sh index c617f9e..7d5f0cc 100755 --- a/scripts/serve-frank-archon.sh +++ b/scripts/serve-frank-archon.sh @@ -12,13 +12,15 @@ # there is no per-boot VRAM planner to second-guess: the boot ledger in the # log states every budget before anything is allocated (weights vs KV # split, the expert host pool's coverage of the worst case, the PLE pin, -# the P2P answer per pair) — read it after every resize. First-boot shape: -# 2 decode slots (Frank's call), 40 GiB/rank pinned expert pool, fp8 KV, -# MTP depth 3, prefix cache host 4 GiB + disk 64 GiB, PLE pinned in RAM. +# the P2P answer per pair) — read it after every resize. +# Deployed shape (2026-10-11): 2 decode slots x 200,000 tokens, bf16 +# (unquantized) KV in a 6,912 MiB pool, the weights slab cut to 22,208 MiB to +# pay for it, 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache host +# 8 GiB + disk 64 GiB, PLE pinned in RAM. # # The root unit (scripts/frank-archon.service, User=dank, -# LimitMEMLOCK=infinity) grants the pin budget: 2 x 40 GiB expert pools + -# the 47.7 GiB PLE table + 4 GiB prefix host tier — ~132 GiB of pinned +# LimitMEMLOCK=infinity) grants the pin budget: 2 x 44 GiB expert pools + +# the 47.7 GiB PLE table + 8 GiB prefix host tier — ~147 GiB of pinned # pages, which is ALSO why this script refuses to start when ttm's # pages_limit/pages_max cap is under FRANK_ARCHON_TTM_MIN_MIB (see below). set -euo pipefail diff --git a/tests/frank_archon_unit.bats b/tests/frank_archon_unit.bats index 36283eb..899b7a3 100755 --- a/tests/frank_archon_unit.bats +++ b/tests/frank_archon_unit.bats @@ -2,21 +2,23 @@ # Seam: frank-archon — the Dankserver LLM endpoint on :7770. # # This file asserts the RADIANCE shape (StillDeadcode engine, digest-pinned -# stilldeadcode/radiance image, BOTH R9700s TP=2 on the wht6 wire, the -# OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 at native block-FP8 precision in a -# rad-convert container). It replaces the FreeToken seam of 2026-10-05 -# (git tag frank-archon-freetoken-pr587-20261005) and pins what flags -# alone do not say: the served window clears the harness worst case, the -# pinned-host ledger has a ttm pages cap floor and the serve script -# ENFORCES it, the served window is what models.yml must quote, the host- -# networking contract (no --publish), the image gate on a name@sha256 ref, -# the .rad payload gate with its size floor, the chat-template override -# mount, the self-healing law (the run returns before the engine is up, so -# the serve script must wait on the container for Restart=always to mean -# anything), the deployer's false-green MainPID check, the vendored hfget -# fetch path (no token for this ungated repo), and the client contract -# (reasoning auto -> reasoning_content; models.yml maps the field on the -# harness side, not here). +# stilldeadcode/radiance image, BOTH R9700s TP=2 on the wht6 wire, serving +# the Swift1.5 Qwen3.8-Flash-Next GPTQ-w4 .rad built by +# scripts/convert-swift15-w4dn8.sh). It replaces the FreeToken seam of +# 2026-10-05 (git tag frank-archon-freetoken-pr587-20261005) and pins what +# flags alone do not say: the served window clears the harness worst case, +# the KV pool backs seqs x max-model-len at the declared cache dtype (an +# engine HARD limit, not a ratio), the per-card VRAM pair stays inside the +# measured card ceiling, the pinned-host ledger has a ttm pages cap floor and +# the serve script ENFORCES it, the served window is what models.yml must +# quote, the host-networking contract (no --publish), the image gate on a +# name@sha256 ref, the .rad payload gate with its size floor, the +# chat-template override mount, the self-healing law (the run returns before +# the engine is up, so the serve script must wait on the container for +# Restart=always to mean anything), the deployer's false-green MainPID check, +# the vendored hfget fetch path (no token for this ungated repo), and the +# client contract (reasoning auto -> reasoning_content; models.yml maps the +# field on the harness side, not here). load helpers @@ -118,25 +120,30 @@ TEMPLATE="$REPO_ROOT/scripts/qwen38-froggeric-v22.5.jinja" grep -q '^FRANK_ARCHON_MAX_MODEL_LEN=200000$' "$ENV_FILE" ml=$(awk -F= '/^FRANK_ARCHON_MAX_MODEL_LEN=/{print $2}' "$ENV_FILE") [ "$ml" -ge 177356 ] - # Decode-sweep shape (2026-10-06): 3 slots, spec depth 2 — the measured - # optimum (33.06 t/s). The fused TP all-reduce is declared over - # seqs x (1 + spec) rows = 9. - grep -q '^FRANK_ARCHON_MAX_SEQS=3$' "$ENV_FILE" + # Concurrency: 2 slots x the served window is the capacity contract, and + # the fused TP all-reduce is declared over seqs x (1 + spec) = 6 rows, + # under the 25-row bound at which the wht6 wire stays bit-exact. The + # "seqs 2 cannot boot" law belongs to the retired tcclaviger vLLM fork + # (torch.compile query_start_loc constraint, 2026-10-02); radiance booted + # seqs 2 in the fp8-native era and the 2026-10-06 sweep had seqs 2 and 3 + # tied inside the +-3% noise floor. + grep -q '^FRANK_ARCHON_MAX_SEQS=2$' "$ENV_FILE" grep -q '^FRANK_ARCHON_TP=2$' "$ENV_FILE" grep -q '^FRANK_ARCHON_TP_WIRE=wht6$' "$ENV_FILE" grep -q '^FRANK_ARCHON_SPEC_TOKENS=2$' "$ENV_FILE" - # Native-precision economics: the expert pool is per rank (56.4 GiB - # slice per rank at TP2), the PLE ngram table is pinned in host RAM per - # Frank's law, KV is fp8, the ratio is the publisher's production - # value. + # expert_tiered placement: vram_weights + the pinned host pool serve the + # per-rank resident set TOGETHER, so cutting the VRAM slab to pay for the + # KV pool moves the pool with it. KV is UNQUANTIZED bf16 (the engine takes + # only bf16|fp8 — no per-tensor K/V split), the PLE n-gram table is pinned + # in host RAM per Frank's law. grep -q '^FRANK_ARCHON_PLACEMENT=expert_tiered$' "$ENV_FILE" - grep -q '^FRANK_ARCHON_HOST_POOL_MIB=40960$' "$ENV_FILE" + grep -q '^FRANK_ARCHON_HOST_POOL_MIB=44928$' "$ENV_FILE" grep -q '^FRANK_ARCHON_VRAM_WEIGHTS_MIB=[1-9]' "$ENV_FILE" grep -q '^FRANK_ARCHON_VRAM_KV_MIB=[1-9]' "$ENV_FILE" - grep -q '^FRANK_ARCHON_KV_DTYPE=fp8$' "$ENV_FILE" + grep -q '^FRANK_ARCHON_KV_DTYPE=bf16$' "$ENV_FILE" grep -q '^FRANK_ARCHON_NGRAM_PLACEMENT=ram$' "$ENV_FILE" # Prefix cache tiers (publisher pattern; disk tier sized for 2 slots). - grep -q '^FRANK_ARCHON_PREFIX_HOST_MIB=4096$' "$ENV_FILE" + grep -q '^FRANK_ARCHON_PREFIX_HOST_MIB=8192$' "$ENV_FILE" grep -q '^FRANK_ARCHON_PREFIX_DISK_MIB=65536$' "$ENV_FILE" # The pinned ledger (2 x pool + 47.7 GiB PLE + prefix host) must be # under the ttm pages cap, and the env states the floor the serve @@ -154,6 +161,42 @@ TEMPLATE="$REPO_ROOT/scripts/qwen38-froggeric-v22.5.jinja" [ "$(awk -F= '/^FRANK_ARCHON_RAD_MIN_MIB=/{print $2}' "$ENV_FILE")" -gt 0 ] } +@test "the KV pool backs seqs x max-model-len at the declared cache dtype" { + # The engine's own hard limit: a request longer than the pool's token + # count is "accepted by the API and then refused as too large to ever fit + # -- this is a hard limit, not a backing ratio". Rates measured from the + # live boot ledger's group table, per rank: kv_attn (13 layers) is a + # 26.00 KiB page per 4 tokens at fp8 and kv_qsa_bkey (13 layers) 3.25 KiB + # -> 7,488 B/token, and bf16 doubles both -> 14,976 B/token. The terms + # that do NOT follow the dtype also live in the pool: ~110 MiB a sequence + # of recurrent state (kv_gdn_state 54 MiB x 2 copies + conv/tail/ple) and + # 448 MiB of prefix-cache checkpoints (8 x 55.99 MiB). Verified against + # the fp8 boot it replaces: 3543 - 330 - 448 = 2765 MiB -> 387,240 tokens, + # ledger said 386,904. + kv=$(awk -F= '/^FRANK_ARCHON_VRAM_KV_MIB=/{print $2}' "$ENV_FILE") + seqs=$(awk -F= '/^FRANK_ARCHON_MAX_SEQS=/{print $2}' "$ENV_FILE") + maxlen=$(awk -F= '/^FRANK_ARCHON_MAX_MODEL_LEN=/{print $2}' "$ENV_FILE") + case "$(awk -F= '/^FRANK_ARCHON_KV_DTYPE=/{print $2}' "$ENV_FILE")" in + fp8) bpt=7488 ;; + bf16) bpt=14976 ;; + *) fail "unknown FRANK_ARCHON_KV_DTYPE" ;; + esac + tokens=$(( (kv - 110 * seqs - 448) * 1048576 / bpt )) + [ "$tokens" -ge $(( seqs * maxlen )) ] +} + +@test "the per-card VRAM pair stays inside the measured card ceiling" { + # Measured on Dankserver 2026-10-11: card 31,856 MiB, a boot consumes + # 1,710 MiB OUTSIDE both budgets (driver context, the 990 MiB activation + # arena, hardware queues, headroom, step staging) and the fp8 pair settled + # at 1,379 MiB free. Claiming more than 29,120 MiB in total is an + # allocation failure at graph capture, not a warning — so the KV pool and + # the weights slab are one budget, and this seam is the guard. + w=$(awk -F= '/^FRANK_ARCHON_VRAM_WEIGHTS_MIB=/{print $2}' "$ENV_FILE") + kv=$(awk -F= '/^FRANK_ARCHON_VRAM_KV_MIB=/{print $2}' "$ENV_FILE") + [ $(( w + kv )) -le 29120 ] +} + @test "the serve script wires the engine flags from the env" { grep -q -- '--model "/models/$FRANK_ARCHON_MODEL_FILE"' "$SERVE" grep -q -- '--served-model-name "$FRANK_ARCHON_SERVED_NAME"' "$SERVE"