From 9de58ed6fd0f992d980c87e33e84dad50df950a1 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 19 Oct 2026 10:18:24 -0700 Subject: [PATCH] feat(frank-archon): move the endpoint to radiance 1.8.1 and its budget model The decode stall is step contention (measured: 18.7 ms decode-only steps against 317 ms steps that carry a prefill chunk, decode at 13 t/s median while prefill runs), and the two engine knobs that fix it arrived after this endpoint's pin. Moving to 1.8.1 (2026-10-18) brings both, and brings two CLI breaks with them: - --vram-weights-mib / --vram-kv-mib were REMOVED at 1.3.0. The per-card split between the expert slab and the KV pool is now derived by the engine's budget resolver from --gpu-headroom-mib, the cache bound (--max-model-len, --max-num-seqs) and --host-pool-mib, and the cache may lend idle blocks to the expert slab. The slab size and the pool's token count are boot-ledger lines from here on, so the capacity contract (3 x 200,000 fp8) is verified live by the probe's capacity leg instead of computed from an env number. - The prefix cache has no disk tier any more: --prefix-cache-dir and --prefix-cache-disk-mib are refusals and --prefix-cache-host-mib is --prefix-cache-mib. 64 GiB of parked KV on NVMe becomes re-prefill, so prefix_hit_rate is the thing to watch after this boot. env: image re-pinned to the 1.8.1 digest; the pair and the disk tier dropped; FRANK_ARCHON_PREFIX_CACHE_MIB replaces the host/disk pair; new FRANK_ARCHON_BUSY_PREFILL_CHUNK=auto (explicit, because auto vs off is the difference between decode waiting one chunk and waiting a whole step) and FRANK_ARCHON_PREFILL_INTERLEAVE=0 (opt-in through a guarded argv array). serve: argv follows the new CLI, /data mount and the pre-created kvcache path go with the disk tier, and the retired env keys are no longer required (they would abort ExecStart). autoresearch-frank-archon.sh takes the same CLI, with a cold prefill now coming from --prefix-cache-mib 0 instead of a per-run disk directory. run-swift15-bf16.sh is untouched: it pins its own engines by design. tests: the two budget-arithmetic seams asserted knobs that no longer exist and are replaced by one seam that asserts the retired knobs are ABSENT from env and argv - a stale line is a boot refusal, not a no-op. The container-contract seam now asserts the tune mount and the absence of /data. --- scripts/autoresearch-frank-archon.sh | 21 +++--- scripts/frank-archon.env | 61 +++++++++++----- scripts/serve-frank-archon.sh | 51 +++++++------- tests/frank_archon_unit.bats | 102 +++++++++++---------------- 4 files changed, 117 insertions(+), 118 deletions(-) diff --git a/scripts/autoresearch-frank-archon.sh b/scripts/autoresearch-frank-archon.sh index ecb5193..1648b8e 100755 --- a/scripts/autoresearch-frank-archon.sh +++ b/scripts/autoresearch-frank-archon.sh @@ -29,10 +29,10 @@ # The scratch mirrors the production container contract exactly (serve-frank- # primary.sh): --network host, --init, seccomp=unconfined (io_uring for the # n-gram table), memlock unbounded (the ~132 GiB pinned ledger), /dev/kfd + -# /dev/dri, the .rad ro, the state dir rw, the tune cache mount. The ONLY -# differences: the container name (arbench, so prod's identity is untouched) -# and a FRESH prefix-cache disk tier (arbench/, wiped per run) so the cold -# prefill is cold — the prod tier at kvcache/frank-archon is left intact. +# /dev/dri, the .rad ro, the tune cache mount. The ONLY differences: the +# container name (arbench, so prod's identity is untouched) and the prefix cache +# switched OFF (--prefix-cache-mib 0), so every prefill is cold — the tier is +# host RAM now, there is no per-run disk directory to wipe. set -uo pipefail ENV_FILE="${1:-/home/dank/serverAppConfigs/scripts/frank-archon.env}" @@ -60,9 +60,8 @@ set -a; . "$ENV_FILE"; set +a sudo systemctl stop frank-archon >/dev/null 2>&1 || true podman rm -f "$SCRATCH" >/dev/null 2>&1 || true -# --- fresh prefix-cache tier for a cold prefill (prod's tier untouched) ------ -FRESH_KV="$FRANK_ARCHON_STATE_DIR/kvcache/$SCRATCH" -rm -rf "$FRESH_KV"; mkdir -p "$FRESH_KV" +# --- the prefix cache is switched off below, so there is no scratch KV tree --- +# to wipe; the cold prefill comes from --prefix-cache-mib 0. mkdir -p "$FRANK_ARCHON_STATE_DIR/tune" # --- boot the scratch: the production run line, scratch name + fresh KV ------ @@ -73,7 +72,6 @@ podman run -d --name "$SCRATCH" \ --ulimit memlock=-1:-1 \ --device /dev/kfd --device /dev/dri \ -v "$FRANK_ARCHON_MODEL_DIR":/models:ro \ - -v "$FRANK_ARCHON_STATE_DIR":/data \ -v "$FRANK_ARCHON_STATE_DIR/tune":/opt/radiance/share/radiance/tune \ -v "$FRANK_ARCHON_TEMPLATE":/templates/template.jinja:ro \ "$FRANK_ARCHON_IMAGE" \ @@ -87,12 +85,9 @@ podman run -d --name "$SCRATCH" \ --placement "$FRANK_ARCHON_PLACEMENT" \ --host-pool-mib "$FRANK_ARCHON_HOST_POOL_MIB" \ --gpu-headroom-mib "$FRANK_ARCHON_GPU_HEADROOM_MIB" \ - --vram-weights-mib "$FRANK_ARCHON_VRAM_WEIGHTS_MIB" \ - --vram-kv-mib "$FRANK_ARCHON_VRAM_KV_MIB" \ --kv-cache-dtype "$FRANK_ARCHON_KV_DTYPE" \ - --prefix-cache-host-mib "$FRANK_ARCHON_PREFIX_HOST_MIB" \ - --prefix-cache-dir "/data/kvcache/$SCRATCH" \ - --prefix-cache-disk-mib "$FRANK_ARCHON_PREFIX_DISK_MIB" \ + --prefix-cache-mib 0 \ + --busy-prefill-chunk "${FRANK_ARCHON_BUSY_PREFILL_CHUNK:-auto}" \ --ngram-placement "$FRANK_ARCHON_NGRAM_PLACEMENT" \ --num-speculative-tokens "$FRANK_ARCHON_SPEC_TOKENS" \ --max-num-batched-tokens "$FRANK_ARCHON_MAX_BATCHED_TOKENS" \ diff --git a/scripts/frank-archon.env b/scripts/frank-archon.env index 0d0885a..2d2aae4 100644 --- a/scripts/frank-archon.env +++ b/scripts/frank-archon.env @@ -7,10 +7,15 @@ # only the reason a reader editing THIS line cannot recover from the page. # Where the two disagree, this file wins on values; the boot ledger arbitrates # the engine shape. -FRANK_ARCHON_IMAGE=stilldeadcode/radiance@sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e -# 1.0.13 is pinned by digest on purpose: 1.2.0 is a DEAD END on this payload -# (acceptance 0.7123 vs 0.7549 — the page's Dead ends section names the suspect). -# The .rad file mounted under /models; /data holds the prefix-cache disk tier. +FRANK_ARCHON_IMAGE=stilldeadcode/radiance@sha256:d590e300234bf6d2f942a0b8d456d183153fd808f28820e6c2fa69c9389178ed +# 1.8.1 (2026-10-18), pinned by digest. Two CLI breaks sit between it and the +# 1.0.13 line this endpoint ran for two eras, and both are boot refusals, not +# warnings: --vram-weights-mib/--vram-kv-mib were REMOVED at 1.3.0 (the card is +# now split by the engine's budget resolver — see below), and the prefix cache +# lost its disk tier. Draft acceptance has to be re-scored on any pin past 1.2.0: +# 1.2.0's MoE weight rotation measured 0.7123 against the 0.7549 control, and the +# suspect is untangled (the page's Dead ends section names it). +# The .rad file mounts under /models; the tune cache mounts separately below. # The model dir holds ONLY .rad files — the HF download and staging stay beside it. FRANK_ARCHON_MODEL_DIR=/home/dank/models/rad FRANK_ARCHON_MODEL_FILE=qwen3.8-next-flash-fp8-iq4r-moe.rad @@ -52,21 +57,25 @@ FRANK_ARCHON_MAX_SEQS=3 # sequence. FRANK_ARCHON_MAX_MODEL_LEN=200000 FRANK_ARCHON_PLACEMENT=expert_tiered -# Pinned host pool for expert rows that do not fit VRAM. It moves WITH -# VRAM_WEIGHTS: the pair and this pool serve the per-rank resident set together, -# so cutting the slab spills here — under-sizing it is a boot failure (36,864 +# Pinned host pool for expert rows that do not fit VRAM. Under the budget +# resolver this is ALSO the lever that decides which experts stay on the card: +# the card's leftover goes to the expert plane first and the KV cache takes the +# rest, so growing the pool moves experts off the card and shrinking it spills +# them onto VRAM the cache wanted. Under-sizing it is a boot failure (36,864 # refused a boot). The ledger's coverage line is the referee, not this file. FRANK_ARCHON_HOST_POOL_MIB=44928 # VRAM left unclaimed per card (publisher default 96 MiB); the engine's settle # floor is built from it. FRANK_ARCHON_GPU_HEADROOM_MIB=96 -# ONE budget per card: VRAM_WEIGHTS + VRAM_KV <= 30,969 MiB, the ceiling the -# engine's own budget table prints (derivation and the slab/pool price: the -# page's VRAM laws). Idle KV VRAM is ALSO the expert cache's spillover, so a -# bigger pool pays twice — which is why the pool sits at the contract and the -# slab takes the rest. -FRANK_ARCHON_VRAM_WEIGHTS_MIB=25600 -FRANK_ARCHON_VRAM_KV_MIB=5120 +# There is no VRAM pair any more. --vram-weights-mib/--vram-kv-mib were removed +# at 1.3.0: the engine takes capacity - already-held - GPU_HEADROOM - arena, +# puts the static (non-expert) weights off the top, then divides the rest between +# the expert plane and the KV cache — experts first when the cache has no idle +# blocks to lend, and never less than ONE MAX_MODEL_LEN sequence for the cache. +# What moves the split from here is HOST_POOL_MIB (which experts stay resident), +# MAX_MODEL_LEN/MAX_SEQS (the cache bound) and GPU_HEADROOM_MIB. The slab size +# the prefill-slope law cares about is a boot-ledger line now, not a knob: read +# it after every resize instead of asserting it here. # fp8 is the ONLY rung below bf16 on this engine: one kv_dtype for the whole # cache, so llama.cpp's exact-K/quantized-V split (frank-zealot) is unservable # here. bf16 costs 1.89x fp8, not 2x — the QSA block-key plane is dtype-fixed. @@ -75,11 +84,25 @@ FRANK_ARCHON_KV_DTYPE=fp8 # 2, and 3 x (1+2) = 9 rows stays inside the wht6 25-row exactness bound. FRANK_ARCHON_SPEC_TOKENS=2 FRANK_ARCHON_MAX_BATCHED_TOKENS=2048 -# Prefix cache tiers. The host tier is sized in ENTRIES and one entry tracks the -# cache dtype (58.50 KiB at fp8), so a dtype change re-sizes the parked set -# without touching these numbers. -FRANK_ARCHON_PREFIX_HOST_MIB=8192 -FRANK_ARCHON_PREFIX_DISK_MIB=65536 +# Step sharing. A prefill chunk that shares a step with decoding sequences holds +# the device for the chunk's whole time, and every decode row in that step waits +# that long for its next token (the census: [[LLM Performance Diagnosis]] §9). +# `auto` leaves each decode row behind the chunk a row plus its draft window and +# keeps a share for whatever is waiting to be admitted; a number pins the chunk's +# cap; `off` is the pre-1.4 behaviour this endpoint ran for two eras, where the +# chunk took the step and admission never ran. Interleave inserts N decode-only +# steps after a chunk step taken beside decoders: it protects their cadence and +# holds a waiting request out of those steps, so it stays 0 until the post-migration +# decode-vs-prefill buckets say it is wanted. +FRANK_ARCHON_BUSY_PREFILL_CHUNK=auto +FRANK_ARCHON_PREFILL_INTERLEAVE=0 +# Prefix cache: pinned host memory only. The disk tier was removed outright +# (--prefix-cache-dir and --prefix-cache-disk-mib are refusals), so what used to +# be restored from NVMe is now re-prefilled. The tier is sized in ENTRIES and one +# entry tracks the cache dtype (58.50 KiB at fp8). Watch prefix_hit_rate after +# this change: a drop is extra prefill, and prefill is exactly the traffic that +# costs decode its cadence. +FRANK_ARCHON_PREFIX_CACHE_MIB=8192 # Frank's pin law: the PLE n-gram table serves from pinned host RAM the cards # read. `ram` REFUSES the start if memlock is too small — the unit's # LimitMEMLOCK=infinity is the other half of this law. diff --git a/scripts/serve-frank-archon.sh b/scripts/serve-frank-archon.sh index d7bc77e..fde4993 100755 --- a/scripts/serve-frank-archon.sh +++ b/scripts/serve-frank-archon.sh @@ -15,12 +15,13 @@ # log states every budget before anything is allocated (weights vs KV # split, the expert host pool's coverage of the worst case, the PLE pin, # the P2P answer per pair) — read it after every resize. -# Deployed shape (2026-10-13): THREE decode slots x 200,000 tokens, fp8 -# (quantized) KV in a 5,120 MiB pool (contract 3 x 200,000 + 1.3 %) and a -# 25,600 MiB resident expert slab — 30,720 MiB of the 30,969 MiB the engine's -# own budget table makes claimable, spent on the slab because prefill is -# slab-limited. 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache host -# 8 GiB + disk 64 GiB, PLE pinned in RAM. +# Deployed shape (2026-10-19, radiance 1.8.1): THREE decode slots x 200,000 +# tokens, fp8 KV, 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache in +# pinned host RAM, PLE pinned in RAM. The per-card split between the expert slab +# and the KV pool is NO LONGER an operator number — 1.3.0 removed the VRAM pair +# and the budget resolver derives it from headroom, the cache bound and the host +# pool, so the slab size and the pool's token count are LEDGER lines: after every +# boot read them out of the log and check the pool still backs 3 x 200,000. # # The root unit (scripts/frank-archon.service, User=dank, # LimitMEMLOCK=infinity) grants the pin budget: 2 x 44 GiB expert pools + @@ -56,13 +57,11 @@ set +a : "${FRANK_ARCHON_PLACEMENT:?FRANK_ARCHON_PLACEMENT must be set}" : "${FRANK_ARCHON_HOST_POOL_MIB:?FRANK_ARCHON_HOST_POOL_MIB must be set}" : "${FRANK_ARCHON_GPU_HEADROOM_MIB:?FRANK_ARCHON_GPU_HEADROOM_MIB must be set}" -: "${FRANK_ARCHON_VRAM_WEIGHTS_MIB:?FRANK_ARCHON_VRAM_WEIGHTS_MIB must be set}" -: "${FRANK_ARCHON_VRAM_KV_MIB:?FRANK_ARCHON_VRAM_KV_MIB must be set}" +: "${FRANK_ARCHON_BUSY_PREFILL_CHUNK:?FRANK_ARCHON_BUSY_PREFILL_CHUNK must be set (auto|off|N)}" : "${FRANK_ARCHON_KV_DTYPE:?FRANK_ARCHON_KV_DTYPE must be set}" : "${FRANK_ARCHON_SPEC_TOKENS:?FRANK_ARCHON_SPEC_TOKENS must be set}" : "${FRANK_ARCHON_MAX_BATCHED_TOKENS:?FRANK_ARCHON_MAX_BATCHED_TOKENS must be set}" -: "${FRANK_ARCHON_PREFIX_HOST_MIB:?FRANK_ARCHON_PREFIX_HOST_MIB must be set}" -: "${FRANK_ARCHON_PREFIX_DISK_MIB:?FRANK_ARCHON_PREFIX_DISK_MIB must be set}" +: "${FRANK_ARCHON_PREFIX_CACHE_MIB:?FRANK_ARCHON_PREFIX_CACHE_MIB must be set}" : "${FRANK_ARCHON_NGRAM_PLACEMENT:?FRANK_ARCHON_NGRAM_PLACEMENT must be set}" : "${FRANK_ARCHON_TTM_MIN_MIB:?FRANK_ARCHON_TTM_MIN_MIB must be set}" : "${FRANK_ARCHON_TTM_TARGET_MIB:?FRANK_ARCHON_TTM_TARGET_MIB must be set}" @@ -103,19 +102,12 @@ if [ ! -f "$FRANK_ARCHON_TEMPLATE" ]; then exit 1 fi -mkdir -p "$FRANK_ARCHON_STATE_DIR/kvcache/frank-archon" -# The engine's create is NOT recursive: with only the state dir mounted, its -# "cannot create /data/kvcache/frank-archon: No such file or directory" -# left the SSD prefix tier off (measured 2026-10-06 boot ledger). Pre-create -# the full path on the host so the tier opens on first boot. - mkdir -p "$FRANK_ARCHON_STATE_DIR/tune" # rad-tune's kernel-selection cache (see the runbook's "Tuning levers"): # the engine's selector reads $RADIANCE_HOME/tune/.tune at boot; # $RADIANCE_HOME defaults to /opt/radiance, which lives in the image, so # the cache must ride a host mount or it dies with the --rm container -# that wrote it. The dir is pre-created for the same reason as kvcache -# (the engine's create is not recursive). +# that wrote it. The engine's create is not recursive, so the dir is pre-created. if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then echo "ERROR: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2 @@ -148,6 +140,15 @@ done LOG=/home/dank/logs/frank_archon.log mkdir -p "$(dirname "$LOG")" +# Interleave is opt-in: 0 is the engine's default and passing it anyway would put +# a flag in argv that changes nothing but the record. The sharing cap is always +# explicit because `auto` vs `off` is the difference between decode waiting one +# chunk and waiting the whole step. +PREFILL_ARGS=() +if [ "${FRANK_ARCHON_PREFILL_INTERLEAVE:-0}" -gt 0 ]; then + PREFILL_ARGS=(--prefill-interleave "$FRANK_ARCHON_PREFILL_INTERLEAVE") +fi + # Publisher-vetted container contract (docs/DOCKER.md, compose/common.yaml): # --network host (the server binds the host's addresses directly, no NAT), # --init (the engine releases the cards on SIGTERM), seccomp=unconfined @@ -156,8 +157,9 @@ mkdir -p "$(dirname "$LOG")" # makes for host memory), memlock unbounded (the pinned ledger — legal # because the root unit grants LimitMEMLOCK=infinity to this dank process), # /dev/kfd + /dev/dri (render/video groups are moot for a root container). -# The template rides a single-file mount; the .rad dir is ro; /data holds -# the prefix-cache disk tier. +# The template rides a single-file mount and the .rad dir is ro. The prefix cache +# has no disk tier any more, so the only host state this container writes is the +# rad-tune cache under /opt/radiance/share/radiance/tune. podman run -d --name "$FRANK_ARCHON_CONTAINER" \ --network host \ --init \ @@ -165,7 +167,6 @@ podman run -d --name "$FRANK_ARCHON_CONTAINER" \ --ulimit memlock=-1:-1 \ --device /dev/kfd --device /dev/dri \ -v "$FRANK_ARCHON_MODEL_DIR":/models:ro \ - -v "$FRANK_ARCHON_STATE_DIR":/data \ -v "$FRANK_ARCHON_STATE_DIR/tune":/opt/radiance/share/radiance/tune \ -v "$FRANK_ARCHON_TEMPLATE":/templates/template.jinja:ro \ "$FRANK_ARCHON_IMAGE" \ @@ -179,15 +180,13 @@ podman run -d --name "$FRANK_ARCHON_CONTAINER" \ --placement "$FRANK_ARCHON_PLACEMENT" \ --host-pool-mib "$FRANK_ARCHON_HOST_POOL_MIB" \ --gpu-headroom-mib "$FRANK_ARCHON_GPU_HEADROOM_MIB" \ - --vram-weights-mib "$FRANK_ARCHON_VRAM_WEIGHTS_MIB" \ - --vram-kv-mib "$FRANK_ARCHON_VRAM_KV_MIB" \ --kv-cache-dtype "$FRANK_ARCHON_KV_DTYPE" \ - --prefix-cache-host-mib "$FRANK_ARCHON_PREFIX_HOST_MIB" \ - --prefix-cache-dir /data/kvcache/frank-archon \ - --prefix-cache-disk-mib "$FRANK_ARCHON_PREFIX_DISK_MIB" \ + --prefix-cache-mib "$FRANK_ARCHON_PREFIX_CACHE_MIB" \ --ngram-placement "$FRANK_ARCHON_NGRAM_PLACEMENT" \ --num-speculative-tokens "$FRANK_ARCHON_SPEC_TOKENS" \ --max-num-batched-tokens "$FRANK_ARCHON_MAX_BATCHED_TOKENS" \ + --busy-prefill-chunk "$FRANK_ARCHON_BUSY_PREFILL_CHUNK" \ + "${PREFILL_ARGS[@]}" \ ${FRANK_ARCHON_EXTRA_ARGS:+$FRANK_ARCHON_EXTRA_ARGS} \ > "$LOG" 2>&1 diff --git a/tests/frank_archon_unit.bats b/tests/frank_archon_unit.bats index befe2c6..386fa90 100755 --- a/tests/frank_archon_unit.bats +++ b/tests/frank_archon_unit.bats @@ -9,9 +9,9 @@ # rollback). It replaces the FreeToken seam of # 2026-10-05 (git tag frank-archon-freetoken-pr587-20261005) and pins what # flags alone do not say: the served window clears the harness worst case, -# the KV pool backs seqs x max-model-len at the declared cache dtype (an -# engine HARD limit, not a ratio), the per-card VRAM pair stays inside the -# the pinned-host ledger has a ttm pages cap floor, the serve script ENFORCES +# the retired per-card budget knobs are ABSENT (the pinned engine refuses +# them at ExecStart), the pinned-host ledger has a ttm pages cap floor, the +# serve script ENFORCES # it and RAISES the knob itself when a reboot reset it (the cap's default is # half of RAM), the served window is what models.yml must # quote, the host-networking contract (no --publish), the image gate on a @@ -146,20 +146,23 @@ nosudo_bin() { grep -q '^FRANK_ARCHON_TP=2$' "$ENV_FILE" grep -q '^FRANK_ARCHON_TP_WIRE=wht6$' "$ENV_FILE" grep -q '^FRANK_ARCHON_SPEC_TOKENS=2$' "$ENV_FILE" - # expert_tiered placement: vram_weights + the pinned host pool serve the - # per-rank resident set TOGETHER, so cutting the VRAM slab to pay for the - # KV pool moves the pool with it. KV is QUANTIZED fp8 (the engine takes - # only bf16|fp8 — no per-tensor K/V split), the PLE n-gram table is pinned - # in host RAM per Frank's law. + # expert_tiered placement: the pinned host pool plus whatever expert VRAM the + # budget resolver leaves serve the per-rank resident set TOGETHER, so the pool + # size is also the lever on which experts stay on the card. KV is QUANTIZED + # fp8 (the engine takes only bf16|fp8 — no per-tensor K/V split), the PLE + # n-gram table is pinned in host RAM per Frank's law. The per-card VRAM pair + # is NOT asserted here: 1.3.0 removed it and the resolver derives the split, + # so the pool's token count is a boot-ledger line — verified live by the + # capacity leg of scripts/probe-frank-archon-kv.py at every cutover. grep -q '^FRANK_ARCHON_PLACEMENT=expert_tiered$' "$ENV_FILE" grep -q '^FRANK_ARCHON_HOST_POOL_MIB=44928$' "$ENV_FILE" - grep -q '^FRANK_ARCHON_VRAM_WEIGHTS_MIB=[1-9]' "$ENV_FILE" - grep -q '^FRANK_ARCHON_VRAM_KV_MIB=[1-9]' "$ENV_FILE" grep -q '^FRANK_ARCHON_KV_DTYPE=fp8$' "$ENV_FILE" grep -q '^FRANK_ARCHON_NGRAM_PLACEMENT=ram$' "$ENV_FILE" - # Prefix cache tiers (publisher pattern; disk tier sized for 2 slots). - grep -q '^FRANK_ARCHON_PREFIX_HOST_MIB=8192$' "$ENV_FILE" - grep -q '^FRANK_ARCHON_PREFIX_DISK_MIB=65536$' "$ENV_FILE" + # Prefix cache: pinned host RAM only (the disk tier was removed), and the + # step-sharing cap is stated explicitly — auto vs off is the difference + # between decode waiting one chunk and waiting a whole step. + grep -q '^FRANK_ARCHON_PREFIX_CACHE_MIB=8192$' "$ENV_FILE" + grep -q '^FRANK_ARCHON_BUSY_PREFILL_CHUNK=auto$' "$ENV_FILE" # The pinned ledger (2 x pool + 47.7 GiB PLE + prefix host) must be # under the ttm pages cap, and the env states the floor the serve # script enforces (>= 140,000 MiB — the kernel's own default cap, half of @@ -170,7 +173,7 @@ nosudo_bin() { ttmt=$(awk -F= '/^FRANK_ARCHON_TTM_TARGET_MIB=/{print $2}' "$ENV_FILE") [ "$ttmt" -gt "$ttmm" ] pool=$(awk -F= '/^FRANK_ARCHON_HOST_POOL_MIB=/{print $2}' "$ENV_FILE") - ph=$(awk -F= '/^FRANK_ARCHON_PREFIX_HOST_MIB=/{print $2}' "$ENV_FILE") + ph=$(awk -F= '/^FRANK_ARCHON_PREFIX_CACHE_MIB=/{print $2}' "$ENV_FILE") # 2 pools + PLE 47.7 GiB (48,824 MiB) + prefix host, rounded up. ledger=$(( 2 * pool + 48824 + ph + 2048 )) [ "$ttmm" -ge "$ledger" ] @@ -179,47 +182,21 @@ nosudo_bin() { [ "$(awk -F= '/^FRANK_ARCHON_RAD_MIN_MIB=/{print $2}' "$ENV_FILE")" -gt 0 ] } -@test "the KV pool backs seqs x max-model-len at the declared cache dtype" { - # The engine's own hard limit: a request longer than the pool's token - # count is "accepted by the API and then refused as too large to ever fit - # -- this is a hard limit, not a backing ratio". Rates are measured, not - # derived: per rank, kv_attn (13 layers) pages 26.00 KiB per 4 tokens at - # fp8 and 52.00 KiB at bf16, while kv_qsa_bkey (13 layers) pages 3.25 KiB - # at BOTH — the QSA block key is an index, not attention state, and does - # not follow --kv-cache-dtype — so the rate is 7,488 B/token at fp8 and - # 14,144 at bf16 (1.89x, not 2x). The dtype-independent terms also live in - # the pool: ~110 MiB a sequence of recurrent state (kv_gdn_state 54 MiB x 2 - # copies + conv/tail/ple) and 448 MiB of prefix checkpoints (8 x 55.99). - # Verified against BOTH live boots: fp8 3543-330-448 = 2765 MiB -> 387,240 - # tokens (ledger 386,904); bf16 6912-220-448 = 6244 MiB -> 462,946 - # (ledger 462,752). Both within 0.1%. - kv=$(awk -F= '/^FRANK_ARCHON_VRAM_KV_MIB=/{print $2}' "$ENV_FILE") - seqs=$(awk -F= '/^FRANK_ARCHON_MAX_SEQS=/{print $2}' "$ENV_FILE") - maxlen=$(awk -F= '/^FRANK_ARCHON_MAX_MODEL_LEN=/{print $2}' "$ENV_FILE") - case "$(awk -F= '/^FRANK_ARCHON_KV_DTYPE=/{print $2}' "$ENV_FILE")" in - fp8) bpt=7488 ;; - bf16) bpt=14144 ;; - *) fail "unknown FRANK_ARCHON_KV_DTYPE" ;; - esac - tokens=$(( (kv - 110 * seqs - 448) * 1048576 / bpt )) - [ "$tokens" -ge $(( seqs * maxlen )) ] -} - -@test "the per-card VRAM pair stays inside the measured card ceiling" { - # The engine's OWN budget table, not a hand-picked reserve. Boot of - # 2026-10-13 at the pair 24000/5120: card 32,624 MiB (31.86 GiB), budgets - # claimed 29,120, `3.42 GiB is neither budgeted nor accounted — the - # driver's context, the activation arena and --gpu-headroom-mib come out of - # it; anything past those is throughput on the floor`, and `card 0 settled - # at 2.01 GiB free against 209.19 MiB (the 96.00 MiB --gpu-headroom-mib and - # 113.19 MiB kept for the static region)`. Claimable reserve above the - # engine's floor: 2,058 - 209 = 1,849 MiB, so the ceiling is 30,969 MiB. - # The 2026-10-11 bound of 29,120 ("keep >= 1 GiB free", card read as 31,856 - # MiB) was MY reserve, which the engine never asked for; the shipped pair - # spends the difference and lands ~458 MiB above the floor. - w=$(awk -F= '/^FRANK_ARCHON_VRAM_WEIGHTS_MIB=/{print $2}' "$ENV_FILE") - kv=$(awk -F= '/^FRANK_ARCHON_VRAM_KV_MIB=/{print $2}' "$ENV_FILE") - [ $(( w + kv )) -le 30969 ] +@test "the retired budget knobs are absent from the env and the argv" { + # Not unused leftovers: the pinned engine REFUSES to start on any of them + # (--vram-weights-mib/--vram-kv-mib went away at 1.3.0, the prefix-cache disk + # tier outright), and a unit that refuses at ExecStart sits in + # `activating (auto-restart)` while the port still looks alive from whatever + # held it last. A stale line here is an outage, so it is asserted away. + for key in FRANK_ARCHON_VRAM_WEIGHTS_MIB FRANK_ARCHON_VRAM_KV_MIB \ + FRANK_ARCHON_PREFIX_HOST_MIB FRANK_ARCHON_PREFIX_DISK_MIB; do + ! grep -q "^${key}=" "$ENV_FILE" + ! grep -q "$key" "$SERVE" + done + for flag in --vram-weights-mib --vram-kv-mib --prefix-cache-host-mib \ + --prefix-cache-dir --prefix-cache-disk-mib; do + ! grep -q -- "$flag" "$SERVE" + done } @test "the serve script wires the engine flags from the env" { @@ -232,11 +209,13 @@ nosudo_bin() { grep -q -- '--max-model-len "$FRANK_ARCHON_MAX_MODEL_LEN"' "$SERVE" grep -q -- '--placement "$FRANK_ARCHON_PLACEMENT"' "$SERVE" grep -q -- '--host-pool-mib "$FRANK_ARCHON_HOST_POOL_MIB"' "$SERVE" - grep -q -- '--vram-weights-mib "$FRANK_ARCHON_VRAM_WEIGHTS_MIB"' "$SERVE" - grep -q -- '--vram-kv-mib "$FRANK_ARCHON_VRAM_KV_MIB"' "$SERVE" grep -q -- '--kv-cache-dtype "$FRANK_ARCHON_KV_DTYPE"' "$SERVE" - grep -q -- '--prefix-cache-host-mib "$FRANK_ARCHON_PREFIX_HOST_MIB"' "$SERVE" - grep -q -- '--prefix-cache-disk-mib "$FRANK_ARCHON_PREFIX_DISK_MIB"' "$SERVE" + grep -q -- '--prefix-cache-mib "$FRANK_ARCHON_PREFIX_CACHE_MIB"' "$SERVE" + grep -q -- '--busy-prefill-chunk "$FRANK_ARCHON_BUSY_PREFILL_CHUNK"' "$SERVE" + # Interleave is opt-in: it reaches argv only through the guarded array, so a + # 0 in the env leaves the engine's own default in place. + grep -q -- 'PREFILL_ARGS=(--prefill-interleave "$FRANK_ARCHON_PREFILL_INTERLEAVE")' "$SERVE" + grep -qF -- '"${PREFILL_ARGS[@]}"' "$SERVE" grep -q -- '--ngram-placement "$FRANK_ARCHON_NGRAM_PLACEMENT"' "$SERVE" grep -q -- '--num-speculative-tokens "$FRANK_ARCHON_SPEC_TOKENS"' "$SERVE" # The boot-ladder escape hatch is opt-in: unset renders NO extra argv. @@ -261,7 +240,10 @@ nosudo_bin() { grep -q -- '--device /dev/kfd' "$SERVE" grep -q -- '--device /dev/dri' "$SERVE" grep -q -- '-v "$FRANK_ARCHON_MODEL_DIR":/models:ro' "$SERVE" - grep -q -- '-v "$FRANK_ARCHON_STATE_DIR":/data' "$SERVE" + grep -q -- '-v "$FRANK_ARCHON_STATE_DIR/tune":/opt/radiance/share/radiance/tune' "$SERVE" + # No /data mount: the prefix cache lives in pinned host RAM now, so the only + # host state the container writes is the rad-tune cache above. + ! grep -q -- ':/data ' "$SERVE" grep -q -- '-v "$FRANK_ARCHON_TEMPLATE":/templates/template.jinja:ro' "$SERVE" }