From e460bbbab55c14f2f5b4ea84a669448166071dee Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 2 Oct 2026 10:46:01 -0700 Subject: [PATCH] frank-primary: revert to known-good fp8 3x262144 The 2 x 200704 fp8 / MNBT 1024 shape (30904ba) crash-loops at boot: the engine dies in determine_available_memory -> profile_run -> _dummy_run with the SAME torch.compile dynamo constraint that killed the bf16 2 x 262144 shape on 2026-10-01: RuntimeError: Worker failed with error 'Constraints violated (L['query_start_loc'].size()[0])! You marked it as dynamic but your code specialized it to be a constant (3)' Not an OOM. Restoring the last shape measured serving (fp8 3 x 262144 / KV 6.1e9 / MNBT 2048 / arena 6,248) so the endpoint serves again; the desired edits are re-applied one at a time from this base. --- scripts/frank-primary.env | 84 +++++++++++++++------------------- scripts/serve-frank-primary.sh | 18 ++++---- tests/frank_primary_unit.bats | 66 ++++++++++---------------- 3 files changed, 67 insertions(+), 101 deletions(-) diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index bdd2d94..c3107a3 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -59,55 +59,43 @@ FRANK_PRIMARY_PUBLISH=7770:7770 # hoc experiment identity (both hold the GPUs / name registrations). FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8" -# --- qualified engine shape (2026-10-02, Frank's own correction) --- -# 2026-10-01 left a shape that did not stay up: bf16 KV at 2 x 262144 with -# --expert-precise-memory crash-looped at graph capture (the arena is -# DEVICE-resident and the shape had 0.13 GiB to spare), and the plain bf16 -# 2 x 262144 boot booted but left 0 bytes free at runtime. This is the -# correction: FP8 KV, TWO windows, each 200704 tokens, and the pool cut so -# the expert arena gets the room back. -# -# LADDER (2026-10-01 night, subtract-16k method): 262144 and 245760 OOM at -# graph capture; 237568 passed at 718,606 tokens; 241664 x 3 passed at -# 732,359 tokens / 3.03x but is the shape this cutover retired. 200704 was -# chosen to sit below the 241664 rung while leaving a full 2.95 GiB/rank of -# pool to cut loose for the arena — the two contexts are a capacity -# contract, not a throughput one (2-way concurrency summed BELOW parity at -# 4064 ctx), so the slot buys the VRAM headroom long generations need. -# -# KV_BYTES is PER RANK. fp8 costs 7,701 B/token (measured 6,138 attention -# + 1,563 mamba/side, dtype-independent). 2 x 200704 = 401,408 tokens -# -> 3.091e9 B floor, pinned at 3.15e9 (~405,000 tokens, ~44 MB margin — -# the same margin convention the 6.1e9 shape used). -# -# THE ARENA IS WHAT REMAINS: the fork's expert planner sizes it AFTER the -# KV override, so every byte cut here is arena. The 6.1e9 shape ran an -# arena of 6,248 slots; this one reclaims ~2.95 GiB/rank, roughly 1,260 -# more slots at 2.344 MiB each. That is the point of the change — the -# gather read path is the measured decode bound, and a bigger arena means -# more experts resident instead of streamed. -# -# MNBT 1024 (down from 2048): 1024-4096 are interchangeable on decode AND -# identical on arena (the sweep measured the same slot count at every -# rung <= 4096), so this is a pure workspace-floor choice, not a decode -# trade. The floor matters because the arena budget is what is left. -# -# PRECISE MEMORY stays DROPPED: it crashed the 2 x 262144 bf16 boot -# (weights 6.28 + pool 6.80 + graphs 0.55 + arena 16.65 + non-torch ~1.45 -# = 31.73 of 31.86 GiB, ~0.13 GiB free, NRestarts=3). The flag survives in -# the serve script as an opt-in; re-enable it only with a pool/context -# reduction that actually pays for the extra overhead. -# -# TUNABLEOP stays OFF (silently skipped under offload unless -# CLAV_TUNABLEOP_WITH_OFFLOAD=1, and then corrupts numerics). -# -# SEQS 2 x MAXLEN 200704 at FP8 KV, pool cut for the arena. No -# speculative decode, no prefetch. -FRANK_PRIMARY_MAXLEN=200704 -FRANK_PRIMARY_SEQS=2 -FRANK_PRIMARY_KV_BYTES=3150000000 +# --- qualified engine shape (2026-10-01, RESTORED KNOWN-GOOD: fp8 3 windows) --- +# bf16 at 2 x 262144 did NOT stay up: with and without --expert-precise-memory +# the boot failed (`Engine core initialization failed`, NRestarts climbing). +# The last shape measured serving was fp8 3 x 262144 / MNBT 2048 / KV 6.1e9 +# (KV 792,069 tokens, 3.02x, arena 6,248, ~3.6 GiB free) and is restored here +# while the bf16 path is investigated in a window (see log.md 2026-10-01). +# Candidate next test: bf16 with SEQS=3 to isolate the KV dtype from the +# sequence count, and/or --enforce-eager to bypass the dynamo constraint path. +# SEQS 2 x MAXLEN 262144 at BF16 KV (Frank's call, 2026-10-01): two contexts of +# the model's FULL native window with UNQUANTIZED KV, plus +# --expert-precise-memory to hand the arena the compile/graph-capture headroom. +# Pool: bf16 costs 13,839 B/token (measured: 11,047,010,304 B -> 798,280 tokens +# on the 3x241664 bf16 boot; decomposes as 12,276 attention + 1,563 mamba/side +# caches, against fp8's 6,138 + 1,563 = 7,701). 2 x 262144 = 524,288 tokens +# -> 7.256e9 B floor, pinned at 7,300,000,000 (~527,500 tokens, 3,206 margin). +# Budget (per rank): 31.22 GiB total; non-expert weights 6.28, graphs 0.55, +# non-torch ~1.45, pool 6.76 -> ~29.35 used, ~2.5 GiB free (measured basis: +# the fp8 3-window shape used 28.27 and left 3.59). +# PRECISE MEMORY: DROPPED 2026-10-01 after it crashed the boot in combination +# with bf16. The flag selects the fitted 5.25 GiB overhead instead of the +# 7.60 GiB default, moving the plan's `room` and therefore the arena +# (6,248 -> ~7,278 slots). But the arena is DEVICE-resident, and the budget +# for this shape does not have 2.35 GiB to spare: +# weights 6.28 + pool 6.80 (bf16 2 x 262144) + graphs 0.55 + arena 16.65 +# + non-torch ~1.45 = 31.73 of 31.86 GiB => ~0.13 GiB free at boot +# measured as a CrashLoop: every attempt reached graph capture, the engine +# exited cleanly, systemd restarted it (NRestarts=3 at ~6.7 min each). +# With the default overhead the arena is 14.31 GiB and the same shape totals +# ~29.4 GiB, leaving ~2.4 GiB for capture. Re-enable the flag only with a +# pool/context reduction that pays for it. +# MNBT 2048 (Frank): the overhead formula only adds bytes above nbt 4096, so +# 2048 pays nothing, and the arena sweep measured the same ceiling <= 4096. +FRANK_PRIMARY_MAXLEN=262144 +FRANK_PRIMARY_SEQS=3 +FRANK_PRIMARY_KV_BYTES=6100000000 FRANK_PRIMARY_KV_DTYPE=fp8 -FRANK_PRIMARY_MNBT=1024 +FRANK_PRIMARY_MNBT=2048 FRANK_PRIMARY_GATHER_CHUNKS=32 FRANK_PRIMARY_GATHER_LANES=64 FRANK_PRIMARY_UTIL=0.98 diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index 78c6578..e3c885f 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -11,16 +11,14 @@ # every flag here was qualified by boot probe on 2026-09-29 — see # frank-primary.env for the qualification record). # -# Shape (2026-10-02, Frank's correction of the 10-01 bf16 shape): MAXLEN -# 200704 x SEQS 2 (401,408 tokens of contract), KV fp8 (--kv-cache-dtype -# fp8; the bf16 unquantized shape is retired to git tag -# frank-primary-fp8-mtp-20261001), KV_BYTES 3,150,000,000 PER RANK (the -# engine measured 405,000 tokens of capacity / ~3.0x concurrency), NO -# speculative decode, --max-num-batched-tokens 1024 (Frank's lock), MoE -# gather geometry R4D_LRU_GATHER_CHUNKS=32 / -# R4D_LRU_GATHER_LANES=64 (+18.7% decode, grows with depth), prefix -# caching, expert prefetch OFF (36% precision measured), TunableOp off, and -# the pool deliberately small so the expert arena keeps what it reclaims. +# Shape (2026-10-01 bf16 cutover): MAXLEN 262144 x 3 sessions (786,432 +# tokens), KV UNQUANTIZED (--kv-cache-dtype auto = bf16; the fp8 q8/q8 shape +# is retired to git tag frank-primary-fp8-mtp-20261001), KV_BYTES +# 11,047,010,304 PER RANK (the engine measured 798,280 tokens of capacity / +# 3.05x concurrency), NO speculative decode, --max-num-batched-tokens 1024 +# (Frank's lock), MoE gather geometry R4D_LRU_GATHER_CHUNKS=32 / +# R4D_LRU_GATHER_LANES=64 (+18.7% decode, grows with depth), prefix caching, +# expert prefetch OFF (36% precision measured), TunableOp off. # # The root unit (scripts/frank-primary.service, User=dank, # LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert host diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index 609b1a9..9822ec1 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -2,11 +2,10 @@ # Seam: frank-primary — the Dankserver LLM endpoint on :7770. # # This file asserts the tcclaviger RDNA4 vLLM ROOT-UNIT shape (the -# 2026-10-02 correction of the 2026-10-01 BF16 cutover): the serve +# 2026-10-01 BF16 cutover, the overnight redo campaign's verdict): the serve # script drives `podman run` DIRECTLY (the image entrypoint execs # `vllm serve`), with every engine flag qualified by boot probe and carried -# in the repo env file. -# The seam pins what the flags alone do not say: the +# in the repo env file. The seam pins what the flags alone do not say: the # KV qualification law (the night ladder: 262144/245760 OOM at graph # capture; 241664 boots at 732,359 tokens / 3.03x, util 0.98; # the bf16-KV trade (double the attention-KV bytes/token, ~14% decode cost, @@ -92,31 +91,29 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" @test "the qualified engine shape is in the env file" { # 2026-10-01 night ladder (Frank's method), then a runtime-survivability - # correction, then Frank's 2026-10-02 decision to give the pool back to - # the expert arena. SEQS=3 bf16 topped out at 241664 (KV 732,359 / 3.03x); + # correction. SEQS=3 bf16 topped out at 241664 (KV 732,359 / 3.03x); # SEQS=6 fp8 does NOT fit (6 x 241664 asked 10.70 GiB against 9.18 GiB # free); SEQS=5 fp8 x 262144 BOOTED (KV 1,350,182 / 5.15x) but left ~0 # bytes free at runtime and a real client request (995-token prompt with # max_tokens=261149) killed the worker with `Tried to allocate 50.00 MiB - # ... 0 bytes free`. 200704 sits below the 241664 rung while leaving a - # full 2.95 GiB/rank to cut loose for the arena. UTIL stays 0.98 - # (lowering it shrinks the engine's own budget instead of buying slack). - # FINAL (2026-10-02, Frank): TWO windows at 200704 with FP8 KV and a pool - # cut for the arena. fp8 costs 7,701 B/token (measured 6,138 attention + - # 1,563 mamba/side), so 2 x 200704 = 401,408 tokens needs 3.091e9 B; - # pinned at 3.15e9 (~405k tokens, ~44 MB margin). The arena is what - # remains — the planner sizes it AFTER the KV override — so the 6,248 - # slots of the 6.1e9 shape become roughly 7,500 here. - grep -q '^FRANK_PRIMARY_MAXLEN=200704$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_SEQS=2$' "$ENV_FILE" + # ... 0 bytes free`. FOUR slots at the full native 262144 window is the + # deployed shape: ~1.93 GiB more headroom per rank for + # activations/graphs. UTIL stays 0.98 (lowering it shrinks the engine's + # own budget instead of buying slack). + # FINAL (2026-10-01, Frank): TWO full-size windows with BF16 KV plus + # --expert-precise-memory. bf16 costs 13,839 B/token (measured), so + # 2 x 262144 = 524,288 tokens needs 7.256e9 B; pinned at 7.3e9 (~527.5k + # tokens, 3,206 margin). The precise flag drops the plan's overhead + # 7.60 -> 5.25 GiB, moving the arena from 6,248 to ~7,278 slots. + grep -q '^FRANK_PRIMARY_MAXLEN=262144$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_SEQS=3$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_UTIL=0.98$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_KV_BYTES=3150000000$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_MNBT=1024$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_KV_BYTES=6100000000$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_MNBT=2048$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE" - # --expert-precise-memory is NOT set: with bf16 2 x 262144 it put the - # boot 0.13 GiB from the 31.86 GiB ceiling (arena 16.65 device-resident) - # and crash-looped at graph capture. The serve script keeps the opt-in - # flag; the smaller pool here is the arena it would have been paying for. + # --expert-precise-memory is NOT set: with bf16 2 x 262144 it put the boot + # 0.13 GiB from the 31.86 GiB ceiling (arena 16.65 device-resident) and + # crash-looped at graph capture. The serve script keeps the opt-in flag. ! grep -q '^FRANK_PRIMARY_PRECISE_MEMORY=' "$ENV_FILE" grep -q '^FRANK_PRIMARY_GATHER_CHUNKS=32$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_GATHER_LANES=64$' "$ENV_FILE" @@ -124,22 +121,6 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" grep -q '^FRANK_PRIMARY_MODEL_SHARDS=131$' "$ENV_FILE" } -@test "the pool is sized as the arena floor, not the context floor" { - # THE LAW this seam pins: the fork's expert planner sizes the arena - # AFTER the KV override, so every byte cut out of KV_BYTES becomes - # arena. A context-size change must move the pool with it, or the - # reduction buys nothing. 2 x 200704 = 401,408 tokens at fp8's 7,701 - # B/token is 3.091e9 B; the pin is 3.15e9, ~44 MB of margin above the - # floor — the same convention the 6.1e9 shape used. - floor=$(awk -v n="401408" -v b="7701" 'BEGIN{printf "%.0f", n*b}') - [ "$floor" = "3091243008" ] - kv=$(grep '^FRANK_PRIMARY_KV_BYTES=' "$ENV_FILE" | cut -d= -f2) - [ "$kv" = "3150000000" ] - margin=$(( kv - floor )) - [ "$margin" -gt 40000000 ] - [ "$margin" -lt 100000000 ] -} - @test "the serve script wires the engine flags from the env" { grep -q -- '--max-num-seqs "$FRANK_PRIMARY_SEQS"' "$SERVE" grep -q -- '--max-model-len "$FRANK_PRIMARY_MAXLEN"' "$SERVE" @@ -227,14 +208,13 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" grep -q -- '-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"' "$SERVE" } -@test "MNBT is at Frank's 1024 with the arena intact" { +@test "MNBT is at Frank's 2048 with the arena intact" { # MNBT 512 doubles TTFT at 4064 ctx (2.46 s vs 1.36 s); 1024-4096 are # interchangeable on decode AND identical on arena (the sweep measured # 6,499 slots at every rung <=4096); 12288 costs 2.5% plus arena. - # 1024 is the 2026-10-02 setting — the pool was cut to feed the expert - # arena, so the workspace floor matters more than it did at 2048: the - # arena budget is what is left, and 1024-4096 pay nothing for it. - grep -q '^FRANK_PRIMARY_MNBT=1024$' "$ENV_FILE" + # 2048 is Frank's setting as of 2026-10-01 — it keeps the workspace floor + # so the expert arena stays max. + grep -q '^FRANK_PRIMARY_MNBT=2048$' "$ENV_FILE" grep -q -- '--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"' "$SERVE" }