diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index e4dee1f..aed39f4 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -1,15 +1,13 @@ # frank-primary.env — tcclaviger RDNA4 vLLM production config (dankserver :7770). # -# 2026-10-03 shape G: the arena forced to the flag ceiling. The expert arena is +# 2026-10-03 shape G (as shipped, after the 1.00 revert): the CUDA-graph plan +# term dropped from the analytic 0.55 GiB to the measured 0.07 GiB (0 base + +# 12 MiB/graph — the engine's own post-capture delta is -0.06 GiB), and UTIL at +# the 0.982 law. Arena 17.78 GiB / ~7,912 slots predicted (F measured 17.52 / +# 7,653). UTIL stays 0.982 ALWAYS: 1.00 is unbootable on this fork (worker init +# refuses budget >= device total) and 0.13-GiB-margin boots die under load. # `budget - non-expert - KV reserve - graphs - overhead` and the KV reserve is a -# registry constant, so exactly two terms are reachable: the CUDA-graph estimate -# (priced at the measured capture — the engine's own post-capture delta is -# -0.06 GiB, i.e. free, so we charge 12 MiB/graph and nothing else) and the -# budget, which is `device_total x gpu_memory_utilization` with the field pinned -# to `le=1`. UTIL 1.00 + graph term 0.07 GiB is therefore the MAXIMUM arena this -# repo can ask for without editing the image: 18.35 GiB / ~8,016 slots. Frank -# approved forcing util to 1.00 for this; the measured cost is the free-VRAM -# margin, which lands at ~1.5 GiB/card (see FRANK_PRIMARY_UTIL). +# registry constant, so those two terms are the whole flag surface. # 2026-10-03 shape F (absorbed): MNBT 2048 + the graph term pinned. Measured on # the live deploy: arena 17.52 GiB / 7,653 slots, budget 31.29, graphs 0.32. # 2026-10-02 shape E: fp8 KV, MAXLEN 200704 x SEQS 3, pool 3.15e9/rank, and @@ -175,18 +173,19 @@ FRANK_PRIMARY_MNBT=2048 FRANK_PRIMARY_GATHER_CHUNKS=32 FRANK_PRIMARY_GATHER_LANES=64 # UTIL is an ARENA lever: the fork plans `budget = device_total x util`, and at -# 31.86 GiB/card every +0.01 is +0.32 GiB of expert cache (~+139 slots). Frank -# held it at 0.982 and then approved FORCING it to 1.00 to buy the last gig of -# arena — with the caveat written down, because this endpoint has twice died at -# the ceiling (the 2x262144 bf16 boot at 0.13 GiB free; the 5x262144 boot at -# "0 bytes free" on a real 995-token request). At util 1.00 the engine's budget -# equals the whole card, so anything the registry under-prices shows up as a -# RUNTIME allocation failure rather than a plan refusal. MEASURED cost of the -# force: the driver high-water mark moves from 28.28 GiB (shape E) to ~30.4 GiB, -# i.e. ~1.5 GiB of real free VRAM per card is the safety margin. If a request -# ever dies with `Tried to allocate ... 0 bytes free`, the rollback is THIS -# number back to 0.982 (-0.57 GiB of arena), not the window. -FRANK_PRIMARY_UTIL=1.0 +# 31.86 GiB/card every +0.01 is +0.32 GiB of expert cache (~+139 slots), and +# 0.982 is Frank's standing law: NEVER 1.00. The 1.00 attempt (2026-10-03, +# shape G v1) died in worker init BEFORE loading a byte of weights — the fork +# refuses budget >= device total with +# ValueError: Free memory on device cuda:N (31.29/31.86 GiB ...) +# on BOTH ranks at 09:53 and 09:54, NRestarts=2 — so 1.0 is not even a runtime +# knife-edge here, it is unbootable. The two historical ceiling deaths (the +# 2x262144 bf16 boot at 0.13 GiB free; the 5x262144 boot at "0 bytes free" on a +# real 995-token request) remain the reason the margin exists. Rollback note +# for the other direction: if a request ever dies with +# `Tried to allocate ... 0 bytes free`, the first knob back is THIS number +# down a notch, not the window. +FRANK_PRIMARY_UTIL=0.982 # CUDA-graph plan term, priced at the measured capture. expert_plan.py charges # estimate_cudagraph_memory() = BASE_MB + PER_GRAPH_MB x n_graphs (defaults # 256/52 = 0.55 GiB for our 6-graph plan), and the estimator's own docstring diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index 83d46b9..fc06b46 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -117,7 +117,7 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" # runtime headroom, not arena. grep -q '^FRANK_PRIMARY_MAXLEN=200704$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_SEQS=3$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_UTIL=1.0$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_UTIL=0.982$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_KV_BYTES=3166500000$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_MNBT=2048$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE" @@ -265,15 +265,20 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" grep -q -- '-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"' "$SERVE" } -@test "the arena is forced to the flag ceiling and the KV ask is honoured" { +@test "the arena sits just under the ceiling and the KV ask is honoured" { # The planner sizes expert cache as budget - non-expert - KV reserve - # graphs - overhead, and two of those five terms are reachable from a - # deploy: budget (= device_total x util, field-capped at le=1) and graphs - # (VLLM_CUDAGRAPH_MEM_*). So the shipped shape must sit at the ceiling of - # BOTH: util at its field maximum, and a graph charge strictly below the - # fork's analytic 563 MiB default while still paying per graph (the measured - # post-capture delta is -0.06 GiB, i.e. the base pool is the over-charge, but - # a future spec-decode shape captures 18 graphs and must still be charged). + # deploy: budget (= device_total x util) and graphs (VLLM_CUDAGRAPH_MEM_*). + # UTIL has a HARD LAW, not a field cap: 0.982, never 1.00. The 1.00 attempt + # is UNBOOTABLE on this fork — worker init raises + # `ValueError: Free memory on device cuda:N (31.29/31.86 GiB ...)` + # on both ranks because the worker demands free > budget and budget == + # total — and even if it booted, 0.13-GiB-margin shapes have died under + # real load. So the arena tops out at 0.982 budget (31.29 GiB). + # Graphs: charge strictly below the fork's analytic 563 MiB default while + # still paying per graph (the measured post-capture delta is -0.06 GiB, i.e. + # the base pool is the over-charge, but a future spec-decode shape captures + # 18 graphs and must still be charged). # MNBT stays 2048: 512 doubles TTFT at 4064 ctx, 1024-4096 are decode- # interchangeable AND arena-identical (the registry prices overhead flat up # to its nbt 4096 anchor), 12288 costs 2.5% plus arena. @@ -282,7 +287,7 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" # must clear 400,700 tokens to keep two full-length resident. grep -q '^FRANK_PRIMARY_MNBT=2048$' "$ENV_FILE" grep -q -- '--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"' "$SERVE" - awk -F= '/^FRANK_PRIMARY_UTIL=/{exit !($2+0 >= 1.0)}' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_UTIL=0.982$' "$ENV_FILE" base=$(awk -F= '/^FRANK_PRIMARY_CUDAGRAPH_BASE_MB=/{print $2}' "$ENV_FILE") per_graph=$(awk -F= '/^FRANK_PRIMARY_CUDAGRAPH_PER_GRAPH_MB=/{print $2}' "$ENV_FILE") [ "$base" -eq 0 ] && [ "$per_graph" -gt 0 ]