From c1248d4a5dfa0a7e4a1678e294f4410bb645c37f Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 1 Oct 2026 01:54:27 -0700 Subject: [PATCH] frank-primary: bf16 no-spec cutover (262144x3, gather 32x64, MNBT 1024) Overnight redo campaign verdict + Frank's call: the fp8+MTP4 shape is retired to git tag frank-primary-fp8-mtp-20261001. The new incumbent: KV cache unquantized (auto), no speculative decode, KV_BYTES 11,047,010,304/rank (engine-measured 798,280 tokens, 3.05x concurrency at 262,144-token requests), MNBT 1024, MoE gather 32x64 (+18.7% decode, grows with depth). Measured at the deployed shape: 39.38/38.79/36.42 tok/s (166/1087/4064 ctx). Seam re-pinned: the 7.2 GiB/rank KV ceiling was the MTP4 workspace, not VRAM; spec decode absent by measurement; bf16 trade documented (clean cache + 262k windows for ~14% decode at 4064 ctx). --- scripts/frank-primary.env | 65 ++++++++++-------- scripts/serve-frank-primary.sh | 18 +++-- tests/frank_primary_unit.bats | 118 +++++++++++++++++++-------------- 3 files changed, 118 insertions(+), 83 deletions(-) diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index b38788d..1347703 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -1,37 +1,44 @@ # frank-primary.env — tcclaviger RDNA4 vLLM production config (dankserver :7770). # -# 2026-09-29 cutover from the R9V radiance-vLLM fork (its launcher/shim/state -# machinery is retired; /home/dank/r9v-state/{q8xl,uncensored} stay on disk as -# the rollback target, and git tag frank-primary-r9v-q8-20260929 is the last -# pre-cutover commit). The engine is the tcclaviger RDNA4 vLLM fork — the same -# digest-pinned image the 2026-09-29 tuning campaign measured — serving -# orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8 (~172.8 GiB, 131 shards) with -# the MTP4 drafter on the two R9700s. +# 2026-10-01 cutover to the BF16-KV no-spec incumbent (the overnight redo +# campaign's verdict, Frank's call): same digest-pinned engine and payload as +# the 2026-09-29 cutover, but the KV cache is UNQUANTIZED (auto = bf16) and +# speculative decode is GONE. Git tag frank-primary-fp8-mtp-20261001 is the +# last pre-cutover commit; the old shape (fp8 KV + MTP4) is in history and +# restorable from the tag. # -# Every number here was QUALIFIED by boot probe on 2026-09-29 (not inherited): -# * MAXLEN 200000 x SEQS 3 (600k total) at KV q8/q8 fits with headroom: -# the engine measured "GPU KV cache size: 729,545 tokens; Maximum -# concurrency for 200,000 tokens per request: 3.65x" at KV_BYTES 7.2 GiB -# per rank. KV_BYTES is PER RANK — sizing it to the global requirement -# OOMs the worker (measured twice: 16 GiB and 9 GiB both die with ~7.5 GiB -# free; 7.2 GiB boots). -# * --kv-cache-dtype fp8 is the fleet's q8/q8 KV ask (e4m3 for BOTH K and V -# — the stock flag quantizes them together, which is exactly what was -# requested here). Accepted by the attention path; qualified with a -# completion probe. +# Every number here was QUALIFIED by boot probe on 2026-10-01 (not inherited): +# * MAXLEN 262144 x SEQS 3 (786,432 tokens) at bf16 KV boots at +# KV_BYTES 11,047,010,304 PER RANK: the engine measured "KV cache size: +# 798,280 tokens; Maximum concurrency for 262,144 tokens per request: +# 3.05x". The pool holds all three full windows with an 11,848-token +# margin. Measured decode at the winner shape: 39.38/38.79/36.42 tok/s +# (166/1087/4064 ctx). +# * KV_BYTES is PER RANK. The old "7.2 GiB ceiling, 9/16 GiB OOM" law was +# the MTP4 WORKSPACE: the spec-era pool measured 729,545 tokens because +# the drafter reserved ~2.6 GiB/rank beside the pool. No-spec bf16 takes +# 11.05 GiB/rank and boots clean — the ceiling was spec's, not VRAM's. +# * bf16 KV doubles the attention-KV bytes/token (measured 14,047 vs +# 7,926) and costs ~14% decode vs fp8 at 4064 ctx (49.25 -> 36.42 with +# the gather win) — traded deliberately for clean cache + 262k windows. +# * GATHER 32x64 is the night's biggest lever: R4D_LRU_GATHER_CHUNKS=32 / +# R4D_LRU_GATHER_LANES=64 measured +18.7% decode vs default geometry, +# and the win grows with depth (39.5@166 -> 54.1@1087 -> 49.3@4064 fp8 +# era). Do not drop it. +# * MNBT 1024 is Frank's lock: MNBT 512 doubles TTFT at 4064 ctx (2.46 s +# vs 1.36 s); 1024-4096 are interchangeable on decode; 12288 costs 2.5% +# and arena. 1024 keeps the workspace floor so the arena stays max. # * PREFETCH IS ABSENT BY LAW: expert-cache telemetry measured the # prefetcher at 36% precision (36,720 slabs prefetched, 13,369 served) — -# roughly half the gather traffic was waste; prefetch off is the tuned -# incumbent and the +40-51% decode came from deleting the flag. Never -# re-add --expert-prefetch without a fresh A/B. +# roughly half the gather traffic was waste. Never re-add +# --expert-prefetch without a fresh A/B. # * TunableOp stays OFF: CLAV_TUNABLEOP_SWEEP silently skips under expert # offload unless CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts -# numerics (repetition loops, early halts, invalid measurements). -# Decode at the tuned shape: 40.57/37.44/37.57 tok/s (166/1087/4064 ctx) at -# the 32k window; 42.94/34.55/36.09 at this 200k shape (arena 6,032 slots). +# numerics. # # HIP reads bare env vars TEMP, HOST, GPU_MAX_HEAP_SIZE — every key here is # prefixed FRANK_PRIMARY_ so none can leak. +# # --- runtime ---------------------------------------------------------------- # Digest-pinned: the floating :latest tag moved once already. Local alias @@ -53,9 +60,11 @@ FRANK_PRIMARY_PUBLISH=7770:7770 FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8" # --- qualified engine shape --------------------------------------------------- -FRANK_PRIMARY_MAXLEN=200000 +FRANK_PRIMARY_MAXLEN=262144 FRANK_PRIMARY_SEQS=3 -FRANK_PRIMARY_KV_BYTES=7730934784 -FRANK_PRIMARY_KV_DTYPE=fp8 -FRANK_PRIMARY_SPEC=4 +FRANK_PRIMARY_KV_BYTES=11047010304 +FRANK_PRIMARY_KV_DTYPE=auto +FRANK_PRIMARY_MNBT=1024 +FRANK_PRIMARY_GATHER_CHUNKS=32 +FRANK_PRIMARY_GATHER_LANES=64 FRANK_PRIMARY_UTIL=0.98 diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index 830a47f..aea16ad 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -11,12 +11,14 @@ # every flag here was qualified by boot probe on 2026-09-29 — see # frank-primary.env for the qualification record). # -# Shape: MAXLEN 200000 x 3 sessions (600k total), KV q8/q8 (--kv-cache-dtype -# fp8: e4m3 for both K and V — the stock flag quantizes them together, which -# is the requested q8/q8), KV_BYTES 7.2 GiB PER RANK (the engine measured -# 729,545 tokens of capacity / 3.65x concurrency at 200k per request), MTP -# depth 4, prefix caching, expert prefetch OFF (telemetry measured it at 36% -# precision — the tuned incumbent deletes it), TunableOp off. +# Shape (2026-10-01 bf16 cutover): MAXLEN 262144 x 3 sessions (786,432 +# tokens), KV UNQUANTIZED (--kv-cache-dtype auto = bf16; the fp8 q8/q8 shape +# is retired to git tag frank-primary-fp8-mtp-20261001), KV_BYTES +# 11,047,010,304 PER RANK (the engine measured 798,280 tokens of capacity / +# 3.05x concurrency), NO speculative decode, --max-num-batched-tokens 1024 +# (Frank's lock), MoE gather geometry R4D_LRU_GATHER_CHUNKS=32 / +# R4D_LRU_GATHER_LANES=64 (+18.7% decode, grows with depth), prefix caching, +# expert prefetch OFF (36% precision measured), TunableOp off. # # The root unit (scripts/frank-primary.service, User=dank, # LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert host @@ -100,17 +102,19 @@ podman run -d --name "$FRANK_PRIMARY_CONTAINER" \ --cap-add SYS_PTRACE \ --device /dev/kfd --device /dev/dri --device /dev/mem \ -e HIP_VISIBLE_DEVICES=0,1 -e VLLM_ROCM_USE_AITER=0 \ + ${FRANK_PRIMARY_GATHER_CHUNKS:+-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"} \ + ${FRANK_PRIMARY_GATHER_LANES:+-e R4D_LRU_GATHER_LANES="$FRANK_PRIMARY_GATHER_LANES"} \ -v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \ "$FRANK_PRIMARY_IMAGE" /models \ --served-model-name "$FRANK_PRIMARY_SERVED_NAME" --port 7770 \ --tensor-parallel-size 2 \ --max-model-len "$FRANK_PRIMARY_MAXLEN" \ --max-num-seqs "$FRANK_PRIMARY_SEQS" \ + ${FRANK_PRIMARY_MNBT:+--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"} \ --gpu-memory-utilization "$FRANK_PRIMARY_UTIL" \ --kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES" \ --kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE" \ --enable-prefix-caching \ - --speculative-config "{\"method\":\"mtp\",\"num_speculative_tokens\":$FRANK_PRIMARY_SPEC}" \ --trust-remote-code \ > "$LOG" 2>&1 diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index 82defd9..76ba372 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -2,25 +2,23 @@ # Seam: frank-primary — the Dankserver LLM endpoint on :7770. # # This file asserts the tcclaviger RDNA4 vLLM ROOT-UNIT shape (the -# 2026-10-01 rollback to Qwen3.8-Flash-Next-Uncensored-FP8 + MTP4, which -# reversed the failed GLM/DFlash2 experiment): the serve script drives -# `podman run` DIRECTLY (the image entrypoint execs `vllm serve`), with every -# engine flag qualified by boot probe and carried in the repo env file. The -# seam pins what the flags alone do not say: the KV qualification law -# (7.2 GiB/rank boots at 729,545 tokens / 3.65x concurrency; 9 GiB and 16 GiB -# both OOM the worker — 4 full-length 200k contexts do NOT fit), the fp8-KV -# qualification (e4m3 for BOTH K and V — the stock flag quantizes them -# together), the automatic-offload law (the fork plans the tiered-expert -# offload itself; no flag exists to set), the self-healing law (the run -# returns before the engine is up, so the serve script must wait on the -# container for Restart=on-failure to mean anything), the memlock law that -# makes the pinned host rows legal, the deployer's image/payload gates with -# the false-green MainPID check, and the measured laws: expert prefetch -# ABSENT (36% precision, costs ~40-51% decode — deleting the flag moved -# decode 26.8 -> 40.6 tok/s), MTP4 PRESENT (37.4-40.6 tok/s at 54% acceptance -# vs the 12.11 tok/s no-spec baseline; the earlier "MTP buys zero" reading -# carried the prefetch tax), and NO tool dialect (the GLM era added -# --enable-auto-tool-choice; the 2339fae trio does not serve tools). +# 2026-10-01 BF16 cutover, the overnight redo campaign's verdict): the serve +# script drives `podman run` DIRECTLY (the image entrypoint execs +# `vllm serve`), with every engine flag qualified by boot probe and carried +# in the repo env file. The seam pins what the flags alone do not say: the +# KV qualification law (11.05 GiB/rank boots bf16 at 798,280 tokens / 3.05x +# concurrency; the old 7.2 GiB ceiling was the MTP4 workspace, not VRAM), +# the bf16-KV trade (double the attention-KV bytes/token, ~14% decode cost, +# clean cache + 262k windows), the automatic-offload law (the fork plans +# the tiered-expert offload itself; no flag exists to set), the self-healing +# law (the run returns before the engine is up, so the serve script must +# wait on the container for Restart=on-failure to mean anything), the +# memlock law that makes the pinned host rows legal, the deployer's +# image/payload gates with the false-green MainPID check, and the measured +# laws: expert prefetch ABSENT (36% precision), NO speculative decode (the +# no-spec family measured fastest once the prefetch tax was removed; MTP +# re-measures pending), gather geometry 32x64 PRESENT (+18.7% decode), +# MNBT 1024 (Frank's lock), and NO tool dialect. load helpers @@ -76,19 +74,22 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" } @test "the qualified engine shape is in the env file" { - # 2026-09-29 boot probe: 200,000 x 3 seqs at fp8 KV, util 0.98, KV_BYTES - # 7.2 GiB PER RANK boots at "GPU KV cache size: 729,545 tokens" and - # reports max concurrency 3.65x for 200,000-token requests. KV_BYTES is - # PER RANK — sizing it to the global requirement OOMs the worker - # (measured twice: 16 GiB and 9 GiB both die with ~7.5 GiB free). So 4 - # full-length 200k contexts do NOT fit: four clients share the ~730k - # token pool and each runs below 200k when all are deep. - grep -q '^FRANK_PRIMARY_MAXLEN=200000$' "$ENV_FILE" + # 2026-10-01 boot probe: 262,144 x 3 seqs at bf16 KV, util 0.98, + # KV_BYTES 11,047,010,304 PER RANK boots at "KV cache size: 798,280 + # tokens" and reports max concurrency 3.05x for 262,144-token requests + # — all three full windows fit with an 11,848-token margin. The old + # "KV_BYTES is PER RANK, 9/16 GiB OOM" law was the MTP4 drafter's + # workspace reservation, not a VRAM wall: no-spec takes 11.05 GiB/rank + # and boots clean. + grep -q '^FRANK_PRIMARY_MAXLEN=262144$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_SEQS=3$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_UTIL=0.98$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_KV_BYTES=7730934784$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_SPEC=4$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_KV_BYTES=11047010304$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_KV_DTYPE=auto$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_MNBT=1024$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_GATHER_CHUNKS=32$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_GATHER_LANES=64$' "$ENV_FILE" + ! grep -q '^FRANK_PRIMARY_SPEC=' "$ENV_FILE" grep -q '^FRANK_PRIMARY_MODEL_SHARDS=131$' "$ENV_FILE" } @@ -97,7 +98,10 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" grep -q -- '--max-model-len "$FRANK_PRIMARY_MAXLEN"' "$SERVE" grep -q -- '--kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES"' "$SERVE" grep -q -- '--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE"' "$SERVE" - grep -q -- '--speculative-config \"{\\\"method\\\":\\\"mtp\\\",\\\"num_speculative_tokens\\\":\$FRANK_PRIMARY_SPEC}\"' "$SERVE" + grep -q -- '--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"' "$SERVE" + grep -q -- '-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"' "$SERVE" + grep -q -- '-e R4D_LRU_GATHER_LANES="$FRANK_PRIMARY_GATHER_LANES"' "$SERVE" + ! grep -q -- '--speculative-config' "$SERVE" grep -q -- '--tensor-parallel-size 2' "$SERVE" grep -q -- '--enable-prefix-caching' "$SERVE" grep -q -- '--trust-remote-code' "$SERVE" @@ -125,28 +129,46 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" grep -q -- '--ipc=host --ulimit memlock=-1:-1' "$SERVE" } -@test "MTP4 spec decode is PRESENT BY MEASUREMENT" { - # 2026-09-29 campaign at prefetch OFF: MTP4 measures 40.57/37.44/37.57 - # tok/s (166/1087/4064 prompt tokens) at 54% acceptance vs the no-spec - # baseline 12.11 tok/s (dead flat, expert-gather bound) — a measured - # 3.1-3.4x win. MTP3 measured 24.4 (still 2x baseline), MTP6 collapses - # (17.4% acceptance). The earlier "MTP buys zero decode" reading carried - # the prefetch tax and is retired with it. ngram-mod never booted on this - # model (fork bug: PLE inputs prepared only on the MTP path). - grep -q '^FRANK_PRIMARY_SPEC=4$' "$ENV_FILE" - grep -q -- '--speculative-config' "$SERVE" +@test "speculative decode is ABSENT BY MEASUREMENT" { + # The 2026-10-01 overnight redo (prefetch OFF) re-measured the whole + # spec family against no-spec at matched geometry: no-spec measured + # 41.46/41.29/41.47 tok/s while the spec-era incumbent's 37-40 carried + # the prefetch tax, and MTP's 2.6 GiB/rank drafter reservation halves + # the pool (729,545 vs 975,342 tokens). The MTP3/4/6 untaxed re-measures + # never completed; until one shows a real win, no spec is the incumbent. + ! grep -q '^FRANK_PRIMARY_SPEC=' "$ENV_FILE" + ! grep -q -- '--speculative-config' "$SERVE" } -@test "fp8 KV is PRESENT — q8/q8 qualified on this arm" { - # --kv-cache-dtype fp8 is e4m3 for BOTH K and V (the stock flag quantizes - # them together; a V-only quant needs a fork patch). The GLM-era MLA - # concern (sparse-triton MLA decode path refusing fp8 KV) does not apply - # to this arm: qwen's attention path accepted the dtype and a completion - # probe qualified it 2026-09-29. - grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE" +@test "bf16 KV is PRESENT — clean cache won the trade" { + # --kv-cache-dtype auto = bf16 attention KV. Doubles attention-KV + # bytes/token (measured 14,047 vs 7,926) and costs ~14% decode vs fp8 + # at 4064 ctx (49.25 -> 36.42 with the gather win) — traded deliberately + # for clean cache and 262k windows (Frank's call). The fp8 shape is + # restorable from git tag frank-primary-fp8-mtp-20261001. + grep -q '^FRANK_PRIMARY_KV_DTYPE=auto$' "$ENV_FILE" grep -q -- '--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE"' "$SERVE" } +@test "gather geometry 32x64 is PRESENT BY MEASUREMENT" { + # R4D_LRU_GATHER_CHUNKS=32 / R4D_LRU_GATHER_LANES=64: the overnight + # sweep's biggest lever, +18.7% decode vs default geometry, and the win + # GROWS with depth (39.5@166 -> 54.1@1087 -> 49.3@4064 in the fp8 era). + # g64x128 is past the peak; g8x16 mild. They are container ENV (the + # fork's knobs), not engine flags — passed through with -e. + grep -q '^FRANK_PRIMARY_GATHER_CHUNKS=32$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_GATHER_LANES=64$' "$ENV_FILE" + grep -q -- '-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"' "$SERVE" +} + +@test "MNBT is locked at 1024" { + # Frank's lock: MNBT 512 doubles TTFT at 4064 ctx (2.46 s vs 1.36 s); + # 1024-4096 are interchangeable on decode; 12288 costs 2.5% plus arena. + # 1024 keeps the workspace floor so the expert arena stays max. + grep -q '^FRANK_PRIMARY_MNBT=1024$' "$ENV_FILE" + grep -q -- '--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"' "$SERVE" +} + @test "expert prefetch is absent BY LAW, not by omission" { # Telemetry (2026-09-29): the prefetcher served 13,369 of 36,720 # prefetched slabs (36% precision) — roughly half the gather traffic was