From fd864b5a425fec4e16d9f7bcdabc986694a13e2a Mon Sep 17 00:00:00 2001 From: Comrade Chief Designer Date: Wed, 30 Sep 2026 13:59:09 -0700 Subject: [PATCH] =?UTF-8?q?frank-primary:=20compile=20changelist=20?= =?UTF-8?q?=E2=80=94=20real=20diffs,=20break=20diagnostics,=20patch=2004,?= =?UTF-8?q?=20corrected=20overclaim?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Working the dyno name-map blocker one step at a time, with the tools to see it: - All four patch artifacts are now genuine diff -u patches (they were prose + pseudo hunks before, useless to patch(1)). Verified against the pinned image: each applies clean and reproduces its recorded patched sha256 (6e9cb81b… / 0751852e… / 732363bb… / 84969993…). - New: scripts/patches/glm53-compile-04-fused-qkv-rmsnorm-optout.patch — takes GLM's MLA block off the fused-norm JIT kernel (R4D_GLM_FUSE_QKV_RMSNORM=0; the flag is already a constructor arg glm5next sets at attention.py:583). Reason: in models/common/ops/fused_qk_rmsnorm.py the kernel object is bound at line 223, AFTER its use at line 211 — the launcher family (@kernel_launcher, 39 call sites) is what breaks dynamo's break analysis. - New diagnostics, both reusable and recorded: TORCH_LOGS=graph_breaks + VLLM_USE_AOT_COMPILE=0 to name each break with its user stack, and /home/dank/dynamo-probe (sitecustomize.py on PYTHONPATH, VLLM_DYNAMO_DUMP_ATTRS=1) to print the dynamo name-map miss detail. Gotchas: a .pth on PYTHONPATH is NOT executed, and the scratch launcher reset ENVARGS (now ${ENVARGS:-}), which silently dropped env vars and wasted a boot. Measurement discipline correction: I first recorded the 'Read uninitialized cell' and 'Unsupported method call' failures as fatal blockers. They are not established as such — they appear only when TORCH_LOGS=graph_breaks is set, while a flag-free break-tolerance run gets past break handling and dies on the name-map KeyError. The likely mechanism is that graph_breaks logging makes dynamo build a break *explanation*, and that analysis itself trips on the uninitialized cell. Patch 04's header, the runbook and the work log now carry the confound explicitly rather than the overclaim. tests/run.sh: 362 seams pass. --- .../glm53-compile-01-model-decorator.patch | 25 ++++- .../glm53-compile-02-trace-safe-probes.patch | 7 +- .../glm53-compile-03-fullgraph-optout.patch | 17 ++- ...-compile-04-fused-qkv-rmsnorm-optout.patch | 106 ++++++++++++++++++ .../LLM/frank-primary-glm53-flash-w4a16.md | 50 +++++++-- serverAppConfigs-docs/log.md | 34 ++++-- 6 files changed, 220 insertions(+), 19 deletions(-) create mode 100644 scripts/patches/glm53-compile-04-fused-qkv-rmsnorm-optout.patch diff --git a/scripts/patches/glm53-compile-01-model-decorator.patch b/scripts/patches/glm53-compile-01-model-decorator.patch index 675190f..2e44cf1 100644 --- a/scripts/patches/glm53-compile-01-model-decorator.patch +++ b/scripts/patches/glm53-compile-01-model-decorator.patch @@ -50,7 +50,25 @@ # the first probe call inside the traced region. --- a/vllm/models/glm5next/common/model.py +++ b/vllm/models/glm5next/common/model.py -@@ Glm5NextModel: add @support_torch_compile (env): R4D_GLM_COMPILE=0 disables @@ +@@ -1,12 +1,14 @@ + # SPDX-License-Identifier: Apache-2.0 + # SPDX-FileCopyrightText: Copyright contributors to the vLLM project + ++import os + from collections.abc import Iterable + from typing import ClassVar, Literal + + import torch + from torch import nn + ++from vllm.compilation.decorators import support_torch_compile + from vllm.config import ParallelConfig, VllmConfig + from vllm.distributed import ( + get_ep_group, +@@ -623,6 +625,18 @@ + ) + + +_GLM_COMPILE = os.environ.get("R4D_GLM_COMPILE", "1") == "1" + + @@ -63,5 +81,6 @@ + }, + enable_if=lambda _cfg: _GLM_COMPILE, +) --class Glm5NextModel(nn.Module): -+class Glm5NextModel(nn.Module): # decorated above + class Glm5NextModel(nn.Module): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() diff --git a/scripts/patches/glm53-compile-02-trace-safe-probes.patch b/scripts/patches/glm53-compile-02-trace-safe-probes.patch index fdaffb3..4077bfb 100644 --- a/scripts/patches/glm53-compile-02-trace-safe-probes.patch +++ b/scripts/patches/glm53-compile-02-trace-safe-probes.patch @@ -45,7 +45,12 @@ # (see 03) — dynamo reports "Dynamo bytecode transform time: 1.08 s" for the trunk. --- a/vllm/model_executor/kernels/r4d_lib.py +++ b/vllm/model_executor/kernels/r4d_lib.py -@@ append: trace-safe capability probes (env): R4D_FREEZE_PROBES=0 disables @@ +@@ -361,3 +361,71 @@ + @functools.lru_cache + def moe_w4a16_mt1(): + return getattr(lib(), "r4d_gemm_moe_w4a16_nt_b16_m64_mt1", None) ++ ++ +# FORK-LOCAL (patch: trace-safe capability probes). torch.compile traces the +# BODY of a functools.lru_cache call even when the cache is warm, so a capability +# probe called from a forward pass is a dynamo graph break. Measured 2026-09-30 diff --git a/scripts/patches/glm53-compile-03-fullgraph-optout.patch b/scripts/patches/glm53-compile-03-fullgraph-optout.patch index d4b49d8..6350466 100644 --- a/scripts/patches/glm53-compile-03-fullgraph-optout.patch +++ b/scripts/patches/glm53-compile-03-fullgraph-optout.patch @@ -67,7 +67,11 @@ # is silently dropped — add the var inside the script or to the branch itself. --- a/vllm/compilation/wrapper.py +++ b/vllm/compilation/wrapper.py -@@ fullgraph opt-out (env VLLM_COMPILE_FULLGRAPH=0) @@ +@@ -19,6 +19,15 @@ + + logger = init_logger(__name__) + ++ +# FORK-LOCAL (patch: fullgraph opt-out). torch.compile(fullgraph=True) turns any +# dynamo graph break into a hard failure; suppress_errors does NOT cover breaks +# (verified in-image, torch 2.11+rocm10.0). With VLLM_COMPILE_FULLGRAPH=0 the @@ -75,5 +79,16 @@ +# eagerly, which is what a model with device-sync metadata (the GLM-5.3-Flash +# indexer) needs. Default 1 keeps stock behaviour for every other model. +_FULLGRAPH = os.environ.get("VLLM_COMPILE_FULLGRAPH", "1") == "1" ++ + R = TypeVar("R") + P = ParamSpec("P") + +@@ -146,7 +155,7 @@ + with aot_context: + self._compiled_callable = torch.compile( + compiled_ptr, - fullgraph=True, + fullgraph=_FULLGRAPH, + dynamic=False, + backend=backend, + options=options, diff --git a/scripts/patches/glm53-compile-04-fused-qkv-rmsnorm-optout.patch b/scripts/patches/glm53-compile-04-fused-qkv-rmsnorm-optout.patch new file mode 100644 index 0000000..0a63597 --- /dev/null +++ b/scripts/patches/glm53-compile-04-fused-qkv-rmsnorm-optout.patch @@ -0,0 +1,106 @@ +# glm53-compile-04-fused-qkv-rmsnorm-optout.patch — take GLM's fused-norm JIT kernel +# out of the traced path. +# +# STATUS: WIP — fourth of the "compile the GLM trunk" changelist. Needs 01 (the +# decorator) to matter; not part of the deployed shape. +# +# MEASUREMENT NOTE — READ BEFORE TRUSTING ANY CLAIM BELOW. Two break-tolerance runs +# disagree, and the difference is a logging flag: +# * 01+02+03, AOT off, breaks tolerated, NO TORCH_LOGS -> gets PAST all graph-break +# handling and dies on the dynamo name-map KeyError ('in_proj_qkvbfg_a'). +# * same + TORCH_LOGS=graph_breaks -> dies on the fatal +# `Unsupported: Read uninitialized cell` at this very call site. +# The likely mechanism is that `graph_breaks` logging makes dynamo build a break +# *explanation* (walk the frames, name the reason), and that analysis is what trips on +# the uninitialized cell; without the flag the break is taken silently. So the cell +# error is PROBABLY an artifact of the diagnostic, not necessarily a blocker for a +# break-tolerant compile. Whether this patch is therefore necessary — as opposed to +# removing a call dynamo cannot *explain* — is being decided by a flag-free run of +# 01+02+03+04. Do not repeat the earlier overclaim ("fatal even with fullgraph=False"); +# it was written from the confounded run. +# +# IT IS STILL THE RIGHT PATTERN TO REMOVE, either way: the launcher family +# (@kernel_launcher, 39 call sites) is exactly what breaks dynamo's break analysis, and +# the object it closes over is bound after use (see below). +# +# THE BREAK (measured, TORCH_LOGS=graph_breaks, rank0, 2026-09-30). With 01+02+03 the +# trunk traces, and with AOT off + fullgraph=True dynamo reports a FATAL break — fatal +# even with VLLM_COMPILE_FULLGRAPH=0, because dynamo cannot build a resume function +# around it: +# +# torch._dynamo.exc.Unsupported: Read uninitialized cell +# Explanation: Attempted to read a cell variable that has not been populated yet. +# Developer debug context: CellVariable() +# +# User stack: +# glm5next/common/model.py:748 Glm5NextModel.forward layer(...) +# glm5next/common/model.py:526 Glm5NextDecoderLayer.forward +# glm5next/common/attention.py:590 +# model_executor/layers/mla.py:208 +# q_proj_input, kv_c_normed = fused_q_kv_rmsnorm(...) +# -> torch/_dynamo/polyfills/__init__.py:392 getattr_and_trace +# +# So dynamo is tracing the fork's JIT-kernel launcher framework +# (VllmTritonJitKernel + @kernel_launcher — 39 call sites) and finds a closure name +# that is not yet bound. In models/common/ops/fused_qk_rmsnorm.py the kernel object +# is bound AFTER the function that closes over it: +# +# def fused_q_kv_rmsnorm(...): # line 194 +# ... +# _FUSED_Q_KV_RMSNORM_KERNEL(...) # line 211 +# _FUSED_Q_KV_RMSNORM_KERNEL = FusedQKVRMSNormKernel() # line 223 +# +# Two ways out, and this patch takes the cheap one: +# (a) WORKAROUND (here): stop calling the fused kernel from the traced region. The +# flag is already a constructor argument (mla.py:80, default False) that glm5next +# turns ON at attention.py:583, so it becomes an env knob. +# (b) ROOT FIX (not done): make the launcher trace-safe for all 39 sites. Any other +# @kernel_launcher-backed kernel still in the traced path can produce the same +# fatal break, so (b) is the durable fix and (a) is what unblocks iterating. +# +# COST of R4D_GLM_FUSE_QKV_RMSNORM=0: one extra small RMSNorm launch per MLA layer per +# token (q_a_layernorm + kv_a_layernorm instead of one fused kernel) — ~2 launches out +# of ~3,700 per token on this shape, against the 44 % of device time currently spent in +# unfused pointwise work. Correct trade for a compile experiment. +# +# TARGET: vllm/models/glm5next/common/attention.py (inside the pinned image) +# base sha256 362bbf0e6c50cdfa446c6551b9b1ddff425e08f88e3d4140fd8db88f8a619e0c +# patched sha256 84969993c250e9f8183a33a6e1ddd607272f48b950b560272eb33b7cdde31189 +# +# APPLY: `patch -p1` from the vllm package root (verified: applies clean, reproduces +# the patched sha256), or bind-mount new/.../attention.py over site-packages. +# R4D_GLM_FUSE_QKV_RMSNORM=1 (default) restores the fused kernel. +--- a/vllm/models/glm5next/common/attention.py ++++ b/vllm/models/glm5next/common/attention.py +@@ -1,6 +1,7 @@ + # SPDX-License-Identifier: Apache-2.0 + # SPDX-FileCopyrightText: Copyright contributors to the vLLM project + ++import os + import torch + import torch.nn.functional as F + from torch import nn +@@ -36,6 +37,14 @@ + from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES + from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec + ++ ++# FORK-LOCAL (patch: keep the traced region trace-safe). R4D_GLM_FUSE_QKV_RMSNORM=0 ++# replaces the fused `fused_q_kv_rmsnorm` JIT-kernel launch (one kernel) with two plain ++# RMSNorms, because dynamo cannot trace the launcher's closure ("Read uninitialized cell", ++# fatal even with fullgraph=False). The fused kernel object in ++# models/common/ops/fused_qk_rmsnorm.py is bound AFTER the function that uses it. ++_FUSE_QKV_RMSNORM = os.environ.get("R4D_GLM_FUSE_QKV_RMSNORM", "1") == "1" ++ + logger = init_logger(__name__) + + # Shared torch.compile config for the indexer's small-kernel leaves. The MLA +@@ -580,7 +589,7 @@ + quant_config, + prefix, + skip_topk=False, +- fuse_qkv_rmsnorm=True, ++ fuse_qkv_rmsnorm=_FUSE_QKV_RMSNORM, + ) + + def forward( diff --git a/serverAppConfigs-docs/LLM/frank-primary-glm53-flash-w4a16.md b/serverAppConfigs-docs/LLM/frank-primary-glm53-flash-w4a16.md index 4f6be6e..3d11759 100644 --- a/serverAppConfigs-docs/LLM/frank-primary-glm53-flash-w4a16.md +++ b/serverAppConfigs-docs/LLM/frank-primary-glm53-flash-w4a16.md @@ -193,13 +193,49 @@ What the boots established, in order (each one ~7 min): `serve_glm_autoround.sh`'s COMPILE_PATCH branch rather than in the outer environment). -That last row is the state of the work: the changelist is one unresolved -submodule-name lookup away from booting, and it must be cleared (plus whatever -follows it) before the compile payoff can be measured. Two verified facts make the -rest of the shape concrete: `suppress_errors` does **not** cover graph breaks (only -`fullgraph=False` does), and the GLM trunk can never satisfy `fullgraph` because -its indexer metadata syncs to host -(`glm5next/amd/sparse_indexer.py:263,419`: `int(...max().item())`). +4. **04 removes the fused-norm call from the traced region** (`R4D_GLM_FUSE_QKV_RMSNORM=0`: + the GLM MLA block stops calling the fork's fused-norm JIT kernel, whose kernel + object in `models/common/ops/fused_qk_rmsnorm.py` is bound *after* the function + that uses it — `_FUSED_Q_KV_RMSNORM_KERNEL` at line 223 vs its use at line 211). + **Its necessity is not yet established** — see the confound below. +5. **A confound worth knowing about, because it cost two boots.** Two break-tolerance + runs of the same config disagree, and the difference is one logging flag: + `01+02+03` with `TORCH_LOGS=graph_breaks` dies on a fatal + `Unsupported: Read uninitialized cell` at `mla.py:208` (the fused-norm call), while + the same config **without** the flag gets past break handling entirely and dies + later on the dynamo name-map `KeyError`. The likely mechanism: `graph_breaks` + logging makes dynamo build an *explanation* for each break (walk frames, name the + reason), and that analysis is what trips on the uninitialized cell — without the + flag, the break is taken silently. So that fatal-looking cell error is probably an + artifact of the diagnostic. Related: with `graph_breaks` on, the next reported + failure is `Unsupported: Unsupported method call` at + `sparse_indexer.py:132 sparse_attn_indexer_kpool → workspace.py:135 + get_simultaneous / :155 _ensure_workspace_size`, i.e. the indexer allocates its + shared workspace *inside* the forward (its own comment: *"will allocate on first + use"*) — also flagged under explanation, also not yet shown to block a plain + break-tolerant compile. + +**How to see the breaks (and its trap).** `TORCH_LOGS=graph_breaks` names each break +with a user stack, and `VLLM_USE_AOT_COMPILE=0` is required because AOT compile +hard-refuses breaks — but per the confound above, treat the flagged fatal errors as +*candidates*, and confirm each one against a flag-free run before believing it blocks +the boot. For the name-map KeyError there is a ready-made probe: put +`/home/dank/dynamo-probe` on `PYTHONPATH` with `VLLM_DYNAMO_DUMP_ATTRS=1` (a +`sitecustomize.py` — a `.pth` on `PYTHONPATH` is NOT executed, and the image's own +`sitecustomize` is chained so its apport hook survives) and it prints, at the miss, +the node target, the near-miss keys, and whether another live `OutputGraph` holds +the key. + +**State of the changelist:** the trunk traces (`Dynamo bytecode transform time: +1.08 s`), and the one failure confirmed by a flag-free run is the dynamo name-map +`KeyError: 'in_proj_qkvbfg_a'`. Two further candidates are queued behind it +(the fused-norm launcher cell, the indexer's in-forward workspace allocation). Two +verified facts bound the rest of the shape: `suppress_errors` does **not** cover +graph breaks (only `fullgraph=False` does), and the GLM trunk can never satisfy +`fullgraph` because its indexer metadata syncs to host +(`glm5next/amd/sparse_indexer.py:263,419`: `int(...max().item())`) — both are dead +fallbacks on the decode path that the fork author already replaced, so they are not +the blockers they appear to be at first glance. ### Two structural constraints on any further fork work diff --git a/serverAppConfigs-docs/log.md b/serverAppConfigs-docs/log.md index a88f694..3d7dc8e 100644 --- a/serverAppConfigs-docs/log.md +++ b/serverAppConfigs-docs/log.md @@ -74,13 +74,33 @@ `RuntimeError: Model does not support EAGLE3 interface` — nothing about the drafter, the method or the payload is wrong; the trunk simply does not implement the aux-hidden-state contract. - - **Compile blocker narrowed to dynamo's name map.** The `KeyError: - 'in_proj_qkvbfg_a'` (see above) is thrown from `torch/_dynamo/output_graph.py`'s - `nn_modules` name lookup (:3000, registered at :1259/:1384, with `_modules[..]` - prefixes stripped at :1225) — i.e. the compiler cannot resolve a submodule by - name while tracing the GLM decoder layer. Next step is `TORCHDYNAMO_VERBOSE=1` - (torch names it in the error), noting the scratch launcher resets `ENVARGS` so - it must be set inside the script's COMPILE_PATCH branch. + - **Compile work: one confirmed blocker, plus a measurement confound worth + recording.** Flag-free break-tolerance runs (01+02+03, AOT off, breaks allowed) + get past break handling and die on the dynamo name-map `KeyError: + 'in_proj_qkvbfg_a'`. Adding `TORCH_LOGS=graph_breaks` to the same config changes + where it dies: a fatal-looking `Unsupported: Read uninitialized cell` at + `mla.py:208` calling the fork's fused-norm JIT kernel + (`models/common/ops/fused_qk_rmsnorm.py`: `_FUSED_Q_KV_RMSNORM_KERNEL` is bound at + line 223, *after* its use at line 211), and with that removed, `Unsupported: + Unsupported method call` in the indexer's in-forward workspace allocation + (`sparse_indexer.py:132 -> workspace.py:135 get_simultaneous / :155 + _ensure_workspace_size`). **The likely mechanism is that `graph_breaks` logging + makes dynamo build a break *explanation*, and that analysis is what trips** — so + those two are candidates, not proven blockers, and I overclaimed them once before + catching it. `scripts/patches/glm53-compile-04-fused-qkv-rmsnorm-optout.patch` takes + GLM off the fused-norm kernel anyway (`R4D_GLM_FUSE_QKV_RMSNORM=0`, a constructor + flag glm5next sets at `attention.py:583`); whether it is *needed* is what the + flag-free 01+02+03+04 run settles. + - **Diagnostics built for this (reusable).** `TORCH_LOGS=graph_breaks` + + `VLLM_USE_AOT_COMPILE=0` names each break with its user stack (AOT compile + hard-refuses breaks outright); `/home/dank/dynamo-probe` on `PYTHONPATH` with + `VLLM_DYNAMO_DUMP_ATTRS=1` prints the dynamo name-map miss detail (node target, + near-miss keys, whether another live `OutputGraph` holds the key). Two gotchas + recorded: a `.pth` on `PYTHONPATH` is NOT executed (use `sitecustomize.py`, + chaining the image's apport hook), and the scratch launcher used to reset + `ENVARGS` (now `${ENVARGS:-}`), which silently dropped env vars and wasted a boot. + All four patch artifacts are now genuine `diff -u` patches, verified to apply to + the pinned image and to reproduce their recorded sha256. - Measured and rejected: **MTP4** (the head loads, arch resolves to `Glm5NextMTPModel`, acceptance is excellent at mean length 3.76-4.90 — but decode was unchanged (9.6-11.2 vs 10.0) while prefill COLLAPSED 918 -> 255.