From b877d875886b24c9218e83e53f54275b2364dbcd Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 30 Sep 2026 17:41:19 -0700 Subject: [PATCH] patches: glm53-dflash-01 v5 -- patch the CONDITIONAL wrapper (the real eagle3 target) The v4 boot named the actual miss: model_runner.py:425 calls set_eagle3_aux_hidden_state_layers(self.model) on the instance model_runner LOADED -- and for GLM-5.3-Flash that is Glm5NextForConditionalGeneration (the multimodal wrapper; APIServer log: 'Resolved architecture: Glm5NextForConditionalGeneration'), which wraps the text model as self.language_model = Glm5NextForCausalLM (glm5next/common/model.py:1113). v4 put the three presence-checked attributes only on Glm5NextForCausalLM, so the isinstance check on the conditional still failed with the same RuntimeError. v5 adds to BOTH wrappers, MRO-safely (attribute route, no Protocol base): - Glm5NextForConditionalGeneration: supports_eagle3 / has_own_lm_head / has_own_embed_tokens + the two methods delegating to self.language_model (the pixtral.py:498 pattern); - Glm5NextForCausalLM: the same three attributes + the two methods that unwrap to self.model (the EagleModelMixin trunk -- v1's correct bodies). Also proven by MRO simulation: the ForCausalLM's five-base ordering cannot take SupportsEagle3 as a base in ANY position, so inheritance was never a viable route for this model. base 008eeaa2... -> patched bc9913a1..., apply-verified against the pinned image (patch -p1 reproduces the sha256 and matches the mounted file). --- .../glm53-dflash-01-aux-hidden-states.patch | 113 ++++++++++++------ 1 file changed, 77 insertions(+), 36 deletions(-) diff --git a/scripts/patches/glm53-dflash-01-aux-hidden-states.patch b/scripts/patches/glm53-dflash-01-aux-hidden-states.patch index aaba9eb..27cafc2 100644 --- a/scripts/patches/glm53-dflash-01-aux-hidden-states.patch +++ b/scripts/patches/glm53-dflash-01-aux-hidden-states.patch @@ -17,46 +17,47 @@ # `glm5next` implements neither half of the contract. # # THE CONTRACT (read out of this image, not assumed): -# * interfaces.py:1663 -- `supports_eagle3: ClassVar[Literal[True]] = True`. THE FLAG -# IS A REQUIRED MEMBER for runtime-checkable presence-based isinstance, so v1 -# (methods only) failed identically. BUT v2 (inheriting the Protocol) taught the -# sharper lesson: adding SupportsEagle3 to Glm5NextForCausalLM's bases passes -# isinstance and then dies at class creation with +# * DEFINITIVE (v5, after four failed boots): the member set is checked against +# the INSTANCE the runner calls -- which for GLM-5.3-Flash is +# Glm5NextForConditionalGeneration, NOT Glm5NextForCausalLM. The multimodal +# wrapper wraps the text model as self.language_model +# (init_vllm_registered_model, architectures=["Glm5NextForCausalLM"], +# glm5next/common/model.py:1113), and model_runner.py:425 calls +# set_eagle3_aux_hidden_state_layers(self.model) on the CONDITIONAL. +# Presence rule (verified in-image with stock typing, python 3.14): +# isinstance walks ALL __protocol_attrs__ -- members of SupportsEagle3 AND +# its base SupportsEagleBase (has_own_lm_head, has_own_embed_tokens) -- and +# getattr_static()s each one off the instance, breaking on the first miss. +# So BOTH wrappers need all three attributes. +# * MRO law: INHERITING SupportsEagle3 as a base dies at class creation with # "Cannot create a consistent method resolution order (MRO) for bases ..." -# -- because GLM's outer wrapper already inherits vLLM Protocols that are NOT -# Protocol-derived mixins in a compatible order. The fix that avoids the MRO -# entirely: declare the flag as a class attribute on the wrapper: -# supports_eagle3: ClassVar[Literal[True]] = True -# which satisfies presence-based isinstance without touching the MRO, and pairs -# with the EagleModelMixin on the trunk that supplies the real aux-layer plumbing. -# CAUGHT LATE BY ANOTHER BOOT: the presence check walks ALL `__protocol_attrs__` -# INCLUDING base-protocol members (typing._ProtocolMeta.__instancecheck__ loops -# `getattr_static(instance, attr)` over every member and breaks on the first -# AttributeError). SupportsEagleBase declares `has_own_lm_head: bool = False` and -# `has_own_embed_tokens: bool = False` (plain instance-var annotations), so the -# flag ALONE still fails isinstance -- replicated in-image with stock typing -# (flag-only class -> False; flag + the two base members -> True). v4 therefore -# declares all three on the wrapper: supports_eagle3, has_own_lm_head, -# has_own_embed_tokens. False is the right value for the two: the wrapper -# delegates embed/lm_head to its trunk. -# * interfaces.py:1673-1719 -- the inherited defaults unwrap -# `holder = getattr(parent_ref, "model", parent_ref)`, assert -# `isinstance(holder, EagleModelMixin)`, set `holder._set_aux_hidden_state_layers`, and -# default to (2, L // 2, L - 3). So the INNER trunk must inherit EagleModelMixin, -# which supplies `aux_hidden_state_layers` and the slot helpers. +# (GLM's wrappers already carry four/five Protocol-derived bases). The +# attribute route avoids the MRO entirely. Also proven by simulation: +# the four-base ForCausalLM ordering cannot take SupportsEagle3 in ANY +# position. +# * Method placement: without inheriting, the Protocol's default +# set_aux_hidden_state_layers / get_eagle3_default_aux_hidden_state_layers +# do not exist on either wrapper, so both are defined explicitly -- +# ForCausalLM unwraps to self.model (the EagleModelMixin trunk, v1's +# correct bodies), the conditional delegates to self.language_model +# (the pixtral.py:498 pattern). +# * interfaces.py:1673-1719 -- for reference, the inherited defaults unwrap +# `holder = getattr(parent_ref, "model", parent_ref)` and assert +# `isinstance(holder, EagleModelMixin)`, defaulting to (2, L // 2, L - 3). # * eagle3_utils.py:26-31 -- layers come from the spec config when it carries them # (this drafter ships target_layer_ids [5, 14, 24, 33, 42]), else from the default. # * deepseek_v2.py:1551-1585 and v1/worker/gpu/cudagraph_utils.py:621 -- the trunk # appends one tensor per aux layer while looping and returns the TUPLE # (hidden_states, aux_hidden_states) only when that list is non-empty. # -# WHAT IT ADDS for GLM: the mixin on Glm5NextModel; aux capture in the layer loop, where -# the aux state is the single-stream `hidden_states` ENTERING that layer (GLM's loop -# carries the already-summed single-stream state -- its non-mHC branch re-splits -# `residual = hidden_states` inside the layer -- so this is the equivalent of -# deepseek_v2's `hidden_states + residual`); the conditional tuple return; and -# the three SupportsEagle3 member attributes on Glm5NextForCausalLM (the flag plus -# SupportsEagleBase's two defaults). +# WHAT IT ADDS for GLM: EagleModelMixin on Glm5NextModel; aux capture in the layer +# loop, where the aux state is the single-stream `hidden_states` ENTERING that layer +# (GLM's loop carries the already-summed single-stream state -- its non-mHC branch +# re-splits `residual = hidden_states` inside the layer -- so this is the equivalent +# of deepseek_v2's `hidden_states + residual`); the conditional tuple return; the +# three SupportsEagle3 member attributes on BOTH wrappers; and the two interface +# methods on both (ForCausalLM: unwrap to trunk; conditional: delegate to +# language_model). # # HONEST SCOPE: this makes the interface PRESENT, which is exactly what the engine # checks -- it does not make drafts GOOD, and per the measured profile it is not a @@ -66,12 +67,12 @@ # # TARGET: vllm/models/glm5next/common/model.py (inside the pinned image) # base sha256 008eeaa2a07e091465319f86900b5a321d921238723ccbe22e6b9e971e1d2b28 -# patched sha256 60f27aad36ab9610d335bd0d9a2e177d8e26b6af1e81b3a31adf5f5cca8cab68 +# patched sha256 bc9913a15c8e7f88e0c36b07bf029b383cc8c2078b58d66dc91aa235f8af77f4 # # APPLY: `patch -p1` from the vllm package root, or bind-mount the patched file over # site-packages (which is what a scratch-launcher DFLASH_PATCH=1 does). ---- orig/vllm/models/glm5next/common/model.py 2026-09-30 15:57:08.910029614 -0700 -+++ new/vllm/models/glm5next/common/model.py 2026-09-30 17:22:36.124257958 -0700 +--- a/vllm/models/glm5next/common/model.py ++++ b/vllm/models/glm5next/common/model.py @@ -95,6 +95,12 @@ logger = init_logger(__name__) @@ -130,3 +131,43 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() self.model_config = vllm_config.model_config +@@ -963,6 +982,14 @@ + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + ++ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: ++ # EAGLE-3 / DFlash2 interface: unwrap to the trunk (the EagleModelMixin). ++ self.model.aux_hidden_state_layers = layers ++ ++ def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]: ++ num_layers = len(self.model.layers) ++ return (2, num_layers // 2, num_layers - 3) ++ + def forward( + self, + input_ids: torch.Tensor | None, +@@ -1033,6 +1060,10 @@ + class Glm5NextForConditionalGeneration( + Glm4vForConditionalGeneration, HasInnerState, IsHybrid, MixtureOfExperts + ): ++ supports_eagle3: ClassVar[Literal[True]] = True ++ has_own_lm_head: bool = False ++ has_own_embed_tokens: bool = False ++ + # The text model (KDA + dense-MLA + MoE) is a hybrid mamba model. The + # multimodal wrapper must declare the same interfaces so vLLM treats it as + # hybrid (auto-aligns mamba/attention block sizes, sizes the mamba state +@@ -1076,6 +1107,13 @@ + + return Glm5NextForCausalLM.get_mamba_state_copy_func() + ++ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: ++ # EAGLE-3 / DFlash2: delegate to the language model (pixtral.py:498). ++ self.language_model.set_aux_hidden_state_layers(layers) ++ ++ def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]: ++ return self.language_model.get_eagle3_default_aux_hidden_state_layers() ++ + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super(Glm4vForConditionalGeneration, self).__init__() + config = vllm_config.model_config.hf_config