# glm53-dflash-01-aux-hidden-states.patch — give the GLM trunk the EAGLE3 / DFlash2 # auxiliary-hidden-state interface. # # STATUS: WIP, and INDEPENDENT of the compile changelist (01-05). It patches the same # file as glm53-compile-01-model-decorator.patch, so if both are ever applied, apply 01 # first. Not part of the deployed shape. # # WHY. `--speculative-config '{"method":"dflash","model":"/path/to/drafter", ...}'` with # incoai/GLM-5.3-Flash-DFlash2 is accepted -- the config parses, the drafter loads -- and # then the worker dies at init with: # # RuntimeError: Model does not support EAGLE3 interface # # raised by v1/worker/gpu/spec_decode/eagle/eagle3_utils.py:20 through # `supports_eagle3(model)` == `isinstance(model, SupportsEagle3)` (a runtime-checkable # Protocol). Nothing about the drafter, the method name or the payload is wrong: # `glm5next` implements neither half of the contract. # # THE CONTRACT (read out of this image, not assumed): # * DEFINITIVE (v5, after four failed boots): the member set is checked against # the INSTANCE the runner calls -- which for GLM-5.3-Flash is # Glm5NextForConditionalGeneration, NOT Glm5NextForCausalLM. The multimodal # wrapper wraps the text model as self.language_model # (init_vllm_registered_model, architectures=["Glm5NextForCausalLM"], # glm5next/common/model.py:1113), and model_runner.py:425 calls # set_eagle3_aux_hidden_state_layers(self.model) on the CONDITIONAL. # Presence rule (verified in-image with stock typing, python 3.14): # isinstance walks ALL __protocol_attrs__ -- members of SupportsEagle3 AND # its base SupportsEagleBase (has_own_lm_head, has_own_embed_tokens) -- and # getattr_static()s each one off the instance, breaking on the first miss. # So BOTH wrappers need all three attributes. # * Why not inheritance: adding SupportsEagle3 as a BASE class dies at # class creation with "Cannot create a consistent method resolution # order (MRO) for bases ..." (GLM's wrappers already carry four/five # Protocol-derived bases; the base ordering cannot take another). The # attribute route avoids the MRO entirely; simulation confirms that # the four-base ForCausalLM ordering cannot take SupportsEagle3 in ANY # position. # * Method placement: without inheriting, the Protocol's default # set_aux_hidden_state_layers / get_eagle3_default_aux_hidden_state_layers # do not exist on either wrapper, so both are defined explicitly -- # ForCausalLM unwraps to self.model (the EagleModelMixin trunk, v1's # correct bodies), the conditional delegates to self.language_model # (the pixtral.py:498 pattern). # * interfaces.py:1673-1719 -- for reference, the inherited defaults unwrap # `holder = getattr(parent_ref, "model", parent_ref)` and assert # `isinstance(holder, EagleModelMixin)`, defaulting to (2, L // 2, L - 3). # * eagle3_utils.py:26-31 -- layers come from the spec config when it carries them # (this drafter ships target_layer_ids [5, 14, 24, 33, 42]), else from the default. # * deepseek_v2.py:1551-1585 and v1/worker/gpu/cudagraph_utils.py:621 -- the trunk # appends one tensor per aux layer while looping and returns the TUPLE # (hidden_states, aux_hidden_states) only when that list is non-empty. # # WHAT IT ADDS for GLM: EagleModelMixin on Glm5NextModel; aux capture in the layer # loop, where the aux state is the single-stream `hidden_states` ENTERING that layer # (GLM's loop carries the already-summed single-stream state -- its non-mHC branch # re-splits `residual = hidden_states` inside the layer -- so this is the equivalent # of deepseek_v2's `hidden_states + residual`); the conditional tuple return; the # three SupportsEagle3 member attributes on BOTH wrappers; and the two interface # methods on both (ForCausalLM: unwrap to trunk; conditional: delegate to # language_model). # # HONEST SCOPE: this makes the interface PRESENT, which is exactly what the engine # checks -- it does not make drafts GOOD, and per the measured profile it is not a # decode-speed fix (a verification pass pays pointwise cost proportional to its token # count, which is why MTP4 measured flat). It is the prerequisite for driving DFlash2 # on this endpoint at all. # # TARGET: vllm/models/glm5next/common/model.py (inside the pinned image) # base sha256 008eeaa2a07e091465319f86900b5a321d921238723ccbe22e6b9e971e1d2b28 # patched sha256 bc9913a15c8e7f88e0c36b07bf029b383cc8c2078b58d66dc91aa235f8af77f4 # # APPLY: `patch -p1` from the vllm package root, or bind-mount the patched file over # site-packages (which is what a scratch-launcher DFLASH_PATCH=1 does). --- a/vllm/models/glm5next/common/model.py +++ b/vllm/models/glm5next/common/model.py @@ -95,6 +95,12 @@ logger = init_logger(__name__) +from vllm.model_executor.models.interfaces import ( + EagleModelMixin, + SupportsEagle3, +) + + class Glm5NextMLP(nn.Module): def __init__( self, @@ -623,7 +629,7 @@ ) -class Glm5NextModel(nn.Module): +class Glm5NextModel(nn.Module, EagleModelMixin): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -730,7 +736,14 @@ if self.is_sequence_parallel: hidden_states = sp_shard(hidden_states) - for layer in self._active_layers: + aux_hidden_states: list[torch.Tensor] = [] + for local_idx, layer in enumerate(self._active_layers): + if (getattr(self, "start_layer", 0) + local_idx) in self.aux_hidden_state_layers: + # Auxiliary hidden state for EAGLE-3 / DFlash2 drafts: the + # single-stream state ENTERING this layer, before its input norm -- + # the same semantic deepseek_v2 implements by appending + # hidden_states + residual at this point in its loop. + aux_hidden_states.append(hidden_states) hidden_states, residual, post, comb = layer( positions, hidden_states, residual, post, comb ) @@ -749,6 +762,8 @@ hidden_states = sp_all_gather(hidden_states)[:full_num_tokens] hidden_states = self.norm(hidden_states) + if len(aux_hidden_states) > 0: + return hidden_states, aux_hidden_states return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: @@ -937,6 +952,10 @@ class Glm5NextForCausalLM( nn.Module, HasInnerState, SupportsPP, MixtureOfExperts, IsHybrid ): + supports_eagle3: ClassVar[Literal[True]] = True + has_own_lm_head: bool = False + has_own_embed_tokens: bool = False + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() self.model_config = vllm_config.model_config @@ -963,6 +982,14 @@ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) + def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: + # EAGLE-3 / DFlash2 interface: unwrap to the trunk (the EagleModelMixin). + self.model.aux_hidden_state_layers = layers + + def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]: + num_layers = len(self.model.layers) + return (2, num_layers // 2, num_layers - 3) + def forward( self, input_ids: torch.Tensor | None, @@ -1033,6 +1060,10 @@ class Glm5NextForConditionalGeneration( Glm4vForConditionalGeneration, HasInnerState, IsHybrid, MixtureOfExperts ): + supports_eagle3: ClassVar[Literal[True]] = True + has_own_lm_head: bool = False + has_own_embed_tokens: bool = False + # The text model (KDA + dense-MLA + MoE) is a hybrid mamba model. The # multimodal wrapper must declare the same interfaces so vLLM treats it as # hybrid (auto-aligns mamba/attention block sizes, sizes the mamba state @@ -1076,6 +1107,13 @@ return Glm5NextForCausalLM.get_mamba_state_copy_func() + def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: + # EAGLE-3 / DFlash2: delegate to the language model (pixtral.py:498). + self.language_model.set_aux_hidden_state_layers(layers) + + def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]: + return self.language_model.get_eagle3_default_aux_hidden_state_layers() + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super(Glm4vForConditionalGeneration, self).__init__() config = vllm_config.model_config.hf_config