diff --git a/scripts/patches/glm53-dflash-01-aux-hidden-states.patch b/scripts/patches/glm53-dflash-01-aux-hidden-states.patch new file mode 100644 index 0000000..21f84fc --- /dev/null +++ b/scripts/patches/glm53-dflash-01-aux-hidden-states.patch @@ -0,0 +1,117 @@ +# glm53-dflash-01-aux-hidden-states.patch — give the GLM trunk the EAGLE3 / DFlash2 +# auxiliary-hidden-state interface. +# +# STATUS: WIP, and INDEPENDENT of the compile changelist (01-05). It patches the same +# file as glm53-compile-01-model-decorator.patch, so if both are ever applied, apply 01 +# first. Not part of the deployed shape. +# +# WHY. `--speculative-config '{"method":"dflash","model":"/path/to/drafter", ...}'` with +# incoai/GLM-5.3-Flash-DFlash2 is accepted -- the config parses, the drafter loads -- and +# then the worker dies at init with: +# +# RuntimeError: Model does not support EAGLE3 interface +# +# raised by v1/worker/gpu/spec_decode/eagle/eagle3_utils.py:20 through +# `supports_eagle3(model)` == `isinstance(model, SupportsEagle3)` (a runtime-checkable +# Protocol). Nothing about the drafter, the method name or the payload is wrong: +# `glm5next` implements neither half of the contract. +# +# THE CONTRACT (read out of this image, not assumed): +# * interfaces.py:1694 -- `holder = getattr(parent_ref, "model", parent_ref)` then +# `assert isinstance(holder, EagleModelMixin)` then +# `holder._set_aux_hidden_state_layers(layers)`. The INNER trunk must inherit +# EagleModelMixin, which supplies `aux_hidden_state_layers` and the slot helpers. +# * interfaces.py:1712-1719 -- the default layer set is (2, L // 2, L - 3) and needs +# the holder to expose `layers`. +# * eagle3_utils.py:26-31 -- layers come from the spec config when it carries them +# (this drafter ships target_layer_ids [5, 14, 24, 33, 42], mapped to trunk +# indices), else from get_eagle3_default_aux_hidden_state_layers(). +# * deepseek_v2.py:1551-1585 and v1/worker/gpu/cudagraph_utils.py:621 -- the trunk +# appends one tensor per aux layer while looping and returns the TUPLE +# (hidden_states, aux_hidden_states) only when that list is non-empty. +# +# WHAT IT ADDS for GLM: the mixin on Glm5NextModel; aux capture in the layer loop, where +# the aux state is the single-stream `hidden_states` ENTERING that layer (GLM's loop +# carries the already-summed single-stream state -- its non-mHC branch re-splits +# `residual = hidden_states` inside the layer -- so this is the equivalent of +# deepseek_v2's `hidden_states + residual`); the conditional tuple return; and +# set_aux_hidden_state_layers / get_eagle3_default_aux_hidden_state_layers on +# Glm5NextForCausalLM, delegating to the inner trunk (the same two-liner shape +# kimi_k25.py:466 and pixtral.py:503 use). +# +# HONEST SCOPE: this makes the interface PRESENT, which is exactly what the engine +# checks -- it does not make drafts GOOD, and per the measured profile it is not a +# decode-speed fix (a verification pass pays pointwise cost proportional to its token +# count, which is why MTP4 measured flat). It is the prerequisite for driving DFlash2 +# on this endpoint at all. +# +# TARGET: vllm/models/glm5next/common/model.py (inside the pinned image) +# base sha256 008eeaa2a07e091465319f86900b5a321d921238723ccbe22e6b9e971e1d2b28 +# patched sha256 4ff6ff8ddf8af3d74bef699f98fc95a99ee47586f6e785248e7dc4b683b72d7a +# +# APPLY: `patch -p1` from the vllm package root, or bind-mount the patched file over +# site-packages (which is what a scratch-launcher DFLASH_PATCH=1 does). +--- a/vllm/models/glm5next/common/model.py ++++ b/vllm/models/glm5next/common/model.py +@@ -95,6 +95,9 @@ + logger = init_logger(__name__) + + ++from vllm.model_executor.models.interfaces import EagleModelMixin ++ ++ + class Glm5NextMLP(nn.Module): + def __init__( + self, +@@ -623,7 +626,7 @@ + ) + + +-class Glm5NextModel(nn.Module): ++class Glm5NextModel(nn.Module, EagleModelMixin): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + +@@ -730,7 +733,14 @@ + if self.is_sequence_parallel: + hidden_states = sp_shard(hidden_states) + +- for layer in self._active_layers: ++ aux_hidden_states: list[torch.Tensor] = [] ++ for local_idx, layer in enumerate(self._active_layers): ++ if (getattr(self, "start_layer", 0) + local_idx) in self.aux_hidden_state_layers: ++ # Auxiliary hidden state for EAGLE-3 / DFlash2 drafts: the ++ # single-stream state ENTERING this layer, before its input norm -- ++ # the same semantic deepseek_v2 implements by appending ++ # hidden_states + residual at this point in its loop. ++ aux_hidden_states.append(hidden_states) + hidden_states, residual, post, comb = layer( + positions, hidden_states, residual, post, comb + ) +@@ -749,6 +759,8 @@ + hidden_states = sp_all_gather(hidden_states)[:full_num_tokens] + + hidden_states = self.norm(hidden_states) ++ if len(aux_hidden_states) > 0: ++ return hidden_states, aux_hidden_states + return hidden_states + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: +@@ -963,6 +975,17 @@ + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + ++ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None: ++ # EAGLE-3 / DFlash2 interface (interfaces.py asserts the holder inherits ++ # EagleModelMixin, which supplies the slot plumbing behind this attribute). ++ self.model.aux_hidden_state_layers = layers ++ ++ def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]: ++ # Only used when the spec config carries no explicit aux layers; the ++ # DFlash2 drafters this fleet runs ship target_layer_ids instead. ++ num_layers = len(self.model.layers) ++ return (2, num_layers // 2, num_layers - 3) ++ + def forward( + self, + input_ids: torch.Tensor | None,